调试

spider/crawler_online/benshanzhufu.py

@@ -58,14 +58,14 @@ class BSZHRecommend(object):
         """
         url = "http://47.236.68.175:8889/crawler/ben_shan_zhu_fu/recommend"
         next_cursor = 1
-        for i in range(5):
+        for i in range(1, 200):
             payload = json.dumps({
                 "cursor": f"{next_cursor}"
             })
             headers = {
                 'Content-Type': 'application/json'
             }
-            for i in range(1, 200):
+            for j in range(3):
                 response = requests.request("POST", url, headers=headers, data=payload)
                 response = response.json()
                 if response['code'] != 0:

spider/crawler_online/xiaoniangaotuijianliu.py

@@ -59,32 +59,33 @@ class XNGTJLRecommend(object):
         headers = {
             'Content-Type': 'application/json'
         }
-        url = "http://47.236.68.175:8889/crawler/xiao_nian_gao_plus/recommend"
-        payload = json.dumps({})
-        response = requests.request("POST", url, headers=headers, data=payload)
-        response = response.json()
-        if response['code'] != 0:
-            self.aliyun_log.logging(
-                code="3000",
-                message="抓取单条视频失败,请求失败"
-            ),
-            return
-        for index, video_obj in enumerate(response['data']['data'], 1):
-            try:
-                self.aliyun_log.logging(
-                    code="1001", message="扫描到一条视频", data=video_obj
-                )
-                self.process_video_obj(video_obj)
-            except Exception as e:
+        for i in range(3):
+            url = "http://47.236.68.175:8889/crawler/xiao_nian_gao_plus/recommend"
+            payload = json.dumps({})
+            response = requests.request("POST", url, headers=headers, data=payload)
+            response = response.json()
+            if response['code'] != 0:
                 self.aliyun_log.logging(
                     code="3000",
-                    message="抓取单条视频失败, 该视频位于第{}页第{}条报错原因是{}".format(
-                        1, index, e
-                    ),
-                )
-            if self.limit_flag:
+                    message="抓取单条视频失败,请求失败"
+                ),
                 return
-            time.sleep(random.randint(5, 10))
+            for index, video_obj in enumerate(response['data']['data'], 1):
+                try:
+                    self.aliyun_log.logging(
+                        code="1001", message="扫描到一条视频", data=video_obj
+                    )
+                    self.process_video_obj(video_obj)
+                except Exception as e:
+                    self.aliyun_log.logging(
+                        code="3000",
+                        message="抓取单条视频失败, 该视频位于第{}页第{}条报错原因是{}".format(
+                            1, index, e
+                        ),
+                    )
+                if self.limit_flag:
+                    return
+                time.sleep(random.randint(5, 10))
 
     def process_video_obj(self, video_obj):
         """

spider/crawler_online/zhufuquanzituijianliu.py

@@ -58,32 +58,33 @@ class ZFQZTJLRecommend(object):
         headers = {
             'Content-Type': 'application/json'
         }
-        url = "http://47.236.68.175:8889/crawler/zhu_fu_quan_zi/recommend"
-        payload = json.dumps({})
-        response = requests.request("POST", url, headers=headers, data=payload)
-        response = response.json()
-        if  response['code'] != 0:
-            self.aliyun_log.logging(
-                code="3000",
-                message="抓取单条视频失败,请求失败"
-                ),
-            return
-        for index, video_obj in enumerate(response['data']['data'], 1):
-            try:
-                self.aliyun_log.logging(
-                    code="1001", message="扫描到一条视频", data=video_obj
-                )
-                self.process_video_obj(video_obj)
-            except Exception as e:
+        for i in range(2):
+            url = "http://47.236.68.175:8889/crawler/zhu_fu_quan_zi/recommend"
+            payload = json.dumps({})
+            response = requests.request("POST", url, headers=headers, data=payload)
+            response = response.json()
+            if  response['code'] != 0:
                 self.aliyun_log.logging(
                     code="3000",
-                    message="抓取单条视频失败, 该视频位于第{}页第{}条报错原因是{}".format(
-                        1, index, e
+                    message="抓取单条视频失败,请求失败"
                     ),
-                )
-            if self.limit_flag:
                 return
-            time.sleep(random.randint(5, 10))
+            for index, video_obj in enumerate(response['data']['data'], 1):
+                try:
+                    self.aliyun_log.logging(
+                        code="1001", message="扫描到一条视频", data=video_obj
+                    )
+                    self.process_video_obj(video_obj)
+                except Exception as e:
+                    self.aliyun_log.logging(
+                        code="3000",
+                        message="抓取单条视频失败, 该视频位于第{}页第{}条报错原因是{}".format(
+                            1, index, e
+                        ),
+                    )
+                if self.limit_flag:
+                    return
+                time.sleep(random.randint(5, 10))
 
     def process_video_obj(self, video_obj):
         """