上线珊瑚祝福代码

shanhuzhufu/shanhuzhufu_recommend/shanhuzhufu_recommend_dev.py

@@ -10,9 +10,9 @@ import datetime
 
 sys.path.append(os.getcwd())
 from common.video_item import VideoItem
-from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies
+from common import AliyunLogger, tunnel_proxies
+from common.pipeline import PiaoQuanPipelineTest
 from common.mq import MQ
-from common.db import MysqlHelper
 from shanhuzhufu.crypt.decrypt import ShanHuZhuFuAes as AES
 
 
@@ -43,7 +43,7 @@ class ShanHuZhuFuRecommend(object):
             'accept-language': 'en-US,en;q=0.9',
             'Cookie': 'PHPSESSID=562dc39e8e68ad3e76c237f687bd049b; lang=zh-cn'
         }
-        for i in range(10):
+        for i in range(100):
             params = {
                 "mark": "",
                 "page": i + 1
@@ -77,28 +77,19 @@ class ShanHuZhuFuRecommend(object):
         item.add_video_info("strategy", self.mode)
         item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
         mq_obj = item.produce_item()
-        print(json.dumps(mq_obj, ensure_ascii=False, indent=4))
-        # pipeline = PiaoQuanPipeline(
-        #     platform=self.platform,
-        #     mode=self.mode,
-        #     rule_dict=self.rule_dict,
-        #     env=self.env,
-        #     item=mq_obj,
-        #     trace_id=trace_id,
-        # )
-        # if pipeline.process_item():
-        #     self.download_cnt += 1
-        #     self.mq.send_msg(mq_obj)
-        #     AliyunLogger.logging(
-        #         code="1002",
-        #         platform=self.platform,
-        #         mode=self.mode,
-        #         env=self.env,
-        #         message="成功发送至 ETL",
-        #         data=mq_obj
-        #     )
-        #     if self.download_cnt >= int(self.rule_dict.get("videos_cnt", {}).get("min", 200)):
-        #         self.limit_flag = True
+        pipeline = PiaoQuanPipelineTest(
+            platform=self.platform,
+            mode=self.mode,
+            rule_dict=self.rule_dict,
+            env=self.env,
+            item=mq_obj,
+            trace_id=trace_id,
+        )
+        if pipeline.process_item():
+            self.download_cnt += 1
+            print(json.dumps(mq_obj, ensure_ascii=False, indent=4))
+            if self.download_cnt >= int(self.rule_dict.get("videos_cnt", {}).get("min", 200)):
+                self.limit_flag = True
 
 
 if __name__ == '__main__':

shanhuzhufu/shanhuzhufu_recommend/shanhuzhufu_recommend_scheduling.py

@@ -46,33 +46,34 @@ class ShanHuZhuFuRecommend(object):
             try:
                 if self.limit_flag:
                     return
-                params = {
-                    "mark": "",
-                    "page": i + 1
-                }
-                response = requests.get(url=base_url, headers=headers, params=params, proxies=tunnel_proxies())
-                encrypted_info = response.json()['data']
-                decrypted_info = json.loads(self.cryptor.decrypt(data=encrypted_info))
-                video_list = decrypted_info['list']
-                for index, video_obj in enumerate(video_list, 1):
-                    try:
-                        AliyunLogger.logging(
-                            code="1001",
-                            platform=self.platform,
-                            mode=self.mode,
-                            env=self.env,
-                            message="扫描到一条视频",
-                            data=video_obj
-                        )
-                        self.process_video_obj(video_obj)
-                    except Exception as e:
-                        AliyunLogger.logging(
-                            code="3000",
-                            platform=self.platform,
-                            mode=self.mode,
-                            env=self.env,
-                            message="抓取第{}页的时候出现问题， 报错信息是{}".format(index, e)
-                        )
+                else:
+                    params = {
+                        "mark": "",
+                        "page": i + 1
+                    }
+                    response = requests.get(url=base_url, headers=headers, params=params, proxies=tunnel_proxies())
+                    encrypted_info = response.json()['data']
+                    decrypted_info = json.loads(self.cryptor.decrypt(data=encrypted_info))
+                    video_list = decrypted_info['list']
+                    for index, video_obj in enumerate(video_list, 1):
+                        try:
+                            AliyunLogger.logging(
+                                code="1001",
+                                platform=self.platform,
+                                mode=self.mode,
+                                env=self.env,
+                                message="扫描到一条视频",
+                                data=video_obj
+                            )
+                            self.process_video_obj(video_obj)
+                        except Exception as e:
+                            AliyunLogger.logging(
+                                code="3000",
+                                platform=self.platform,
+                                mode=self.mode,
+                                env=self.env,
+                                message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e)
+                            )
             except Exception as e:
                 AliyunLogger.logging(
                     code="3000",