修改万物复苏代码

zhuwanwufusu/zhuwanwufusu_recommend/zwwfs_recommend.py

@@ -24,7 +24,7 @@ class ZhuWanWuFuSuRecommend(object):
         self.env = env
         self.download_cnt = 0
         self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
-        self.expire_flag = False
+        self.limit_flag = False
         self.cryptor = AES()
 
     def get_recommend_list(self):
@@ -141,7 +141,6 @@ class ZhuWanWuFuSuRecommend(object):
                                                    proxies=tunnel_proxies())
                 detail_video = json.loads(self.cryptor.aes_decrypt(detail_response.text))
                 if detail_video['success']:
-                    # print(json.dumps(detail_video, ensure_ascii=False, indent=4))
                     try:
                         AliyunLogger.logging(
                             code="1001",
@@ -196,8 +195,7 @@ class ZhuWanWuFuSuRecommend(object):
         )
         if pipeline.process_item():
             self.download_cnt += 1
-            print(mq_obj)
-            # self.mq.send_msg(mq_obj)
+            self.mq.send_msg(mq_obj)
             AliyunLogger.logging(
                 code="1002",
                 platform=self.platform,
@@ -206,6 +204,8 @@ class ZhuWanWuFuSuRecommend(object):
                 message="成功发送至 ETL",
                 data=mq_obj
             )
+            if self.download_cnt >= int(self.rule_dict.get("videos_cnt", {}).get("min", 300)):
+                self.limit_flag = True
 
     def manage_auth_id(self, out_user_id, out_user_name):
         """
@@ -273,6 +273,15 @@ class ZhuWanWuFuSuRecommend(object):
         if user_list:
             for index, user_id in enumerate(user_list):
                 try:
+                    if self.limit_flag:
+                        AliyunLogger.logging(
+                            code="2000",
+                            platform=self.platform,
+                            mode=self.mode,
+                            env=self.env,
+                            message="本轮已经抓取足够数量的视频"
+                        )
+                        return
                     self.get_user_videos(user_id=user_id)
                 except Exception as e:
                     AliyunLogger.logging(