修改小年糕 V2 的退出逻辑

xiaoniangao/xiaoniangao_author/xiaoniangao_author_test.py

@@ -101,7 +101,9 @@ class XiaoNianGaoAuthor:
                     try:
                         message = f"扫描到一条视频"
                         print(message)
-                        self.process_video_obj(video_obj, user_dict)
+                        flag = self.process_video_obj(video_obj, user_dict)
+                        if not flag:
+                            return
                     except Exception as e:
                         message = "抓取单条视频异常, 报错原因是: {}".format(e)
                         print(message)
@@ -160,7 +162,15 @@ class XiaoNianGaoAuthor:
             "strategy": self.mode,
             "out_video_id": video_obj.get("vid", ""),
         }
-        print(video_dict)
+        if (
+                int(time.time()) - publish_time_stamp
+                > 3600 * 24 * int(self.rule_dict.get("period", {}).get("max", 1000))
+        ):
+            message = "发布时间超过{}天".format(
+                int(self.rule_dict.get("period", {}).get("max", 1000))
+            )
+            print(message)
+            return False
         pipeline = PiaoQuanPipelineTest(
             platform=self.platform,
             mode=self.mode,
@@ -176,11 +186,12 @@ class XiaoNianGaoAuthor:
             video_dict["crawler_rule"] = json.dumps(self.rule_dict)
             video_dict["user_id"] = user_dict["uid"]
             video_dict["publish_time"] = video_dict["publish_time_str"]
-            # print(video_dict)
-            self.mq.send_msg(video_dict)
+            print(video_dict)
+            # self.mq.send_msg(video_dict)
             self.download_count += 1
             message = "成功发送 MQ 至 ETL"
             print(message)
+        return True
 
 
 if __name__ == "__main__":

xiaoniangao/xiaoniangao_author/xiaoniangao_author_v2.py

@@ -168,7 +168,9 @@ class XiaoNianGaoAuthor:
                             env=self.env,
                             message=f"扫描到一条视频"
                         )
-                        self.process_video_obj(video_obj, user_dict)
+                        date_flag = self.process_video_obj(video_obj, user_dict)
+                        if not date_flag:
+                            return
                     except Exception as e:
                         AliyunLogger.logging(
                             code="3000",
@@ -239,8 +241,22 @@ class XiaoNianGaoAuthor:
             "strategy": self.mode,
             "out_video_id": video_obj.get("vid", ""),
         }
-        for k, v in video_dict.items():
-            Common.logger(self.mode, self.platform).info(f"{k}:{v}")
+        if (
+                int(time.time()) - publish_time_stamp
+                > 3600 * 24 * int(self.rule_dict.get("period", {}).get("max", 1000))
+        ):
+            AliyunLogger.logging(
+                code="2004",
+                trace_id=trace_id,
+                platform=self.platform,
+                mode=self.mode,
+                env=self.env,
+                data=video_dict,
+                message="发布时间超过{}天".format(
+                    int(self.rule_dict.get("period", {}).get("max", 1000))
+                ),
+            )
+            return False
         pipeline = PiaoQuanPipeline(
             platform=self.platform,
             mode=self.mode,
@@ -249,6 +265,7 @@ class XiaoNianGaoAuthor:
             item=video_dict,
             trace_id=trace_id,
         )
+        # try:
         flag = pipeline.process_item()
         if flag:
             video_dict["width"] = video_dict["video_width"]
@@ -274,6 +291,7 @@ class XiaoNianGaoAuthor:
                 env=self.env,
                 message="成功发送 MQ 至 ETL",
             )
+        return True
 
 # if __name__ == "__main__":
 #     XNGA = XiaoNianGaoAuthor(