中青看点到第二天小于半小时停止任务

spider/crawler_author/zhongqingkandian_author.py

@@ -5,7 +5,7 @@ import random
 import uuid
 import time
 import traceback
-from datetime import datetime
+from datetime import datetime, timedelta
 import requests
 
 
@@ -198,7 +198,7 @@ class ZhongQingKanDianAuthor:
                     time.sleep(10)
                     continue
                 for user_info in self.zqkd_user_list:
-                    if not self.limit_flag:
+                    if not self.limit_flag :
                         self.aliyun_log.logging(
                             code="2010",
                             message=f"今日已经达到最大量",
@@ -206,6 +206,10 @@ class ZhongQingKanDianAuthor:
                         )
                         self.LocalLog.info("视频数量已达到预期")
                         return
+                    if is_less_than_30_minutes():
+                        self.LocalLog.info("时间已不足，停止执行")
+                        return
+
                     current_id, user_id = user_info
                     author_resp = self.req_user_list(user_id)
                     self.redis_ops.set_last_scanned_id(current_id)
@@ -402,7 +406,12 @@ class ZhongQingKanDianAuthor:
         """
         self.LocalLog.info("开始执行中青看点用户视频抓取...")
         self.control_request_author()
-
+def is_less_than_30_minutes():
+    now = datetime.now()
+    tomorrow = now.date() + timedelta(days=1)
+    midnight = datetime.combine(tomorrow, datetime.min.time())
+    time_left = midnight - now
+    return time_left.total_seconds() < 30 * 60
 
 if __name__ == '__main__':
     ZhongQingKanDianAuthor(
@@ -410,4 +419,4 @@ if __name__ == '__main__':
         mode="author",
         rule_dict={'videos_cnt': {'min': 2, 'max': 0}, 'duration': {'min': 30, 'max': 1200}},
         user_list=[{"uid": 81525568, "link": "中青看点推荐", "nick_name": "芸芸众生"}]
-    ).run()
+    ).run()

spider/crawler_online/zhongqingkandian.py

@@ -184,6 +184,9 @@ class ZhongQingKanDianRecommend:
                     )
                     self.LocalLog.info(f"当日视频已达到最大爬取量{self.download_cnt}")
                     return
+                if is_less_than_30_minutes():
+                    self.LocalLog.info("时间已不足，停止执行")
+                    return
                 self.LocalLog.info(f"开始推荐视频列表的请求和处理流程,今日已爬推荐 {self.download_cnt} 个视频")
                 recommend_resp = self.req_recommend_list()
                 if not recommend_resp:
@@ -343,7 +346,12 @@ class ZhongQingKanDianRecommend:
         """
         self.LocalLog.info("开始执行中青看点推荐抓取...")
         self.control_request_recommend()
-
+def is_less_than_30_minutes():
+    now = datetime.now()
+    tomorrow = now.date() + timedelta(days=1)
+    midnight = datetime.combine(tomorrow, datetime.min.time())
+    time_left = midnight - now
+    return time_left.total_seconds() < 30 * 60
 
 if __name__ == '__main__':
     ZhongQingKanDianRecommend(

spider/crawler_online/zhongqingkandian_related_recommend.py

@@ -190,7 +190,9 @@ class ZhongQingKanDianRelatedRecommend:
                     return
 
                 self.LocalLog.info(f"开始推荐视频列表的请求和处理流程,今日已爬 {self.download_cnt} 个视频")
-
+                if is_less_than_30_minutes():
+                    self.LocalLog.info("时间已不足，停止执行")
+                    return
                 content_id = self.redis_ops.get_recommend_video()
                 if not content_id:
                     self.LocalLog.info("缓存中【task:zqkd_video_id】没有数据")
@@ -335,7 +337,12 @@ class ZhongQingKanDianRelatedRecommend:
         self.LocalLog.info("开始执行中青看点相关推荐抓取...")
         self.control_request_related()
 
-
+def is_less_than_30_minutes():
+    now = datetime.now()
+    tomorrow = now.date() + timedelta(days=1)
+    midnight = datetime.combine(tomorrow, datetime.min.time())
+    time_left = midnight - now
+    return time_left.total_seconds() < 30 * 60
 if __name__ == '__main__':
     ZhongQingKanDianRelatedRecommend(
         platform="zhongqingkandianrelated",