经典福气旺增加redis

common/redis_db.py

@@ -34,10 +34,13 @@ class SyncRedisHelper:
             self._pool.disconnect(inuse_connections=True)
 
 
-def store_data(platform, out_video_id):
+def store_data(platform, out_video_id, condition, day_time):
     key = f"crawler:duplicate:{platform}:{out_video_id}"
     value = 1
-    timeout = timedelta(days=60)  # 60天超时时间
+    if condition:
+        timeout = timedelta(days=int(day_time))
+    else:
+        timeout = timedelta(hours=int(day_time))
     helper = SyncRedisHelper()
     client = helper.get_client()
 
@@ -54,7 +57,7 @@ def get_data(platform, out_video_id):
 
 
 # 示例：存储一个数据
-# store_data('xiaoniangao', '123457')
+# store_data('xiaoniangao', '123457', True, 60)
 
 # 示例：获取一个数据
 # value = get_data('xiaoniangao', '1234857')

jingdianfuqiwang/jingdianfuqiwang_recommend/jingdianfuqiwang_recommend_scheduling.py

@@ -11,6 +11,7 @@ sys.path.append(os.getcwd())
 from common.video_item import VideoItem
 from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies
 from common.mq import MQ
+from common.redis_db import get_data, store_data
 
 
 class TFuQiWangRecommend(object):
@@ -122,6 +123,14 @@ class TFuQiWangRecommend(object):
         item.add_video_info("strategy", self.mode)
         item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
         mq_obj = item.produce_item()
+        value = get_data(self.platform, video_obj.get("vid", ""))
+        if int(value) == 1:
+            AliyunLogger.logging(
+                code="2004",
+                platform=self.platform,
+                mode=self.mode,
+                env=self.env,
+                message="redis重复视频")
         pipeline = PiaoQuanPipeline(
             platform=self.platform,
             mode=self.mode,
@@ -133,6 +142,7 @@ class TFuQiWangRecommend(object):
         if pipeline.process_item():
             self.download_cnt += 1
             self.mq.send_msg(mq_obj)
+            store_data(self.platform, video_obj.get("vid", ""), False, 12)
             time.sleep(60 * random.randint(1, 5))
             AliyunLogger.logging(
                 code="1002",

xiaoniangao/xiaoniangao_author/xiaoniangao_author_v2.py

@@ -310,7 +310,7 @@ class XiaoNianGaoAuthor:
             limit_flag = self.limiter.author_limitation(user_id=video_dict['user_id'])
             if limit_flag:
                 self.mq.send_msg(video_dict)
-                store_data(self.platform, video_obj.get("vid", ""))
+                store_data(self.platform, video_obj.get("vid", ""), True, 60)
                 Common.logging(
                     log_type=self.mode,
                     crawler=self.platform,