增加功能——添加标题去重

增加组件  pyredis

application/common/redis/pyredis.py

@@ -0,0 +1,33 @@
+"""
+Redis client Python
+@author luojunhui
+"""
+
+import redis
+
+
+class RedisClient(object):
+    """
+    Redis client by python
+    """
+
+    def __init__(self):
+        self.host = 'r-bp1mb0v08fqi4hjffu.redis.rds.aliyuncs.com'
+        self.port = 6379
+        self.db = 2
+        self.password = 'Wqsd@2019'
+        self.pool = redis.Redis(host=self.host, port=self.port, db=self.db, password=self.password)
+
+    def select(self, key):
+        """
+        read info from redis
+        :return:
+        """
+        return self.pool.get(key)
+
+    def insert(self, key, value):
+        """
+        insert info from redis
+        :return:
+        """
+        self.pool.set(key, value)

application/pipeline/pipeline.py

@@ -131,7 +131,12 @@ class PiaoQuanPipeline(object):
         if self.platform == "jixiangxingfu" and self.mode == "recommend":
             return True
         out_id = self.item["out_video_id"]
-        sql = f""" select 1 from crawler_video where platform = "{self.platform}" and out_video_id="{out_id}"; """
+        # 判断加上标题去重
+        if self.mode == "recommend" and self.platform == "yuannifuqichangzai":
+            title = self.item["video_title"]
+            sql = f""" select 1 from crawler_video where platform = "{self.platform}" and out_video_id="{out_id}" and video_title="{title}"; """
+        else:
+            sql = f""" select 1 from crawler_video where platform = "{self.platform}" and out_video_id="{out_id}"; """
         repeat_video = self.mysql.select(sql=sql)
         if repeat_video:
             # 喜事多多平台 4 天去重一次