Przeglądaj źródła

视频刷刷增加去重逻辑

zhangyong 1 rok temu
rodzic
commit
5c04ed3443
1 zmienionych plików z 9 dodań i 0 usunięć
  1. 9 0
      spider/crawler_offline/shipinshuashua_id.py

+ 9 - 0
spider/crawler_offline/shipinshuashua_id.py

@@ -151,6 +151,10 @@ class SPSSIdRecommend:
             return
         video_id = re.sub(r'\D', '', video_id)
         video_id = int(video_id)
+        repeat_video_id = self.repeat_video_id(video_id)
+        if repeat_video_id != 0:
+            self.driver.press_keycode(AndroidKey.BACK)
+            return
         data_list = self.get_user_list(video_id)
         if len(data_list) == 0:
             self.driver.press_keycode(AndroidKey.BACK)
@@ -169,6 +173,11 @@ class SPSSIdRecommend:
         print(insert_sql)
         MysqlHelper(self.env).update(insert_sql)
 
+    def repeat_video_id(self, mid):
+        sql = f"SELECT `link`  FROM `crawler_user_v3` WHERE  `source` = 'xiaoniangao'  and `link` = {mid}"
+        repeat_video_id = MysqlHelper(self.env).select(sql)
+        return len(repeat_video_id)
+
     def get_user_list(self, mid):
         next_t = -1
         url = "https://kapi-xng-app.xiaoniangao.cn/v1/album/user_public"