Merge branch 'master' of https://git.yishihui.com/Server/crawler_shipinhao

.gitignore

@@ -127,6 +127,7 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
+.DS_Store
 
 # Pyre type checker
 .pyre/

main/publish.py

@@ -8,6 +8,7 @@
 import json
 import os
 import random
+import shutil
 import time
 import oss2
 import requests
@@ -254,4 +255,6 @@ class Publish:
                 else:
                     Common.logger(log_type).error('file not a dir = {}'.format(fi_d))
             except Exception as e:
+                # 删除视频文件夹
+                shutil.rmtree("./videos/" + f + "/")
                 Common.logger(log_type).exception('upload_and_publish error', e)

main/shipinhao_get_url.py

@@ -51,8 +51,8 @@ class GetUrl:
             Common.logger(log_type).info('点击视频')
             driver.find_elements(By.NAME, '消息')[-1].click()
 
-            Common.logger(log_type).info('休眠 8 秒，退出视频号')
-            time.sleep(8)
+            Common.logger(log_type).info('休眠 10 秒，退出视频号')
+            time.sleep(10)
             cls.kill_pid(log_type)
 
             Common.logger(log_type).info('退出微信')

main/shipinhao_recommend.py

@@ -71,7 +71,7 @@ class Recommend:
             cls.quit(log_type, driver)
 
         except Exception as e:
-            Common.logger(log_type).error('start_wechat异常\n', e)
+            Common.logger(log_type).error('start_wechat异常:{}\n', e)
 
     # 退出 APP
     @classmethod
@@ -197,17 +197,37 @@ class Recommend:
                     Common.logger(log_type).info('不满足抓取规则，滑动到下一个视频\n')
                     driver.swipe(10, 1600, 10, 300, 200)
 
+                # 过滤词库(视频标题)
+                elif any(word if word in video_title else False for word in cls.filter_words(log_type)) is True:
+                    Common.logger(log_type).info('视频已中过滤词:{}\n', video_title)
+                    driver.swipe(10, 1600, 10, 300, 200)
+
+                # 黑名单(用户名)
+                elif any(word if word in user_name else False for word in cls.username_blacklist(log_type)) is True:
+                    Common.logger(log_type).info('用户名已中过滤词:{}\n', user_name)
+                    driver.swipe(10, 1600, 10, 300, 200)
+
                 # 连续下载判断
                 elif user_name in Feishu.get_values_batch('recommend', 'shipinhao', 'c77cf9')[1][14] \
                         and user_name in Feishu.get_values_batch('recommend', 'shipinhao', 'c77cf9')[2][14]:
                     Common.logger(log_type).info('该用户已连续下载2条视频，滑动到下一个视频\n')
                     driver.swipe(10, 1600, 10, 300, 200)
 
-                # 已下载表去重
+                # 视频号推荐_已下载表
                 elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x in
                                           y]:
                     Common.logger(log_type).info('视频已下载，滑动到下一个视频\n')
                     driver.swipe(10, 1600, 10, 300, 200)
+                # 新视推荐_已下载表
+                elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'WAG7Dq') for x in
+                                          y]:
+                    Common.logger(log_type).info('视频已下载，滑动到下一个视频\n')
+                    driver.swipe(10, 1600, 10, 300, 200)
+                # 新视热门_已下载表
+                elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', '0i4jmV') for x
+                                          in y]:
+                    Common.logger(log_type).info('视频已下载，滑动到下一个视频\n')
+                    driver.swipe(10, 1600, 10, 300, 200)
 
                 # feeds 表去重
                 elif str(video_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy') for x in
@@ -243,11 +263,11 @@ class Recommend:
                             Common.logger(log_type).info('等待更新 URL 信息')
                             time.sleep(10)
                         else:
-                            Common.logger(log_type).info('URL 信息已更新，滑动到下一个视频\n')
-                            driver.swipe(10, 1600, 10, 300, 200)
+                            Common.logger(log_type).info('URL 信息已更新\n')
                             break
 
-                cls.download_publish(log_type, env)
+                    cls.download_publish(log_type, env)
+                    driver.swipe(10, 1600, 10, 300, 200)
 
         except Exception as e:
             Common.logger(log_type).error('get_feeds异常，滑动到下一个视频\n', e)
@@ -350,14 +370,47 @@ class Recommend:
                     # 删除行或列，可选 ROWS、COLUMNS
                     time.sleep(1)
                     Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", i + 1, i + 1)
-                    Common.logger(log_type).info("下载/上传成功:{}\n", download_title)
+                    Common.logger(log_type).info("下载/上传成功，滑动到下一个视频")
                     return
         except Exception as e:
             Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", 2, 2)
             Common.logger(log_type).error('download_publish异常，删除视频信息成功:{}\n', e)
 
+    # 过滤词库
+    @classmethod
+    def filter_words(cls, log_type):
+        try:
+            filter_words_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'PYennl')
+            filter_words_list = []
+            for x in filter_words_sheet:
+                for y in x:
+                    if y is None:
+                        pass
+                    else:
+                        filter_words_list.append(y)
+            return filter_words_list
+        except Exception as e:
+            Common.logger(log_type).error('filter_words异常:{}\n', e)
+
+    # 用户名黑名单
+    @classmethod
+    def username_blacklist(cls, log_type):
+        try:
+            username_blacklist_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'nkKrdl')
+            username_blacklist_list = []
+            for x in username_blacklist_sheet:
+                for y in x:
+                    if y is None:
+                        pass
+                    else:
+                        username_blacklist_list.append(y)
+            return username_blacklist_list
+        except Exception as e:
+            Common.logger(log_type).error('filter_words异常:{}\n', e)
+
 
 if __name__ == '__main__':
-    Publish.upload_and_publish('recommend', 'dev', "play")
+    # print(Recommend.username_blacklist('recommend'))
+    Recommend.run_recommend('recommend', 'prod')
 
     pass