update

common/public.py

@@ -29,9 +29,9 @@ def filter_word(log_type, crawler, source, env):
     return word_list
 
 
-def get_user_from_mysql(log_type, crawler, source, env):
+def get_user_from_mysql(log_type, crawler, source, env, action=''):
     sql = f"select * from crawler_author_map where source='{source}' and is_del=1"
-    results = MysqlHelper.get_values(log_type, crawler, sql, env, action='get_author_map')
+    results = MysqlHelper.get_values(log_type, crawler, sql, env, action=action)
     if results:
         return results
     else:

kuaishou/kuaishou_follow/kuaishou_follow.py

@@ -741,7 +741,7 @@ class KuaiShouFollow:
     def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env, machine):
         # user_list = cls.get_user_list(log_type=log_type, crawler=crawler, sheetid="bTSzxW", env=env, machine=machine)
 
-        user_list = get_user_from_mysql(log_type, crawler, crawler, env, machine)
+        user_list = get_user_from_mysql(log_type, crawler, crawler, env)
         for user in user_list:
             spider_link = user["spider_link"]
             out_uid = spider_link.split('/')[-1]

main/process.sh

@@ -137,7 +137,7 @@ if [ "$?" -eq 1 ];then
   if [ ${env} = "dev" ];then
     cd ${piaoquan_crawler_dir} && sh main/scheduling_main.sh ./youtube/youtube_main/run_youtube_follow.py --log_type="follow" --crawler="youtube" --env="dev" youtube/follow.log
   else
-    cd ${piaoquan_crawler_dir} && /usr/bin/sh ./main/main.sh ./youtube/youtube_main/run_youtube_follow.py --log_type="follow" --crawler="youtube" --strategy="定向爬虫策略" --oss_endpoint="hk" --env="prod" --machine="aliyun_hk" youtube/follow.log
+    cd ${piaoquan_crawler_dir} && /usr/bin/sh ./main/main.sh ./youtube/youtube_main/run_youtube_follow.py --log_type="follow" --crawler="youtube" --strategy="定向爬虫策略" --oss_endpoint="hk" --env="hk" --machine="aliyun_hk" youtube/follow.log
   fi
   echo "$(date "+%Y-%m-%d %H:%M:%S") 重启完成!\n" >> ${log_path}
 else

xigua/xigua_follow/xigua_follow.py

@@ -998,7 +998,7 @@ class Follow:
     def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env, machine):
         try:
             # user_list = cls.get_user_list(log_type=log_type, crawler=crawler, sheetid="5tlTYB", env=env, machine=machine)
-            user_list = get_user_from_mysql(log_type, crawler, crawler, env, machine)
+            user_list = get_user_from_mysql(log_type, crawler, crawler, env)
 
             for user in user_list:
                 spider_link = user["spider_link"]

youtube/youtube_follow/youtube_follow_api.py

@@ -1147,7 +1147,7 @@ class YoutubeFollow:
     def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env, machine):
         try:
             # user_list = cls.get_user_from_feishu(log_type, crawler, 'c467d7', env, machine)
-            user_list = get_user_from_mysql(log_type, crawler, crawler, env)
+            user_list = get_user_from_mysql(log_type, crawler, crawler, env, action='get_author_map')
             if len(user_list) == 0:
                 Common.logger(log_type, crawler).warning('用户列表为空\n')
             else: