update

common/public.py

@@ -31,7 +31,7 @@ def filter_word(log_type, crawler, source, env):
 
 
 def get_user_from_mysql(log_type, crawler, source, env, action=''):
-    sql = f"select * from crawler_author_map where source='{source}' and task_type='{log_type}'  and is_del=1"
+    sql = f"select * from crawler_user_v3 where source='{source}' and mode='{log_type}'"
     results = MysqlHelper.get_values(log_type, crawler, sql, env, action=action)
     if results:
         return results

douyin/douyin_follow/follow_dy.py

@@ -337,10 +337,10 @@ class DyFollow(object):
         rule_dict = cls.get_rule(log_type, crawler)
 
         for user in user_list:
-            spider_link = user["spider_link"]
+            spider_link = user["link"]
             out_uid = spider_link
             user_name = user["nick_name"]
-            our_uid = user["media_id"]
+            our_uid = user["uid"]
             Common.logger(log_type, crawler).info(f"开始抓取 {user_name} 用户主页视频\n")
             cls.get_videoList(log_type=log_type,
                               crawler=crawler,

kuaishou/kuaishou_follow/kuaishou_follow.py

@@ -649,10 +649,10 @@ class KuaiShouFollow:
 
         user_list = get_user_from_mysql(log_type, crawler, crawler, env)
         for user in user_list:
-            spider_link = user["spider_link"]
+            spider_link = user["link"]
             out_uid = spider_link.split('/')[-1]
             user_name = user["nick_name"]
-            our_uid = user["media_id"]
+            our_uid = user["uid"]
             Common.logger(log_type, crawler).info(f"开始抓取 {user_name} 用户主页视频\n")
             cls.get_videoList(log_type=log_type,
                               crawler=crawler,

xigua/xigua_follow/xigua_follow.py

@@ -1018,10 +1018,10 @@ class Follow:
     def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env, machine):
         user_list = get_user_from_mysql(log_type, crawler, crawler, env)
         for user in user_list:
-            spider_link = user["spider_link"]
+            spider_link = user["link"]
             out_uid = spider_link.split('/')[-1]
             user_name = user["nick_name"]
-            our_uid = user["media_id"]
+            our_uid = user["uid"]
 
             Common.logger(log_type, crawler).info(f"开始抓取 {user_name} 用户主页视频\n")
             cls.get_videolist(log_type=log_type,

youtube/youtube_follow/youtube_follow_api.py

@@ -1098,10 +1098,10 @@ class YoutubeFollow:
                 Common.logger(log_type, crawler).warning('用户列表为空\n')
             else:
                 for user_dict in user_list:
-                    out_user_url = user_dict['spider_link']
+                    out_user_url = user_dict['link']
                     out_uid = out_user_url.split('/')[3]
                     user_name = user_dict['nick_name']
-                    our_uid = user_dict['media_id']
+                    our_uid = user_dict['uid']
                     Common.logger(log_type, crawler).info(f'获取 {user_name} 主页视频\n')
                     cls.get_videos(log_type, crawler, strategy, oss_endpoint, env, out_uid, our_uid, machine,
                                    out_user_url)