Merge branch 'lierqiang'

kuaishou/kuaishou_follow/kuaishou_follow.py

@@ -649,19 +649,22 @@ class KuaiShouFollow:
 
         user_list = get_user_from_mysql(log_type, crawler, crawler, env)
         for user in user_list:
-            spider_link = user["link"]
-            out_uid = spider_link.split('/')[-1]
-            user_name = user["nick_name"]
-            our_uid = user["uid"]
-            Common.logger(log_type, crawler).info(f"开始抓取 {user_name} 用户主页视频\n")
-            cls.get_videoList(log_type=log_type,
-                              crawler=crawler,
-                              strategy=strategy,
-                              our_uid=our_uid,
-                              out_uid=out_uid,
-                              oss_endpoint=oss_endpoint,
-                              env=env,
-                              machine=machine)
+            try:
+                spider_link = user["link"]
+                out_uid = spider_link.split('/')[-1]
+                user_name = user["nick_name"]
+                our_uid = user["uid"]
+                Common.logger(log_type, crawler).info(f"开始抓取 {user_name} 用户主页视频\n")
+                cls.get_videoList(log_type=log_type,
+                                  crawler=crawler,
+                                  strategy=strategy,
+                                  our_uid=our_uid,
+                                  out_uid=out_uid,
+                                  oss_endpoint=oss_endpoint,
+                                  env=env,
+                                  machine=machine)
+            except Exception as e:
+                continue
 
 
 if __name__ == "__main__":

xigua/xigua_follow/xigua_follow.py

@@ -1018,21 +1018,24 @@ class Follow:
     def get_follow_videos(cls, log_type, crawler, strategy, oss_endpoint, env, machine):
         user_list = get_user_from_mysql(log_type, crawler, crawler, env)
         for user in user_list:
-            spider_link = user["link"]
-            out_uid = spider_link.split('/')[-1]
-            user_name = user["nick_name"]
-            our_uid = user["uid"]
+            try
+                spider_link = user["link"]
+                out_uid = spider_link.split('/')[-1]
+                user_name = user["nick_name"]
+                our_uid = user["uid"]
 
-            Common.logger(log_type, crawler).info(f"开始抓取 {user_name} 用户主页视频\n")
-            cls.get_videolist(log_type=log_type,
-                              crawler=crawler,
-                              strategy=strategy,
-                              our_uid=our_uid,
-                              out_uid=out_uid,
-                              oss_endpoint=oss_endpoint,
-                              env=env,
-                              machine=machine)
-            cls.offset = 0
+                Common.logger(log_type, crawler).info(f"开始抓取 {user_name} 用户主页视频\n")
+                cls.get_videolist(log_type=log_type,
+                                  crawler=crawler,
+                                  strategy=strategy,
+                                  our_uid=our_uid,
+                                  out_uid=out_uid,
+                                  oss_endpoint=oss_endpoint,
+                                  env=env,
+                                  machine=machine)
+                cls.offset = 0
+            except Exception as e:
+                continue
 
 
 if __name__ == '__main__':