抓取调试

extract_data/douyin/douyin_author.py

@@ -76,7 +76,11 @@ class douyinAuthor():
                 Common.logger("douyin").info(f"用户主页ID：{account_id}")
                 next_cursor = 0
                 count = 0
+                exit_flag = False
                 while True:
+                    if exit_flag:
+                        # 结束 while 循环
+                        break
                     if next_cursor == None:
                         break
                     if count > 5:
@@ -149,6 +153,7 @@ class douyinAuthor():
                                     if count > 5:
                                         Common.logger("douyin").info(
                                             f"重复视频不在抓取该用户，用户主页id：{account_id}")
+                                        exit_flag = True
                                         break
                                     continue
                                 video_url = data[j].get('video').get('play_addr').get('url_list')[0]  # 视频链接
@@ -167,6 +172,7 @@ class douyinAuthor():
                         except Exception as e:
                             Common.logger("douyin").warning(f"抓取单条视频异常:{e}\n")
                             continue
+
         except Exception as e:
             Common.logger("douyin").warning(f"抓取异常:{e}\n")
             return

extract_data/kuaishou/kuaishou_author.py

@@ -72,7 +72,10 @@ class kuaishouAuthor():
                 Common.logger("kuaishou").info(f"用户主页ID：{account_id}")
                 pcursor = ""
                 count = 0
+                exit_flag = False
                 while True:
+                    if exit_flag:
+                        break
                     if count > 5:
                         continue
                     time.sleep(random.randint(10, 50))
@@ -143,6 +146,7 @@ class kuaishouAuthor():
                                 if count > 5:
                                     Common.logger("kuaishou").info(
                                         f"重复视频不在抓取该用户，用户主页id：{account_id}")
+                                    exit_flag = True
                                     break
                                 continue
                             channel_name = mark+'/kuaishou'