搜狐抓取

tasks/crawler_tasks/crawler_video/crawler_sohu_recommend_videos.py

@@ -83,7 +83,7 @@ class CrawlerSohuRecommendVideos:
                 response = get_recommendation_video_list(outside_url, author_id, article_id, page)
                 if response:
                     video_list = response['data']['recommendVideoFeed']['list']
-                    for video in tqdm(video_list):
+                    for video in tqdm(video_list, desc=f"page: {page}"):
                         self.crawler_each_video(video)
 
             except Exception as e:
@@ -105,7 +105,7 @@ class CrawlerSohuRecommendVideos:
 
     def deal(self):
         task_list = self.fetch_seed_videos()
-        for task in tqdm(task_list[:1]):
+        for task in tqdm(task_list):
             try:
                 self.get_each_video_recommendation(task)
                 self.update_seed_video_status(task_id=task["id"])