福小顺修改异步处理

spider/crawler_online/fuxiaoshun.py

@@ -138,6 +138,7 @@ class FuXiaoShunRecommend(object):
         }
         url = "https://shun.nnjuxing.cn/videos/api.videos/getItem"
         params = {"mark": "", "page": page_index}
+        await asyncio.sleep(5)
         async with session.get(
             url, headers=headers, params=params
         ) as response:
@@ -163,14 +164,26 @@ class FuXiaoShunRecommend(object):
         :return: None
         """
         async with aiohttp.ClientSession() as session:
-            tasks = [self.get_recommend_list(session, index) for index in range(1, 100)]
-            await asyncio.gather(*tasks)
-            done, pending = await asyncio.wait(
-                tasks, return_when=asyncio.FIRST_COMPLETED
-            )
-            # 取消所有剩余的任务
-            for task in pending:
-                task.cancel()
+            for i in range(0, 100, 20):
+                if self.expire_flag:
+                    return
+                tasks = [self.get_recommend_list(session, index) for index in range(i, min(i + 20, 200))]
+                await asyncio.gather(*tasks)
+                done, pending = await asyncio.wait(
+                    tasks, return_when=asyncio.FIRST_COMPLETED
+                )
+                # 取消所有剩余的任务
+                for task in pending:
+                    task.cancel()
+
+            # tasks = [self.get_recommend_list(session, index) for index in range(1, 100)]
+            # await asyncio.gather(*tasks)
+            # done, pending = await asyncio.wait(
+            #     tasks, return_when=asyncio.FIRST_COMPLETED
+            # )
+            # # 取消所有剩余的任务
+            # for task in pending:
+            #     task.cancel()
             # for page in range(1, 100):
             #     if self.expire_flag:
             #         self.aliyun_log.logging(