福小顺修改异步处理

spider/crawler_online/fuxiaoshun.py

@@ -164,17 +164,17 @@ class FuXiaoShunRecommend(object):
         :return: None
         """
         async with aiohttp.ClientSession() as session:
-            for i in range(0, 100, 20):
-                if self.expire_flag:
-                    return
-                tasks = [self.get_recommend_list(session, index) for index in range(i, min(i + 20, 200))]
-                await asyncio.gather(*tasks)
-                done, pending = await asyncio.wait(
-                    tasks, return_when=asyncio.FIRST_COMPLETED
-                )
-                # 取消所有剩余的任务
-                for task in pending:
-                    task.cancel()
+            # for i in range(0, 100, 20):
+            #     if self.expire_flag:
+            #         return
+            #     tasks = [self.get_recommend_list(session, index) for index in range(i, min(i + 20, 200))]
+            #     await asyncio.gather(*tasks)
+            #     done, pending = await asyncio.wait(
+            #         tasks, return_when=asyncio.FIRST_COMPLETED
+            #     )
+            #     # 取消所有剩余的任务
+            #     for task in pending:
+            #         task.cancel()
 
             # tasks = [self.get_recommend_list(session, index) for index in range(1, 100)]
             # await asyncio.gather(*tasks)
@@ -184,20 +184,20 @@ class FuXiaoShunRecommend(object):
             # # 取消所有剩余的任务
             # for task in pending:
             #     task.cancel()
-            # for page in range(1, 100):
-            #     if self.expire_flag:
-            #         self.aliyun_log.logging(
-            #             code="2000",
-            #             message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
-            #         )
-            #         message = "本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt)
-            #         print(message)
-            #         return
-            #     else:
-            #         try:
-            #             await self.get_recommend_list(session, page_index=page)
-            #         except Exception as e:
-            #             self.aliyun_log.logging(
-            #                 code="3000",
-            #                 message="抓取第{}页时候出现错误， 报错信息是{}".format(page, e),
-            #             )
+            for page in range(1, 100):
+                if self.expire_flag:
+                    self.aliyun_log.logging(
+                        code="2000",
+                        message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
+                    )
+                    message = "本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt)
+                    print(message)
+                    return
+                else:
+                    try:
+                        await self.get_recommend_list(session, page_index=page)
+                    except Exception as e:
+                        self.aliyun_log.logging(
+                            code="3000",
+                            message="抓取第{}页时候出现错误， 报错信息是{}".format(page, e),
+                        )