kevin.yang 1 week ago
parent
commit
d46fd6ebc9
1 changed files with 8 additions and 3 deletions
  1. 8 3
      scheduler/run_spider_online.py

+ 8 - 3
scheduler/run_spider_online.py

@@ -3,6 +3,8 @@ import os
 import sys
 import asyncio
 import argparse
+import inspect
+import traceback
 
 sys.path.append(os.getcwd())
 
@@ -72,11 +74,14 @@ class OnlineManager(object):
                     user_list=user_list,
                     env=self.env
                 )
-                loop = asyncio.get_event_loop()
-                loop.run_until_complete(main_process.run())
+                if inspect.iscoroutinefunction(main_process.run):
+                    loop = asyncio.get_event_loop()
+                    loop.run_until_complete(main_process.run())
+                else:
+                    main_process.run()
                 self.logger.logging(code=1004, message="完成一轮抓取")
             except Exception as e:
-                self.logger.logging(code=1006, message="启动爬虫出现错误, 报错原因是: {}".format(e))
+                self.logger.logging(code=1006, message=f"启动爬虫出现错误, 报错原因是: {e}\n{traceback.format_exc()}")
 
 
 if __name__ == "__main__":