automatic_crawler/tt.py

import asyncio
import time


async def run(task_id, mode, platform):
    """
    传入参数，然后根据参数执行爬虫代码
    :return: None
    """
    # 创建并等待一个子进程
    await asyncio.create_subprocess_shell("python3 scheduler/run_spider_online.py --task_id {} --mode {} --platform {}".format(task_id, mode, platform))


async def main():
    # 创建爬虫task
    while True:
        for task_id in range(95, 96):
            print("start:{:02},  {}".format(task_id, int(time.time())))
            await asyncio.create_task(run(task_id, "recommend", "test"))
            time.sleep(1)


if __name__ == '__main__':
    # 运行主事件循环
    asyncio.run(main())