1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556 |
- from common import Material
- from extract_data.douyin.douyin_author import douyinAuthor
- from extract_data.kuaishou.kuaishou_author import kuaishouAuthor
- from extract_data.zhannei.zhannei_author import ZhanNeiAuthor
- import schedule
- import time
- import concurrent.futures
- def gs_start(platform, user_data):
- print(f"执行{platform}数据抓取{user_data}")
- if platform == "douyin":
- douyinAuthor.get_videoList(user_data)
- elif platform == "kuaishou":
- kuaishouAuthor.get_kuaishou_videoList(user_data)
- elif platform == "zhannei":
- ZhanNeiAuthor.get_zhannei_videoList(user_data)
- def gs_task(platform):
- data = Material.get_all_gs_user(platform)
- valid_data = [user_data for user_data in data if user_data['sheet'] is not None]
- with concurrent.futures.ThreadPoolExecutor() as executor:
- futures = {executor.submit(gs_start, platform, user_data): user_data for user_data in valid_data}
- for future in concurrent.futures.as_completed(futures):
- result = future.result()
- print("处理结果:", result)
- print(f"{platform.capitalize()}数据抓取定时任务执行完成.")
- def cg_start(platform, user_data):
- print(f"执行{platform}数据抓取{user_data}")
- if platform == "douyin":
- douyinAuthor.get_videoList(user_data)
- elif platform == "kuaishou":
- kuaishouAuthor.get_kuaishou_videoList(user_data)
- def cg_task(platform):
- data = Material.get_all_user(platform)
- with concurrent.futures.ThreadPoolExecutor() as executor:
- futures = {executor.submit(cg_start, platform, user_data): user_data for user_data in data}
- for future in concurrent.futures.as_completed(futures):
- result = future.result()
- print("处理结果:", result)
- print(f"{platform.capitalize()}数据抓取定时任务执行完成.")
- schedule.every().day.at("19:20").do(gs_task, "kuaishou")
- schedule.every().day.at("19:30").do(gs_task, "douyin")
- schedule.every().day.at("18:00").do(gs_task, "zhannei")
- schedule.every(8).hours.do(cg_task, "douyin")
- schedule.every(8).hours.do(cg_task, "kuaishou")
- while True:
- schedule.run_pending()
- time.sleep(1)
|