agc_data.py 2.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556
  1. from common import Material
  2. from extract_data.douyin.douyin_author import douyinAuthor
  3. from extract_data.kuaishou.kuaishou_author import kuaishouAuthor
  4. from extract_data.zhannei.zhannei_author import ZhanNeiAuthor
  5. import schedule
  6. import time
  7. import concurrent.futures
  8. def gs_start(platform, user_data):
  9. print(f"执行{platform}数据抓取{user_data}")
  10. if platform == "douyin":
  11. douyinAuthor.get_videoList(user_data)
  12. elif platform == "kuaishou":
  13. kuaishouAuthor.get_kuaishou_videoList(user_data)
  14. elif platform == "zhannei":
  15. ZhanNeiAuthor.get_zhannei_videoList(user_data)
  16. def gs_task(platform):
  17. data = Material.get_all_gs_user(platform)
  18. valid_data = [user_data for user_data in data if user_data['sheet'] is not None]
  19. with concurrent.futures.ThreadPoolExecutor() as executor:
  20. futures = {executor.submit(gs_start, platform, user_data): user_data for user_data in valid_data}
  21. for future in concurrent.futures.as_completed(futures):
  22. result = future.result()
  23. print("处理结果:", result)
  24. print(f"{platform.capitalize()}数据抓取定时任务执行完成.")
  25. def cg_start(platform, user_data):
  26. print(f"执行{platform}数据抓取{user_data}")
  27. if platform == "douyin":
  28. douyinAuthor.get_videoList(user_data)
  29. elif platform == "kuaishou":
  30. kuaishouAuthor.get_kuaishou_videoList(user_data)
  31. def cg_task(platform):
  32. data = Material.get_all_user(platform)
  33. with concurrent.futures.ThreadPoolExecutor() as executor:
  34. futures = {executor.submit(cg_start, platform, user_data): user_data for user_data in data}
  35. for future in concurrent.futures.as_completed(futures):
  36. result = future.result()
  37. print("处理结果:", result)
  38. print(f"{platform.capitalize()}数据抓取定时任务执行完成.")
  39. schedule.every().day.at("19:20").do(gs_task, "kuaishou")
  40. schedule.every().day.at("19:30").do(gs_task, "douyin")
  41. schedule.every().day.at("18:00").do(gs_task, "zhannei")
  42. schedule.every(8).hours.do(cg_task, "douyin")
  43. schedule.every(8).hours.do(cg_task, "kuaishou")
  44. while True:
  45. schedule.run_pending()
  46. time.sleep(1)