修改定时任务

agc_main.py

@@ -1,9 +1,9 @@
 from common import Feishu, Material
 from common.sql_help import sqlHelp
 from video_agc.agc_video_method import AgcVidoe
+import concurrent.futures
 
 import schedule
-import threading
 import time
 
 
@@ -11,7 +11,7 @@ import time
 returned_usernames_today = set()
 def video_start(user_data):
     global returned_usernames_today
-    user_data_mark = user_data[0]["mark"]
+    user_data_mark = user_data["mark"]
     # 开始准备执行生成视频脚本
     if user_data_mark is not None and user_data_mark in returned_usernames_today:
         print(f"视频脚本参数中的用户名 {user_data_mark} 今天已经返回过，不再启动线程。")
@@ -19,22 +19,23 @@ def video_start(user_data):
     elif user_data_mark is not None:
         print(f"视频脚本参数{user_data}")
         mark = AgcVidoe.video_stitching(user_data)
-        returned_usernames_today.add(mark)
+        if mark:
+            returned_usernames_today.add(mark)
 
 
 # 定义定时任务
 def video_task():
     print("开始执行生成视频脚.")
     data = Material.feishu_list()
-    threads = []
-    for _, user_data in data.iterrows():
-        thread = threading.Thread(target=video_start, args=(user_data,))
-        threads.append(thread)
-        thread.start()
-    for thread in threads:
-        thread.join()
+    # 创建一个线程池
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = [executor.submit(video_start, user_data) for user_data in data]
+        # 等待所有任务执行完成
+        for future in concurrent.futures.as_completed(futures):
+            # 获取每个任务的执行结果
+            result = future.result()
+            print("处理结果:", result)
     print("执行生成视频脚结束")
-
 schedule.every(10).minutes.do(video_task)
 
 # 每天0点清空集合

data_main.py

@@ -3,15 +3,15 @@ from extract_data.douyin.douyin_author import douyinAuthor
 from extract_data.kuaishou.kuaishou_author import kuaishouAuthor
 
 import schedule
-import threading
 import time
+import concurrent.futures
 
 # 定义读取表格的函数
-def douyin_data(user_data):
+def douyin_start(user_data):
     print(f"执行抖音数据抓取{user_data}")
     douyinAuthor.get_videoList(user_data)
 
-def kuaishou_data(user_data):
+def kuaishou_start(user_data):
     print(f"执行快手数据抓取{user_data}")
     kuaishouAuthor.get_kuaishou_videoList(user_data)
 
@@ -19,26 +19,28 @@ def kuaishou_data(user_data):
 # 定义定时任务
 def douyin_task():
     data = Material.get_all_user("douyin")
-    threads = []
-    for _, user_data in data.iterrows():
-        thread = threading.Thread(target=douyin_data, args=(user_data,))
-        threads.append(thread)
-        thread.start()
-    for thread in threads:
-        thread.join()
+    # 创建一个线程池
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = [executor.submit(douyin_start, user_data) for user_data in data]
+        # 等待所有任务执行完成
+        for future in concurrent.futures.as_completed(futures):
+            # 获取每个任务的执行结果
+            result = future.result()
+            print("处理结果:", result)
     print("抖音数据抓取定时任务执行完成")
 
 
 # 定义定时任务
 def kuanshou_task():
     data = Material.get_all_user("kuanshou")
-    threads = []
-    for _, user_data in data.iterrows():
-        thread = threading.Thread(target=kuaishou_data, args=(user_data,))
-        threads.append(thread)
-        thread.start()
-    for thread in threads:
-        thread.join()
+    # 创建一个线程池
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = [executor.submit(kuaishou_start, user_data) for user_data in data]
+        # 等待所有任务执行完成
+        for future in concurrent.futures.as_completed(futures):
+            # 获取每个任务的执行结果
+            result = future.result()
+            print("处理结果:", result)
     print("快手数据抓取定时任务执行完成.")
 
 # schedule.every(2).minutes.do(douyin_task)

extract_data/douyin/douyin_author.py

@@ -62,11 +62,11 @@ class douyinAuthor():
     @classmethod
     def get_videoList(cls, data):
         try:
-            mark = data[0]['mark']
-            token = data[0]['token']
-            feishu_id = data[0]['feishu_id']
-            channel_id = data[0]['channel'][0]
-            channel = data[0]['channel'][1]
+            mark = data['mark']
+            token = data['token']
+            feishu_id = data['feishu_id']
+            channel_id = data['channel'][0]
+            channel = data['channel'][1]
             Material.insert_user(feishu_id, channel_id, mark, channel)
             cookie = Material.get_cookie(feishu_id, token, channel)
             # 获取 用户主页id

extract_data/kuaishou/kuaishou_author.py

@@ -59,11 +59,11 @@ class kuaishouAuthor():
     @classmethod
     def get_kuaishou_videoList(cls, data):
         try:
-            mark = data[0]['mark']
-            token = data[0]['token']
-            feishu_id = data[0]['feishu_id']
-            channel_id = data[0]['channel'][0]
-            channel = data[0]['channel'][1]
+            mark = data['mark']
+            token = data['token']
+            feishu_id = data['feishu_id']
+            channel_id = data['channel'][0]
+            channel = data['channel'][1]
             Material.insert_user(feishu_id, channel_id, mark, channel)
             cookie = Material.get_cookie(feishu_id, token, channel)
             # 获取 用户主页id

video_agc/agc_video_method.py

@@ -277,12 +277,12 @@ class AgcVidoe():
     @classmethod
     def video_stitching(cls, ex_list):
         try:
-            pq_ids = ex_list[0]["pq_id"]
+            pq_ids = ex_list["pq_id"]
             pq_ids_list = pq_ids.split(',')
 
-            mark = ex_list[0]["mark"]
-            feishu_id = ex_list[0]["feishu_id"]
-            video_call = ex_list[0]["video_call"]
+            mark = ex_list["mark"]
+            feishu_id = ex_list["feishu_id"]
+            video_call = ex_list["video_call"]
             parts = video_call.split(',')
             result = []
             for part in parts: