增加去重

job_cover_method.py

@@ -1,5 +1,5 @@
 import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor, wait
 from common.redis import get_pq_id
 from video_cover_method.cover_method import CoverMethod
 
@@ -8,14 +8,15 @@ max_workers = 10
 
 
 def get_video_id():
-    video_ids = []
+    video_ids = set()  # 使用集合去重
     for i in range(10):
         video_id = get_pq_id()
         if video_id:
-            video_ids.append( video_id )
+            video_ids.add(video_id)
         else:
-           return video_ids
-    return video_ids
+            return list( video_ids )
+    return list( video_ids )
+
 
 def process_video_cover(video_id):
     try:
@@ -26,18 +27,15 @@ def process_video_cover(video_id):
 
 def video_cover_task_start():
     with ThreadPoolExecutor( max_workers=max_workers) as executor:
-        futures = []
         while True:
             video_ids = get_video_id()
             if not video_ids:
                 print("没有数据等待30秒")
                 time.sleep(30)
                 continue
-            for video_id in video_ids:
-                futures.append( executor.submit( process_video_cover, video_id ) )
-                # 等待完成的任务并清理
-            for future in as_completed( futures ):
-                futures.remove( future )
+            # 提交所有任务并等待完成
+            futures = [executor.submit( process_video_cover, video_id ) for video_id in video_ids]
+            wait( futures )  # 等待所有任务完成
 
 
 if __name__ == '__main__':