newContentIdTask.py

debug

history_task.py 增加一个查看代码

tasks/history_task.py

@@ -252,7 +252,10 @@ class historyContentIdTask(object):
             function="History Contents Task"
         )
         if task_list:
+            a = time.time()
             tasks = [self.process_task(params) for params in task_list]
             await asyncio.gather(*tasks)
+            b = time.time()
+            print("{} s 内处理了{}个任务".format(b - a, len(task_list)))
         else:
             print("暂时未获得历史已存在文章")

tasks/newContentIdTask.py

@@ -89,12 +89,12 @@ class NewContentIdTask(object):
         )
         # 获取  process_times <= 3 且  content_status = 0 的任务
         select_sql = f"""
-            SELECT 
+            SELECT
                 trace_id, content_id, flow_pool_level, gh_id, process_times
-            FROM 
-                {self.article_match_video_table} 
-            WHERE 
-                    content_status = {self.TASK_INIT_STATUS} 
+            FROM
+                {self.article_match_video_table}
+            WHERE
+                    content_status = {self.TASK_INIT_STATUS}
                 and process_times <= {self.TASK_MAX_PROCESS_TIMES}
             LIMIT {self.spider_coroutines};
         """
@@ -165,7 +165,7 @@ class NewContentIdTask(object):
                             SET
                                 content_status = %s, 
                                 content_status_update_time = %s,
-                                process_times = %s,
+                                process_times = %s
                             WHERE trace_id = %s and content_status = %s;
                         """
         await self.mysql_client.async_insert(