Merge branch '2025-01-24-remove-illegal' of Server/title_with_video into master

tasks/history_task.py

@@ -52,7 +52,7 @@ class historyContentIdTask(object):
             JOIN (
                 select content_id, count(1) as cnt 
                 from {self.article_crawler_video_table}
-                where download_status = {self.const.VIDEO_DOWNLOAD_SUCCESS_STATUS} and is_illegal = {self.const.VIDEO_SAFE}
+                where download_status = {self.const.VIDEO_DOWNLOAD_SUCCESS_STATUS}
                 group by content_id
             ) VID on ART.content_id = VID.content_id and VID.cnt >= 3
             WHERE ART.content_status = {self.const.TASK_INIT_STATUS} and ART.process_times <= {self.const.TASK_MAX_PROCESS_TIMES}

tasks/new_contentId_task.py

@@ -49,7 +49,6 @@ class NewContentIdTask(object):
                 SELECT content_id, count(1) as cnt
                 FROM {self.article_crawler_video_table}
                 WHERE download_status = {NewContentIdTaskConst.VIDEO_DOWNLOAD_SUCCESS_STATUS}
-                    AND is_illegal = {NewContentIdTaskConst.VIDEO_SAFE}
                 GROUP BY content_id
             ) t2
             ON t1.content_id = t2.content_id

tasks/utils/spider_task.py

@@ -18,8 +18,7 @@ async def whether_downloaded_videos_exists(content_id, article_crawler_video_tab
     SELECT id
     FROM {article_crawler_video_table}
     WHERE content_id = '{content_id}' 
-        AND download_status = {new_content_id_task_const.VIDEO_DOWNLOAD_SUCCESS_STATUS}
-        AND is_illegal = {new_content_id_task_const.VIDEO_SAFE};
+        AND download_status = {new_content_id_task_const.VIDEO_DOWNLOAD_SUCCESS_STATUS};
     """
     res_tuple = await db_client.async_select(sql)
     if len(res_tuple) >= new_content_id_task_const.MIN_MATCH_VIDEO_NUM:
@@ -38,7 +37,6 @@ async def get_downloaded_videos(content_id, article_crawler_video_table, db_clie
         FROM {article_crawler_video_table}
         WHERE content_id = '{content_id}' 
             AND download_status = {new_content_id_task_const.VIDEO_DOWNLOAD_SUCCESS_STATUS}
-            AND is_illegal = {new_content_id_task_const.VIDEO_SAFE}
         ORDER BY score DESC;
     """