Prechádzať zdrojové kódy

deep_seek_api function rename

luojunhui 4 mesiacov pred
rodič
commit
9a559c5b8e
1 zmenil súbory, kde vykonal 18 pridanie a 12 odobranie
  1. 18 12
      coldStartTasks/crawler/baidu/video_crawler.py

+ 18 - 12
coldStartTasks/crawler/baidu/video_crawler.py

@@ -43,7 +43,7 @@ class BaiduVideoCrawler(object):
         sql = f"""
             select account_id, account_name, max_cursor 
             from baidu_account_for_videos
-            where status = 1;
+            where status = 1 and priority = 0;
         """
         account_list = self.db.fetch(query=sql, cursor_type=DictCursor)
         return account_list
@@ -196,20 +196,21 @@ class BaiduVideoCrawler(object):
         update cursor for each account
         """
         select_sql = f"""
-            select max(publish_timestamp) as from publish_single_video_source where out_account_id = '{account_id}';
+            select max(publish_timestamp) as max_cursor from publish_single_video_source where out_account_id = '{account_id}';
         """
         response_mysql = self.db.fetch(query=select_sql)
         max_publish_timestamp = response_mysql[0][0]
-        max_cursor = max_publish_timestamp * 10000
-        update_sql = f"""
-            update baidu_account_for_videos
-            set max_cursor = %s
-            where account_id = %s;
-        """
-        self.db.save(
-            query=update_sql,
-            params=(max_cursor, account_id)
-        )
+        if max_publish_timestamp:
+            max_cursor = max_publish_timestamp * 10000
+            update_sql = f"""
+                update baidu_account_for_videos
+                set max_cursor = %s
+                where account_id = %s;
+            """
+            self.db.save(
+                query=update_sql,
+                params=(max_cursor, account_id)
+            )
 
     def deal(self):
         """
@@ -219,3 +220,8 @@ class BaiduVideoCrawler(object):
         for account in account_list:
             self.crawler_each_account(account)
             self.update_cursor(account['account_id'])
+
+
+b = BaiduVideoCrawler()
+b.connect_db()
+b.deal()