|
@@ -43,7 +43,7 @@ class BaiduVideoCrawler(object):
|
|
|
sql = f"""
|
|
|
select account_id, account_name, max_cursor
|
|
|
from baidu_account_for_videos
|
|
|
- where status = 1;
|
|
|
+ where status = 1 and priority = 0;
|
|
|
"""
|
|
|
account_list = self.db.fetch(query=sql, cursor_type=DictCursor)
|
|
|
return account_list
|
|
@@ -196,20 +196,21 @@ class BaiduVideoCrawler(object):
|
|
|
update cursor for each account
|
|
|
"""
|
|
|
select_sql = f"""
|
|
|
- select max(publish_timestamp) as from publish_single_video_source where out_account_id = '{account_id}';
|
|
|
+ select max(publish_timestamp) as max_cursor from publish_single_video_source where out_account_id = '{account_id}';
|
|
|
"""
|
|
|
response_mysql = self.db.fetch(query=select_sql)
|
|
|
max_publish_timestamp = response_mysql[0][0]
|
|
|
- max_cursor = max_publish_timestamp * 10000
|
|
|
- update_sql = f"""
|
|
|
- update baidu_account_for_videos
|
|
|
- set max_cursor = %s
|
|
|
- where account_id = %s;
|
|
|
- """
|
|
|
- self.db.save(
|
|
|
- query=update_sql,
|
|
|
- params=(max_cursor, account_id)
|
|
|
- )
|
|
|
+ if max_publish_timestamp:
|
|
|
+ max_cursor = max_publish_timestamp * 10000
|
|
|
+ update_sql = f"""
|
|
|
+ update baidu_account_for_videos
|
|
|
+ set max_cursor = %s
|
|
|
+ where account_id = %s;
|
|
|
+ """
|
|
|
+ self.db.save(
|
|
|
+ query=update_sql,
|
|
|
+ params=(max_cursor, account_id)
|
|
|
+ )
|
|
|
|
|
|
def deal(self):
|
|
|
"""
|
|
@@ -219,3 +220,8 @@ class BaiduVideoCrawler(object):
|
|
|
for account in account_list:
|
|
|
self.crawler_each_account(account)
|
|
|
self.update_cursor(account['account_id'])
|
|
|
+
|
|
|
+
|
|
|
+b = BaiduVideoCrawler()
|
|
|
+b.connect_db()
|
|
|
+b.deal()
|