|
@@ -1,14 +1,9 @@
|
|
-# -*- coding: utf-8 -*-
|
|
|
|
-# @Author: wangkun
|
|
|
|
-# @Time: 2023/3/28
|
|
|
|
import datetime
|
|
import datetime
|
|
import json
|
|
import json
|
|
import os
|
|
import os
|
|
import random
|
|
import random
|
|
-import shutil
|
|
|
|
import sys
|
|
import sys
|
|
import time
|
|
import time
|
|
-from hashlib import md5
|
|
|
|
import requests
|
|
import requests
|
|
import urllib3
|
|
import urllib3
|
|
from selenium.webdriver import DesiredCapabilities
|
|
from selenium.webdriver import DesiredCapabilities
|
|
@@ -44,8 +39,6 @@ class GongzhonghaoAuthor:
|
|
"update_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(configs[0]["update_time"]/1000))),
|
|
"update_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(configs[0]["update_time"]/1000))),
|
|
"operator": configs[0]["operator"].strip()
|
|
"operator": configs[0]["operator"].strip()
|
|
}
|
|
}
|
|
- # for k, v in token_dict.items():
|
|
|
|
- # print(f"{k}:{type(v)}, {v}")
|
|
|
|
return token_dict
|
|
return token_dict
|
|
|
|
|
|
# 获取用户 fakeid
|
|
# 获取用户 fakeid
|
|
@@ -333,7 +326,7 @@ class GongzhonghaoAuthor:
|
|
video_dict["user_id"] = user_dict["uid"] # 站内 UID?爬虫获取不到了(随机发布到原 5 个账号中)
|
|
video_dict["user_id"] = user_dict["uid"] # 站内 UID?爬虫获取不到了(随机发布到原 5 个账号中)
|
|
video_dict["publish_time"] = video_dict["publish_time_str"]
|
|
video_dict["publish_time"] = video_dict["publish_time_str"]
|
|
mq.send_msg(video_dict)
|
|
mq.send_msg(video_dict)
|
|
- time.sleep(random.randint(1, 3))
|
|
|
|
|
|
+ time.sleep(random.randint(1, 8))
|
|
except Exception as e:
|
|
except Exception as e:
|
|
Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n")
|
|
Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n")
|
|
Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n")
|
|
Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n")
|
|
@@ -348,8 +341,6 @@ class GongzhonghaoAuthor:
|
|
repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
|
|
repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
|
|
return len(repeat_video)
|
|
return len(repeat_video)
|
|
|
|
|
|
- # 下载/上传
|
|
|
|
-
|
|
|
|
@classmethod
|
|
@classmethod
|
|
def get_all_videos(cls, log_type, crawler, task_dict, token_index, rule_dict, user_list, env):
|
|
def get_all_videos(cls, log_type, crawler, task_dict, token_index, rule_dict, user_list, env):
|
|
for user_dict in user_list:
|
|
for user_dict in user_list:
|
|
@@ -372,9 +363,4 @@ class GongzhonghaoAuthor:
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if __name__ == "__main__":
|
|
- # GongzhonghaoAuthor1.get_token("author", "gongzhonghao", "prod")
|
|
|
|
- # print(GongzhonghaoAuthor1.get_users("author", "gongzhonghao", "Bzv72P", "dev"))
|
|
|
|
- # print(get_config_from_mysql("author", "gongzhonghao", "dev", "filter", action=""))
|
|
|
|
- # print(title_like("author", "gongzhonghao", "公众号", "123", "dev"))
|
|
|
|
- # print(GongzhonghaoAuthor1.get_user_info("author", "gongzhonghao", "幸福花朵", "dev"))
|
|
|
|
pass
|
|
pass
|