|
|
@@ -106,7 +106,6 @@ class GongzhonghaoAuthor3:
|
|
|
}
|
|
|
urllib3.disable_warnings()
|
|
|
r = requests.get(url=url, headers=headers, params=params, verify=False)
|
|
|
- Common.logger(log_type, crawler).info(f"get_user_info_res:{r.text}\n")
|
|
|
r.close()
|
|
|
if r.json()["base_resp"]["err_msg"] == "invalid session":
|
|
|
Common.logger(log_type, crawler).warning(f"status_code:{r.status_code}")
|
|
|
@@ -453,25 +452,21 @@ class GongzhonghaoAuthor3:
|
|
|
time.sleep(2)
|
|
|
continue
|
|
|
len_sheet = len(user_sheet)
|
|
|
- Common.logger(log_type, crawler).info(f"len_sheet:{len_sheet}")
|
|
|
if len_sheet <= 201:
|
|
|
Common.logger(log_type, crawler).info("抓取用户数<=200，无需启动第三套抓取脚本\n")
|
|
|
return
|
|
|
if len_sheet >= 301:
|
|
|
len_sheet = 301
|
|
|
for i in range(201, len_sheet):
|
|
|
+ user_dict = cls.get_users(log_type=log_type,
|
|
|
+ crawler=crawler,
|
|
|
+ user_sheet=user_sheet,
|
|
|
+ sheetid=sheetid,
|
|
|
+ i=i,
|
|
|
+ env=env)
|
|
|
+ Common.logger(log_type, crawler).info(f'获取 {user_dict["user_name"]} 公众号视频\n')
|
|
|
try:
|
|
|
- user_dict = cls.get_users(log_type=log_type,
|
|
|
- crawler=crawler,
|
|
|
- user_sheet=user_sheet,
|
|
|
- sheetid=sheetid,
|
|
|
- i=i,
|
|
|
- env=env)
|
|
|
- except Exception as e:
|
|
|
- Common.logger(log_type, crawler).info(f"获取用户信息失败:{e}\n")
|
|
|
-
|
|
|
- try:
|
|
|
- Common.logger(log_type, crawler).info(f'获取 {user_dict["user_name"]} 公众号视频\n')
|
|
|
+ # Common.logger(log_type, crawler).info(f'获取 {user_dict["user_name"]} 公众号视频\n')
|
|
|
cls.get_videoList(log_type=log_type,
|
|
|
crawler=crawler,
|
|
|
rule_dict=rule_dict,
|
