|
|
@@ -708,7 +708,7 @@ class Follow:
|
|
|
Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.text}\n")
|
|
|
cls.offset = 0
|
|
|
return
|
|
|
- elif 'videoList' not in response.json()["data"]:
|
|
|
+ elif not response.json()["data"]['videoList']:
|
|
|
Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.json()}\n")
|
|
|
cls.offset = 0
|
|
|
return
|
|
|
@@ -811,13 +811,9 @@ class Follow:
|
|
|
else:
|
|
|
cover_url = videoList[i]['video_detail_info']['detail_video_large_image']['url_list'][0]['url']
|
|
|
|
|
|
- while True:
|
|
|
- rule_dict = cls.get_rule(log_type, crawler)
|
|
|
- if rule_dict is None:
|
|
|
- Common.logger(log_type, crawler).warning(f"rule_dict:{rule_dict}, 10秒后重试")
|
|
|
- time.sleep(10)
|
|
|
- else:
|
|
|
- break
|
|
|
+ Common.logger(log_type, crawler).info(f'---开始读取规则---')
|
|
|
+ rule_dict = cls.get_rule(log_type, crawler)
|
|
|
+ Common.logger(log_type, crawler).info(f'---读取规则完成---')
|
|
|
|
|
|
if gid == 0 or video_id == 0 or cover_url == 0:
|
|
|
Common.logger(log_type, crawler).info('无效视频\n')
|
|
|
@@ -1022,19 +1018,4 @@ class Follow:
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
- # print(Follow.get_signature("follow", "xigua", "95420624045", "local"))
|
|
|
- # Follow.get_videolist(log_type="follow",
|
|
|
- # crawler="xigua",
|
|
|
- # strategy="定向爬虫策略",
|
|
|
- # our_uid="6267141",
|
|
|
- # out_uid="95420624045",
|
|
|
- # oss_endpoint="out",
|
|
|
- # env="dev",
|
|
|
- # machine="local")
|
|
|
- # print(Follow.random_signature())
|
|
|
- # rule = Follow.get_rule("follow", "xigua")
|
|
|
- # print(type(rule))
|
|
|
- # print(type(json.dumps(rule)))
|
|
|
- # print(json.dumps(rule))
|
|
|
- Follow.get_user_list("follow", "xigua", "5tlTYB", "prod", "local")
|
|
|
- pass
|
|
|
+ Follow.get_follow_videos('follow','xigua','定向抓取策略', 'inner','prod', 'aliyun')
|
