|
|
@@ -213,6 +213,7 @@ class XiguaSearch:
|
|
|
s.mount('https://', HTTPAdapter(max_retries=3))
|
|
|
response = s.get(url=url, headers=headers, params=params, cookies=cookies, verify=False,
|
|
|
proxies=Common.tunnel_proxies(), timeout=5)
|
|
|
+ # response = s.get(url=url, headers=headers, params=params, cookies=cookies, verify=False)
|
|
|
response.close()
|
|
|
if 'data' not in response.json() or response.json()['data'] == '':
|
|
|
Common.logger(log_type, crawler).warning('get_video_info: response: {}', response)
|
|
|
@@ -659,7 +660,13 @@ class XiguaSearch:
|
|
|
'user-agent': get_random_user_agent('pc'),
|
|
|
}
|
|
|
try:
|
|
|
- res = requests.request("GET", url, headers=headers, proxies=Common.tunnel_proxies())
|
|
|
+ proxies = Common.tunnel_proxies()
|
|
|
+ s = requests.session()
|
|
|
+ # max_retries=3 重试3次
|
|
|
+ s.mount('http://', HTTPAdapter(max_retries=3))
|
|
|
+ s.mount('https://', HTTPAdapter(max_retries=3))
|
|
|
+ res = s.request("GET", url, headers=headers, proxies=proxies, timeout=5)
|
|
|
+ Common.logger(log_type, crawler).info(f"proxies:{proxies}\n")
|
|
|
Common.logger(log_type, crawler).info(f"get_videolist:{res.json()}\n")
|
|
|
search_list = res.json()['data']['data']
|
|
|
except Exception as e:
|
|
|
@@ -948,5 +955,5 @@ class XiguaSearch:
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
- XiguaSearch.get_search_videos('search', 'xigua', 'xigua_search', 'inner', 'prod', 'aliyun')
|
|
|
+ XiguaSearch.get_search_videos('search', 'xigua', 'xigua_search', 'out', 'dev', 'aliyun')
|
|
|
|
