update

kuaishou/kuaishou_follow/kuaishou_follow.py

@@ -10,6 +10,7 @@ import requests
 import json
 
 import urllib3
+from requests.adapters import HTTPAdapter
 
 sys.path.append(os.getcwd())
 from common.common import Common
@@ -145,7 +146,11 @@ class Follow:
                 'sec-ch-ua-platform': '"macOS"'
             }
             urllib3.disable_warnings()
-            response = requests.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies(), verify=False)
+            s = requests.session()
+            # max_retries=3 重试3次
+            s.mount('http://', HTTPAdapter(max_retries=3))
+            s.mount('https://', HTTPAdapter(max_retries=3))
+            response = s.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies(), verify=False, timeout=5)
             response.close()
             # Common.logger(log_type, crawler).info(f"get_out_user_info_response:{response.text}")
             if response.status_code != 200:
@@ -341,7 +346,11 @@ class Follow:
                     # 'sec-ch-ua-platform': '"macOS"'
                 }
                 urllib3.disable_warnings()
-                response = requests.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies(), verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                response = s.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies(), verify=False, timeout=5)
                 response.close()
                 # Common.logger(log_type, crawler).info(f"get_videoList:{response.text}\n")
                 if response.status_code != 200:

xigua/xigua_follow/xigua_follow.py

@@ -11,6 +11,7 @@ import sys
 import time
 import requests
 import urllib3
+from requests.adapters import HTTPAdapter
 
 from selenium.webdriver import DesiredCapabilities
 from selenium.webdriver.chrome.service import Service
@@ -104,7 +105,11 @@ class Follow:
                        'Cookie': f'ixigua-a-s=1; support_webp=true; support_avif=false; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; __ac_signature={cls.random_signature()}; MONITOR_WEB_ID=67cb5099-a022-4ec3-bb8e-c4de6ba51dd0; s_v_web_id=verify_lef4i99x_32SosrdH_Qrtk_4LJn_8S7q_fhu16xe3s8ZV; tt_scid=QLJjPuHf6wxVqu6IIq6gHiJXQpVrCwrdhjH2zpm7-E3ZniE1RXBcP6M8b41FJOdo41e1; ttwid=1%7CHHtv2QqpSGuSu8r-zXF1QoWsvjmNi1SJrqOrZzg-UCY%7C1677047013%7C5866a444e5ae10a9df8c11551db75010fb77b657f214ccf84e503fae8d313d09; msToken=PerXJcDdIsZ6zXkGITsftXX4mDaVaW21GuqtzSVdctH46oXXT2GcELIs9f0XW2hunRzP6KVHLZaYElRvNYflLKUXih7lC27XKxs3HjdZiXPK9NQaoKbLfA==; ixigua-a-s=1',}
             url = f"https://www.ixigua.com/home/{out_uid}"
             urllib3.disable_warnings()
-            response = requests.get(url=url, headers=headers, proxies=Common.tunnel_proxies(), verify=False).text
+            s = requests.session()
+            # max_retries=3 重试3次
+            s.mount('http://', HTTPAdapter(max_retries=3))
+            s.mount('https://', HTTPAdapter(max_retries=3))
+            response = s.get(url=url, headers=headers, proxies=Common.tunnel_proxies(), verify=False, timeout=5).text
             html = etree.HTML(response)
             out_follow_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[1]/span')[0].text.encode('raw_unicode_escape').decode()
             out_fans_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[2]/span')[0].text.encode('raw_unicode_escape').decode()
@@ -285,7 +290,11 @@ class Follow:
                 'xiguavideopcwebid.sig': 'xxRww5R1VEMJN_dQepHorEu_eAc',
             }
             urllib3.disable_warnings()
-            response = requests.get(url=url, headers=headers, params=params, cookies=cookies, verify=False, proxies=Common.tunnel_proxies())
+            s = requests.session()
+            # max_retries=3 重试3次
+            s.mount('http://', HTTPAdapter(max_retries=3))
+            s.mount('https://', HTTPAdapter(max_retries=3))
+            response = s.get(url=url, headers=headers, params=params, cookies=cookies, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
             response.close()
             if 'data' not in response.json() or response.json()['data'] == '':
                 Common.logger(log_type, crawler).warning('get_video_info: response: {}', response)
@@ -677,7 +686,11 @@ class Follow:
                     # 'x-secsdk-csrf-token': '00010000000119e3f9454d1dcbb288704cda1960f241e2d19bd21f2fd283520c3615a990ac5a17448bfbb902a249'
                 }
                 urllib3.disable_warnings()
-                response = requests.get(url=url, headers=headers, params=params, proxies=Common.tunnel_proxies(), verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                response = s.get(url=url, headers=headers, params=params, proxies=Common.tunnel_proxies(), verify=False, timeout=5)
                 response.close()
                 cls.offset += 30
                 if response.status_code != 200: