update

kuaishou/kuaishou_follow/kuaishou_follow.py

@@ -9,6 +9,8 @@ import time
 import requests
 import json
 
+import urllib3
+
 sys.path.append(os.getcwd())
 from common.common import Common
 from common.feishu import Feishu
@@ -142,7 +144,9 @@ class Follow:
                 'sec-ch-ua-mobile': '?0',
                 'sec-ch-ua-platform': '"macOS"'
             }
-            response = requests.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies())
+            urllib3.disable_warnings()
+            response = requests.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies(), verify=False)
+            response.close()
             # Common.logger(log_type, crawler).info(f"get_out_user_info_response:{response.text}")
             if response.status_code != 200:
                 Common.logger(log_type, crawler).warning(f"get_out_user_info_response:{response.text}\n")
@@ -335,7 +339,9 @@ class Follow:
                     # 'sec-ch-ua-mobile': '?0',
                     # 'sec-ch-ua-platform': '"macOS"'
                 }
-                response = requests.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies())
+                urllib3.disable_warnings()
+                response = requests.post(url=url, headers=headers, data=payload, proxies=Common.tunnel_proxies(), verify=False)
+                response.close()
                 # Common.logger(log_type, crawler).info(f"get_videoList:{response.text}\n")
                 if response.status_code != 200:
                     Common.logger(log_type, crawler).warning(f"get_videoList_response:{response.text}\n")

xigua/xigua_follow/xigua_follow.py

@@ -103,8 +103,8 @@ class Follow:
                        'referer': f'https://www.ixigua.com/home/{out_uid}',
                        'Cookie': f'ixigua-a-s=1; support_webp=true; support_avif=false; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; __ac_signature={cls.random_signature()}; MONITOR_WEB_ID=67cb5099-a022-4ec3-bb8e-c4de6ba51dd0; s_v_web_id=verify_lef4i99x_32SosrdH_Qrtk_4LJn_8S7q_fhu16xe3s8ZV; tt_scid=QLJjPuHf6wxVqu6IIq6gHiJXQpVrCwrdhjH2zpm7-E3ZniE1RXBcP6M8b41FJOdo41e1; ttwid=1%7CHHtv2QqpSGuSu8r-zXF1QoWsvjmNi1SJrqOrZzg-UCY%7C1677047013%7C5866a444e5ae10a9df8c11551db75010fb77b657f214ccf84e503fae8d313d09; msToken=PerXJcDdIsZ6zXkGITsftXX4mDaVaW21GuqtzSVdctH46oXXT2GcELIs9f0XW2hunRzP6KVHLZaYElRvNYflLKUXih7lC27XKxs3HjdZiXPK9NQaoKbLfA==; ixigua-a-s=1',}
             url = f"https://www.ixigua.com/home/{out_uid}"
-
-            response = requests.get(url=url, headers=headers, proxies=Common.tunnel_proxies()).text
+            urllib3.disable_warnings()
+            response = requests.get(url=url, headers=headers, proxies=Common.tunnel_proxies(), verify=False).text
             html = etree.HTML(response)
             out_follow_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[1]/span')[0].text.encode('raw_unicode_escape').decode()
             out_fans_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[2]/span')[0].text.encode('raw_unicode_escape').decode()
@@ -286,6 +286,7 @@ class Follow:
             }
             urllib3.disable_warnings()
             response = requests.get(url=url, headers=headers, params=params, cookies=cookies, verify=False, proxies=Common.tunnel_proxies())
+            response.close()
             if 'data' not in response.json() or response.json()['data'] == '':
                 Common.logger(log_type, crawler).warning('get_video_info: response: {}', response)
             else:
@@ -677,6 +678,7 @@ class Follow:
                 }
                 urllib3.disable_warnings()
                 response = requests.get(url=url, headers=headers, params=params, proxies=Common.tunnel_proxies(), verify=False)
+                response.close()
                 cls.offset += 30
                 if response.status_code != 200:
                     Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.text}\n")