调试

spider/crawler_online/boqingzhufu.py

@@ -122,7 +122,7 @@ class BoQingZhuFu(object):
         }
         await asyncio.sleep(5)
         async with session.get(
-                url, headers=headers, params=params, proxy=tunnel_proxies()['https']
+                url, headers=headers, params=params
         ) as response:
             response_text = await response.text()
             response_json = json.loads(response_text)

spider/crawler_online/huanhuanxixizhufudao_2.py

@@ -112,7 +112,7 @@ class HHXXZFDRecommend(object):
                 params = {
                     "v": self.cryptor.aes_encrypt(data=json.dumps(query))
                 }
-                response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
+                response = requests.request("GET", url, headers=headers, params=params)
                 result = json.loads(self.cryptor.aes_decrypt(response.text))
                 total_page = result['list']['pages']
                 page_index = result['list']['current'] + 1
@@ -162,7 +162,7 @@ class HHXXZFDRecommend(object):
             params = {
                 "v": self.cryptor.aes_encrypt(data=json.dumps(query))
             }
-            response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
+            response = requests.request("GET", url, headers=headers, params=params)
             result = json.loads(self.cryptor.aes_decrypt(response.text))
             total_page = result['list']['pages']
             page_index = result['list']['current'] + 1
@@ -185,8 +185,7 @@ class HHXXZFDRecommend(object):
                     'referer': 'https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html',
                     'accept-language': 'en-US,en;q=0.9'
                 }
-                detail_response = requests.request("GET", url, headers=headers, params=detail_params,
-                                                   proxies=tunnel_proxies())
+                detail_response = requests.request("GET", url, headers=headers, params=detail_params)
                 detail_video = json.loads(self.cryptor.aes_decrypt(detail_response.text))
                 if detail_video['success']:
                     try:

spider/crawler_online/jiajiezhufuxishiduoduo.py

@@ -123,7 +123,7 @@ class XiShiDuoDuoRecommend(object):
         url = "https://www.angjukk.cn/index/home/get_home_list.html"
         await asyncio.sleep(5)
         async with session.post(
-                url, headers=headers, json=data, proxy=tunnel_proxies()['https']
+                url, headers=headers, json=data
         ) as response:
             video_list = await response.json()
             # print(video_list)

spider/crawler_online/jierizhufu.py

@@ -124,7 +124,7 @@ class JieRiZhuFu(object):
             'Accept-Language': 'en-US,en;q=0.9'
         }
         response = requests.request(
-            "GET", url=url, headers=headers, params=params, proxies=tunnel_proxies()
+            "GET", url=url, headers=headers, params=params
         )
         data = response.json()
         for index, video_obj in enumerate(data["data"]["list"], 1):

spider/crawler_online/jixiangxingfu.py

@@ -117,7 +117,7 @@ class JXXFRecommend(object):
             for j in range(2):
                 url = f"https://api.huanqiwl.top/index.php?s=mobile/Video/getList&cid={j}&page={i}&api_version=4&appid=wx6692a24ad2a88bfb&version=1.9.5&env_version=release&scene=1053"
                 payload = {}
-                response = requests.request("GET", url, headers=headers, data=payload, proxies=tunnel_proxies())
+                response = requests.request("GET", url, headers=headers, data=payload)
                 if "未登录" in response.text:
                     self.aliyun_log.logging(
                         code="3000",

spider/crawler_online/shayuzhufu.py

@@ -99,7 +99,7 @@ class SharkZhuFuRecommend(object):
                         url=base_url,
                         headers=headers,
                         params=params,
-                        proxies=tunnel_proxies(),
+
                     )
                     encrypted_info = response.json()["data"]
                     decrypted_info = json.loads(

spider/crawler_online/tiantianjufuqi.py

@@ -76,7 +76,7 @@ class TTJFFQRecommend(object):
                 }
             })
 
-            response = requests.request("POST", url, headers=headers, data=payload, proxies=tunnel_proxies())
+            response = requests.request("POST", url, headers=headers, data=payload)
             for index, video_obj in enumerate(response.json()['data'], 1):
                 try:
                     self.aliyun_log.logging(

spider/crawler_online/xiaoniangaohuati.py

@@ -104,7 +104,7 @@ class XNGHTecommend(object):
                 }
             })
 
-            response = requests.request("POST", url, headers=headers, data=payload, proxies=tunnel_proxies())
+            response = requests.request("POST", url, headers=headers, data=payload)
             score = response.json()['data']['score']
             for index, video_obj in enumerate(response.json()['data']['list'], 1):
                 try:

spider/crawler_online/yuannifuqichangzai.py

@@ -137,7 +137,7 @@ class YuanNiFuQiChangZai(object):
         url = "https://ynfqcz.jiabeijian.cn/index.php/v111/index/index"
         time.sleep(5)
         response = requests.request(
-            "GET", url=url, headers=headers, params=params, proxies=tunnel_proxies()
+            "GET", url=url, headers=headers, params=params
         )
         data = response.json()
         for index, video_obj in enumerate(data["data"]["list"], 1):

spider/crawler_online/zhuhaoshiduomo.py

@@ -119,7 +119,7 @@ class ZhuHaoShiDuoMoRecommend(object):
                 }
                 params = {"v": self.cryptor.aes_encrypt(data=json.dumps(query))}
                 response = requests.get(
-                    url, headers=headers, params=params, proxies=tunnel_proxies()
+                    url, headers=headers, params=params
                 )
                 result = json.loads(self.cryptor.aes_decrypt(response.text))
                 total_page = result["list"]["pages"]
@@ -164,7 +164,7 @@ class ZhuHaoShiDuoMoRecommend(object):
             query = {"pageNo": page_index, "pageSize": 10, "authid": user_id}
             params = {"v": self.cryptor.aes_encrypt(data=json.dumps(query))}
             response = requests.request(
-                "GET", url, headers=headers, params=params, proxies=tunnel_proxies()
+                "GET", url, headers=headers, params=params
             )
             result = json.loads(self.cryptor.aes_decrypt(response.text))
             total_page = result["list"]["pages"]
@@ -190,8 +190,8 @@ class ZhuHaoShiDuoMoRecommend(object):
                     "GET",
                     url,
                     headers=headers,
-                    params=detail_params,
-                    proxies=tunnel_proxies(),
+                    params=detail_params
+
                 )
                 detail_video = json.loads(
                     self.cryptor.aes_decrypt(detail_response.text)

spider/crawler_online/zhujinshanjinmei_2.py

@@ -113,7 +113,7 @@ class ZhuJinShanJinMeiRecommend(object):
                 params = {
                     "v": self.cryptor.aes_encrypt(data=json.dumps(query))
                 }
-                response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
+                response = requests.request("GET", url, headers=headers, params=params)
                 result = json.loads(self.cryptor.aes_decrypt(response.text))
                 total_page = result['list']['pages']
                 page_index = result['list']['current'] + 1
@@ -163,7 +163,7 @@ class ZhuJinShanJinMeiRecommend(object):
             params = {
                 "v": self.cryptor.aes_encrypt(data=json.dumps(query))
             }
-            response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
+            response = requests.request("GET", url, headers=headers, params=params)
             result = json.loads(self.cryptor.aes_decrypt(response.text))
             total_page = result['list']['pages']
             page_index = result['list']['current'] + 1
@@ -186,8 +186,7 @@ class ZhuJinShanJinMeiRecommend(object):
                     'referer': 'https://servicewechat.com/wxbf44c7b26b0b3280/4/page-frame.html',
                     'accept-language': 'en-US,en;q=0.9'
                 }
-                detail_response = requests.request("GET", url, headers=headers, params=detail_params,
-                                                   proxies=tunnel_proxies())
+                detail_response = requests.request("GET", url, headers=headers, params=detail_params)
                 detail_video = json.loads(self.cryptor.aes_decrypt(detail_response.text))
                 if detail_video['success']:
                     try:

spider/crawler_online/zhuwanwufusu_2.py

@@ -119,7 +119,7 @@ class ZhuWanWuFuSuRecommend(object):
                 }
                 params = {"v": self.cryptor.aes_encrypt(data=json.dumps(query))}
                 response = requests.get(
-                    url, headers=headers, params=params, proxies=tunnel_proxies()
+                    url, headers=headers, params=params
                 )
                 result = json.loads(self.cryptor.aes_decrypt(response.text))
                 total_page = result["list"]["pages"]
@@ -169,7 +169,7 @@ class ZhuWanWuFuSuRecommend(object):
             query = {"pageNo": page_index, "pageSize": 10, "authid": user_id}
             params = {"v": self.cryptor.aes_encrypt(data=json.dumps(query))}
             response = requests.request(
-                "GET", url, headers=headers, params=params, proxies=tunnel_proxies()
+                "GET", url, headers=headers, params=params
             )
             result = json.loads(self.cryptor.aes_decrypt(response.text))
             total_page = result["list"]["pages"]
@@ -195,8 +195,7 @@ class ZhuWanWuFuSuRecommend(object):
                     "GET",
                     url,
                     headers=headers,
-                    params=detail_params,
-                    proxies=tunnel_proxies(),
+                    params=detail_params
                 )
                 detail_video = json.loads(
                     self.cryptor.aes_decrypt(detail_response.text)

spider/crawler_online/zuihaodesongni.py

@@ -137,7 +137,7 @@ class ZuiHaoDeSongNi(object):
         url = "https://zhdsn.wentingyou.cn/index.php/v111/index/index"
         time.sleep(5)
         response = requests.request(
-            "GET", url=url, headers=headers, params=params, proxies=tunnel_proxies()
+            "GET", url=url, headers=headers, params=params
         )
         data = response.json()
         for index, video_obj in enumerate(data["data"]["list"], 1):