update gongzhonghao add tunnel_proxies

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow.py

@@ -9,7 +9,7 @@ import shutil
 import sys
 import time
 from hashlib import md5
-
+from requests.adapters import HTTPAdapter
 import requests
 import urllib3
 from selenium.webdriver import DesiredCapabilities
@@ -117,8 +117,12 @@ class GongzhonghaoFollow:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
-
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
@@ -242,7 +246,12 @@ class GongzhonghaoFollow:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).info(f"get_videoList:{r.text}\n")

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow_2.py

@@ -12,6 +12,7 @@ from hashlib import md5
 
 import requests
 import urllib3
+from requests.adapters import HTTPAdapter
 from selenium.webdriver import DesiredCapabilities
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.common.by import By
@@ -115,7 +116,12 @@ class GongzhonghaoFollow2:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
@@ -238,7 +244,12 @@ class GongzhonghaoFollow2:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).info(f"get_videoList:{r.text}")

gongzhonghao/gongzhonghao_follow/gongzhonghao_follow_3.py

@@ -12,6 +12,7 @@ from hashlib import md5
 
 import requests
 import urllib3
+from requests.adapters import HTTPAdapter
 from selenium.webdriver import DesiredCapabilities
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.common.by import By
@@ -115,7 +116,12 @@ class GongzhonghaoFollow3:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).warning(f"get_fakeid:{r.text}\n")
@@ -238,7 +244,12 @@ class GongzhonghaoFollow3:
                     "ajax": "1",
                 }
                 urllib3.disable_warnings()
-                r = requests.get(url=url, headers=headers, params=params, verify=False)
+                s = requests.session()
+                # max_retries=3 重试3次
+                s.mount('http://', HTTPAdapter(max_retries=3))
+                s.mount('https://', HTTPAdapter(max_retries=3))
+                r = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
+                r.close()
                 if r.json()["base_resp"]["err_msg"] == "invalid session":
                     Common.logger(log_type, crawler).info(f"status_code:{r.status_code}")
                     Common.logger(log_type, crawler).info(f"get_videoList:{r.text}\n")