update

xigua/xigua_main/run_xigua_recommend.py

@@ -16,7 +16,7 @@ def main(log_type, crawler, env):
     else:
         oss_endpoint = "inner"
     Common.logger(log_type, crawler).info('开始抓取 西瓜视频 推荐榜\n')
-    XiguaRecommend.get_videolist(log_type, crawler, oss_endpoint, env)
+    XiguaRecommend.get_videoList(log_type, crawler, oss_endpoint, env)
     Common.del_logs(log_type, crawler)
     Common.logger(log_type, crawler).info('抓取完一轮，休眠 1 分钟\n')
 

xigua/xigua_recommend/xigua_recommend.py

@@ -616,8 +616,9 @@ class XiguaRecommend:
         return len(repeat_video)
 
     @classmethod
-    def get_videolist(cls, log_type, crawler, oss_endpoint, env):
+    def get_videoList(cls, log_type, crawler, oss_endpoint, env):
         while True:
+            queryCount = 1
             signature = cls.get_signature(log_type, crawler, env)
             if signature is None:
                 Common.logger(log_type, crawler).warning(f"signature:{signature}")
@@ -627,20 +628,17 @@ class XiguaRecommend:
                 "channelId": "94349543909",
                 "count": "9",
                 "maxTime": str(int(time.time())),
-                "queryCount": "1",
+                "queryCount": str(queryCount),
                 "_signature": signature,
-                # "_signature": '_02B4Z6wo00001O38UmAAAIDBlTK5ZUm9hMDt7HbAAF9Se5',
-                # "_signature": '_02B4Z6wo0000158YzJQAAIDC59YnkMoXHRufGMgAAIP97SpOQxVfKP5yN1rB9OQ2Be5sOOQWgCiFaeOyxlnCG4RZUX7NfDmED3tHWe2-vSJ-icJj7GZCBorr2AT2MY.Tm6TzjyGTXhKwp98X5f'
-                # "maxTime": "1680867875",
-                # "request_from": "701",
-                # "offset": "0",
-                # "referrer:": "https://open.weixin.qq.com/",
-                # "aid": "1768",
-                # "msToken": "Tqe-W_gibxblmWtCV1PoAUBjAb9W9lPoz8iX8OK9MS1XfRogNdVXeoxc69AKWSEObCuHssPmeRuJe1IH_G3nmTxrJc4XJMEs5iQ2ea36jFmKCTVkJ-9p-M7gcdQz3fw=",
-                # "X-Bogus": "DFSzswVuZ6UAN9WvtV34uY/F6qyN",
+                "request_from": "701",
+                "offset": "0",
+                "referrer:": "https://open.weixin.qq.com/",
+                "aid": "1768",
+                "msToken": "_QDo1dOs65zLGMBsrVaihnOi-Jd12nK93RU9xMW1bbJi28zFVGIt1ohtduTsppL5c7EIRpDa7znh88Px8ENw9HYfmVYGkAq3a_lJeQzW0pEzZGkhzmPo",
+                "X-Bogus": "DFSzswVOzV2ANt0TtCAPrMm4pIpi",
             }
             headers = {
-                'referer': 'https://www.ixigua.com/?is_new_connect=0&is_new_user=0',
+                'referer': 'https://www.ixigua.com/',
                 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54',
                 # 'authority': 'www.ixigua.com',
                 # 'accept': 'application/json, text/plain, */*',
@@ -664,6 +662,7 @@ class XiguaRecommend:
             s.mount('https://', HTTPAdapter(max_retries=3))
             response = requests.get(url=url, headers=headers, params=params, proxies=Common.tunnel_proxies(), verify=False, timeout=5)
             response.close()
+            queryCount += 1
             if response.status_code != 200:
                 Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.text}\n")
                 return
@@ -861,8 +860,6 @@ class XiguaRecommend:
         Common.logger(log_type, crawler).info('视频信息插入数据库成功！\n')
 
 
-
-
 if __name__ == "__main__":
     # XiguaRecommend.get_signature("recommend", "xigua", "dev")
     # XiguaRecommend.get_videolist("recommend", "xigua", "dev")