update

xiaoniangao/xiaoniangao_hour/video_detail.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/6/19
+import json
+import requests
+v_id = 53456805
+p_mid = 13988857
+album_id = 5077977542
+url = "https://kapi-xng-app.xiaoniangao.cn/v3/album/detail"
+serial_id = 'sQ0Gh01aa3sw0002'
+tpl_id = 100022
+
+payload = "{\"log_ext\":{\"from_page\":\"\"},\"detail\":{\"tpl_id\":0,\"id\":53509608,\"share_width\":300,\"ab\":null,\"album_id\":5082121772,\"serial_id\":\"sQ0Gh01aa2Gy000H\",\"profile_mid\":1160417275,\"ref\":\"\",\"stpl_id\":null,\"share_height\":240,\"rec_sign\":\"app_dft_v1\"},\"refresh\":false,\"log\":{\"net\":\"wifi\",\"product\":\"xng\",\"page\":\"\",\"idfa\":\"\",\"brand\":\"iPad\",\"pf\":\"4\",\"os_version\":\"15.7\",\"uid\":\"2F310D09-5E32-5985-8644-3BCB6920E76F\",\"channel\":\"ios_app_store\",\"device\":\"iPad Pro (12.9-inch) (3rd generation)\",\"resolution\":\"750*1334\",\"session_id\":\"F4249B34-B3C6-4AEE-824E-B95BFE3AC8AE\",\"app_version\":\"1.22.5\"},\"rec\":{\"ext\":{\"current_item\":53509608,\"items\":[{\"play_du\":2,\"id\":53446151}]}},\"staggered_style\":0,\"token\":\"\"}"
+payload_dic = json.loads(payload)
+# payload_dic['detail']['id'] = v_id
+# payload_dic['detail']['profile_mid'] = p_mid
+# payload_dic['detail']['album_id'] = album_id
+# payload_dic['detail']['serial_id'] = serial_id
+# payload_dic['detail']['tpl_id'] = tpl_id
+payload_new = json.dumps(payload_dic)
+headers = {
+  'Host': 'kapi-xng-app.xiaoniangao.cn',
+  'content-type': 'application/json; charset=utf-8',
+  'accept': 'application/json',
+  'authorization': 'f1uxuL49wq2h3T+rYLA4ZeiIemQ=',
+  'verb': 'POST',
+  'content-md5': '5ab99134ed723cd49c0b30b0da24fd10',
+  'x-b3-traceid': '5449e3265c96701b',
+  'accept-language': 'zh-Hans-CN;q=1.0',
+  'date': 'Mon, 19 Jun 2023 11:20:37 GMT',
+  'x-token-id': '',
+  'x-signaturemethod': 'hmac-sha1',
+  'user-agent': 'xngapp/1.22.5 (cn.xiaoniangao.xngapp; build:157; iOS 15.7.0) Alamofire/5.2.2'
+}
+
+response = requests.request("POST", url, headers=headers, data=payload)
+
+print(response.text)

xiaoniangao/xiaoniangao_hour/xiaoniangao_hour_scheduling.py

@@ -44,71 +44,24 @@ class XiaoniangaoHourScheduling:
     @classmethod
     def get_videoList(cls, log_type, crawler, rule_dict, env):
         uid_token_dict = cls.uid_token_dict
-        url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends"
+        url = "https://kapi-xng-app.xiaoniangao.cn/v2/trends/recommend"
+        payload = "{\"topic_name\":\"recommend\",\"ext\":{\"current_item\":0,\"items\":[]},\"tag_id\":0,\"refresh\":false,\"share_width\":300,\"share_height\":240,\"staggered_style\":0,\"qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!750x500r\\/interlace\\/1\\/format\\/jpg\",\"topic_id\":2,\"h_qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!80x80r\\/interlace\\/1\\/format\\/jpg\",\"log\":{\"net\":\"wifi\",\"product\":\"xng\",\"uid\":\"2F310D09-5E32-5985-8644-3BCB6920E76F\",\"brand\":\"iPad\",\"page\":\"\",\"session_id\":\"71C77648-3224-4083-894C-B6282131F286\",\"resolution\":\"750*1334\",\"pf\":\"4\",\"app_version\":\"1.22.5\",\"device\":\"iPad Pro (12.9-inch) (3rd generation)\",\"os_version\":\"15.7\",\"idfa\":\"\",\"channel\":\"ios_app_store\"},\"token\":\"\"}"
         headers = {
-            "x-b3-traceid": '1c403a4aa72e3c',
-            "X-Token-Id": 'ab619e96d801f1567388629260aa68ec-1202200806',
-            "uid": uid_token_dict['uid'],
-            "content-type": "application/json",
-            "Accept-Encoding": "gzip,compress,br,deflate",
-            "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
-                          ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
-                          'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
-            "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/624/page-frame.html'
-        }
-        data = {
-            "log_params": {
-                "page": "discover_rec",
-                "common": {
-                    "brand": "iPhone",
-                    "device": "iPhone 11",
-                    "os": "iOS 14.7.1",
-                    "weixinver": "8.0.20",
-                    "srcver": "2.24.2",
-                    "net": "wifi",
-                    "scene": 1089
-                }
-            },
-            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg",
-            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg",
-            "share_width": 625,
-            "share_height": 500,
-            "ext": {
-                "fmid": 0,
-                "items": {}
-            },
-            "app": "xng",
-            "rec_scene": "discover_rec",
-            "log_common_params": {
-                "e": [{
-                    "data": {
-                        "page": "discoverIndexPage",
-                        "topic": "recommend"
-                    },
-                    "ab": {}
-                }],
-                "ext": {
-                    "brand": "iPhone",
-                    "device": "iPhone 11",
-                    "os": "iOS 14.7.1",
-                    "weixinver": "8.0.20",
-                    "srcver": "2.24.3",
-                    "net": "wifi",
-                    "scene": "1089"
-                },
-                "pj": "1",
-                "pf": "2",
-                "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29"
-            },
-            "refresh": False,
-            "token": uid_token_dict["token"],
-            "uid": uid_token_dict["uid"],
-            "proj": "ma",
-            "wx_ver": "8.0.20",
-            "code_ver": "3.62.0"
+            'Host': 'kapi-xng-app.xiaoniangao.cn',
+            'content-type': 'application/json; charset=utf-8',
+            'accept': 'application/json',
+            'authorization': 'PsrUTBCQ5G7UVZdgx+JxymPHcKU=',
+            'verb': 'POST',
+            'content-md5': '08fa0e6bf725fd6ef83c16d2ceb8a544',
+            'x-b3-traceid': '45a6c5b4c471eecc',
+            'accept-language': 'zh-Hans-CN;q=1.0',
+            'date': 'Mon, 19 Jun 2023 09:47:40 GMT',
+            'x-token-id': '',
+            'x-signaturemethod': 'hmac-sha1',
+            'user-agent': 'xngapp/1.22.5 (cn.xiaoniangao.xngapp; build:157; iOS 15.7.0) Alamofire/5.2.2'
         }
         urllib3.disable_warnings()
-        r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False)
+        r = requests.post(url=url, headers=headers, data=payload, proxies=proxies, verify=False)
         if 'data' not in r.text or r.status_code != 200:
             Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
             Common.logging(log_type, crawler, env, f"get_videoList:{r.text}\n")
@@ -147,7 +100,7 @@ class XiaoniangaoHourScheduling:
                     publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))
                     # 用户名 / 头像
                     user_name = feeds[i].get("user", {}).get("nick", "").strip().replace("\n", "") \
-                            .replace("/", "").replace("快手", "").replace(" ", "") \
+                            .replace("/", "").replace(" ", "") \
                             .replace(" ", "").replace("&NBSP", "").replace("\r", "")
 
                     video_dict = {
@@ -161,8 +114,8 @@ class XiaoniangaoHourScheduling:
                         "user_name": user_name,
                         "publish_time_stamp": publish_time_stamp,
                         "publish_time_str": publish_time_str,
-                        "video_width": int(feeds[i].get("w", 0)),
-                        "video_height": int(feeds[i].get("h", 0)),
+                        "video_width": int(feeds[i].get("vw", 0)),
+                        "video_height": int(feeds[i].get("vh", 0)),
                         "avatar_url": feeds[i].get("user", {}).get("hurl", ""),
                         "profile_id": feeds[i]["id"],
                         "profile_mid": feeds[i]["user"]["mid"],

xiaoniangao/xiaoniangao_play/play_test.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# @Author: wangkun
+# @Time: 2023/6/19
+import requests
+
+url = "https://kapi-xng-app.xiaoniangao.cn/v2/trends/recommend"
+
+payload = "{\"topic_name\":\"recommend\",\"ext\":{\"current_item\":0,\"items\":[]},\"tag_id\":0,\"refresh\":false,\"share_width\":300,\"share_height\":240,\"staggered_style\":0,\"qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!750x500r\\/interlace\\/1\\/format\\/jpg\",\"topic_id\":2,\"h_qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!80x80r\\/interlace\\/1\\/format\\/jpg\",\"log\":{\"net\":\"wifi\",\"product\":\"xng\",\"uid\":\"2F310D09-5E32-5985-8644-3BCB6920E76F\",\"brand\":\"iPad\",\"page\":\"\",\"session_id\":\"71C77648-3224-4083-894C-B6282131F286\",\"resolution\":\"750*1334\",\"pf\":\"4\",\"app_version\":\"1.22.5\",\"device\":\"iPad Pro (12.9-inch) (3rd generation)\",\"os_version\":\"15.7\",\"idfa\":\"\",\"channel\":\"ios_app_store\"},\"token\":\"\"}"
+
+headers = {
+  'Host': 'kapi-xng-app.xiaoniangao.cn',
+  'content-type': 'application/json; charset=utf-8',
+  'accept': 'application/json',
+  'authorization': 'PsrUTBCQ5G7UVZdgx+JxymPHcKU=',
+  'verb': 'POST',
+  'content-md5': '08fa0e6bf725fd6ef83c16d2ceb8a544',
+  'x-b3-traceid': '45a6c5b4c471eecc',
+  'accept-language': 'zh-Hans-CN;q=1.0',
+  'date': 'Mon, 19 Jun 2023 09:47:40 GMT',
+  'x-token-id': '',
+  'x-signaturemethod': 'hmac-sha1',
+  'user-agent': 'xngapp/1.22.5 (cn.xiaoniangao.xngapp; build:157; iOS 15.7.0) Alamofire/5.2.2'
+}
+
+response = requests.request("POST", url, headers=headers, data=payload)
+
+print(response.text)

xiaoniangao/xiaoniangao_play/xiaoniangao_play_scheduling.py

@@ -29,78 +29,29 @@ class XiaoniangaoplayScheduling:
     # 获取列表
     @classmethod
     def get_videoList(cls, log_type, crawler, rule_dict, our_uid, env):
-        uid_token_dict = cls.uid_token_dict
+        # uid_token_dict = cls.uid_token_dict
         for page in range(1, 101):
             try:
                 Common.logger(log_type, crawler).info(f"正在抓取第{page}页")
                 Common.logging(log_type, crawler, env, f"正在抓取第{page}页")
-                url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends"
+                url = "https://kapi-xng-app.xiaoniangao.cn/v2/trends/recommend"
+                payload = "{\"topic_name\":\"recommend\",\"ext\":{\"current_item\":0,\"items\":[]},\"tag_id\":0,\"refresh\":false,\"share_width\":300,\"share_height\":240,\"staggered_style\":0,\"qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!750x500r\\/interlace\\/1\\/format\\/jpg\",\"topic_id\":2,\"h_qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!80x80r\\/interlace\\/1\\/format\\/jpg\",\"log\":{\"net\":\"wifi\",\"product\":\"xng\",\"uid\":\"2F310D09-5E32-5985-8644-3BCB6920E76F\",\"brand\":\"iPad\",\"page\":\"\",\"session_id\":\"71C77648-3224-4083-894C-B6282131F286\",\"resolution\":\"750*1334\",\"pf\":\"4\",\"app_version\":\"1.22.5\",\"device\":\"iPad Pro (12.9-inch) (3rd generation)\",\"os_version\":\"15.7\",\"idfa\":\"\",\"channel\":\"ios_app_store\"},\"token\":\"\"}"
                 headers = {
-                    "x-b3-traceid": '1dc0a6d0929a2b',
-                    "X-Token-Id": 'ae99a4953804085ebb0ae36fa138031d-1146052582',
-                    "uid": uid_token_dict['uid'],
-                    "content-type": "application/json",
-                    "Accept-Encoding": "gzip,compress,br,deflate",
-                    "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
-                                  ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
-                                  'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
-                    "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/620/page-frame.html'
-                }
-                data = {
-                    "log_params": {
-                        "page": "discover_rec",
-                        "common": {
-                            "brand": "iPhone",
-                            "device": "iPhone 11",
-                            "os": "iOS 14.7.1",
-                            "weixinver": "8.0.20",
-                            "srcver": "2.24.2",
-                            "net": "wifi",
-                            "scene": 1089
-                        }
-                    },
-                    "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg",
-                    "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg",
-                    "share_width": 625,
-                    "share_height": 500,
-                    "ext": {
-                        "fmid": 0,
-                        "items": {}
-                    },
-                    "app": "xng",
-                    "rec_scene": "discover_rec",
-                    "log_common_params": {
-                        "e": [{
-                            "data": {
-                                "page": "discoverIndexPage",
-                                "topic": "recommend"
-                            },
-                            "ab": {}
-                        }],
-                        "ext": {
-                            "brand": "iPhone",
-                            "device": "iPhone 11",
-                            "os": "iOS 14.7.1",
-                            "weixinver": "8.0.20",
-                            "srcver": "2.24.3",
-                            "net": "wifi",
-                            "scene": "1089"
-                        },
-                        "pj": "1",
-                        "pf": "2",
-                        "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29"
-                    },
-                    "refresh": False,
-                    # "token": cls.play_token,
-                    "token": uid_token_dict['token'],
-                    # "uid": cls.play_uid,
-                    "uid": uid_token_dict['uid'],
-                    "proj": "ma",
-                    "wx_ver": "8.0.20",
-                    "code_ver": "3.62.0"
+                    'Host': 'kapi-xng-app.xiaoniangao.cn',
+                    'content-type': 'application/json; charset=utf-8',
+                    'accept': 'application/json',
+                    'authorization': 'PsrUTBCQ5G7UVZdgx+JxymPHcKU=',
+                    'verb': 'POST',
+                    'content-md5': '08fa0e6bf725fd6ef83c16d2ceb8a544',
+                    'x-b3-traceid': '45a6c5b4c471eecc',
+                    'accept-language': 'zh-Hans-CN;q=1.0',
+                    'date': 'Mon, 19 Jun 2023 09:47:40 GMT',
+                    'x-token-id': '',
+                    'x-signaturemethod': 'hmac-sha1',
+                    'user-agent': 'xngapp/1.22.5 (cn.xiaoniangao.xngapp; build:157; iOS 15.7.0) Alamofire/5.2.2'
                 }
                 urllib3.disable_warnings()
-                r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False)
+                r = requests.post(url=url, headers=headers, data=payload, proxies=proxies, verify=False)
                 if "data" not in r.text or r.status_code != 200:
                     Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
                     Common.logging(log_type, crawler, env, f"get_videoList:{r.text}\n")
@@ -153,8 +104,8 @@ class XiaoniangaoplayScheduling:
                                 "user_name": user_name,
                                 "publish_time_stamp": publish_time_stamp,
                                 "publish_time_str": publish_time_str,
-                                "video_width": int(feeds[i].get("w", 0)),
-                                "video_height": int(feeds[i].get("h", 0)),
+                                "video_width": int(feeds[i].get("vw", 0)),
+                                "video_height": int(feeds[i].get("vh", 0)),
                                 "avatar_url": feeds[i].get("user", {}).get("hurl", ""),
                                 "profile_id": feeds[i]["id"],
                                 "profile_mid": feeds[i]["user"]["mid"],