Merge remote-tracking branch 'origin/master'

xiaoniangao/xiaoniangao_author/author_test.py

@@ -1,32 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Author: wangkun
-# @Time: 2023/6/19
-import json
-
-import requests
-
-url = "https://kapi-xng-app.xiaoniangao.cn/v1/album/user_public"
-next_t = -1
-visited_mid = 297080453
-payload = "{\"share_width\":300,\"log\":{\"brand\":\"iPad\",\"net\":\"wifi\",\"resolution\":\"750*1334\",\"uid\":\"2F310D09-5E32-5985-8644-3BCB6920E76F\",\"app_version\":\"1.22.5\",\"channel\":\"ios_app_store\",\"page\":\"\",\"product\":\"xng\",\"os_version\":\"15.7\",\"pf\":\"4\",\"session_id\":\"47D7817B-AAB1-4E70-BA7F-B868FC9AA21F\",\"idfa\":\"\",\"device\":\"iPad Pro (12.9-inch) (3rd generation)\"},\"qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!690x385r\\/interlace\\/1\\/format\\/jpg\",\"share_height\":240,\"start_t\":-1,\"token\":\"\",\"visited_mid\":211201301,\"limit\":20}"
-payload_dic = json.loads(payload)
-payload_dic['visited_mid'] = visited_mid
-payload_new = json.dumps(payload_dic)
-headers = {
-  'Host': 'kapi-xng-app.xiaoniangao.cn',
-  'content-type': 'application/json; charset=utf-8',
-  'accept': '*/*',
-  'authorization': 'hSNQ2s9pvPxvFn4LaQJxKQ6/7Is=',
-  'verb': 'POST',
-  'content-md5': 'c7b7f8663984e8800e3bcd9b44465083',
-  'x-b3-traceid': '2f9da41f960ae077',
-  'accept-language': 'zh-cn',
-  'date': 'Mon, 19 Jun 2023 06:41:17 GMT',
-  'x-token-id': '',
-  'x-signaturemethod': 'hmac-sha1',
-  'user-agent': 'xngapp/157 CFNetwork/1335.0.3.1 Darwin/21.6.0'
-}
-
-response = requests.post(url, headers=headers, data=payload_new)
-
-print(response.text)

xiaoniangao/xiaoniangao_author/xiaoniangao_author_v2.py

@@ -43,7 +43,7 @@ class XiaoNianGaoAuthor:
 
     def get_author_list(self):
         # 每轮只抓取定量的数据，到达数量后自己退出
-        max_count = int(self.rule_dict.get("videos_cnt", {}).get("min", 200))
+        max_count = int(self.rule_dict.get("videos_cnt", {}).get("min", 300))
         for user_dict in self.user_list:
             if self.download_count <= max_count:
                 self.get_video_list(user_dict)
@@ -218,6 +218,7 @@ class XiaoNianGaoAuthor:
             video_dict["publish_time"] = video_dict["publish_time_str"]
             # print(video_dict)
             self.mq.send_msg(video_dict)
+            self.download_count += 1
             AliyunLogger.logging(
                 code="1002",
                 platform=self.platform,
@@ -229,12 +230,12 @@ class XiaoNianGaoAuthor:
             )
 
 
-if __name__ == "__main__":
-    XNGA = XiaoNianGaoAuthor(
-        platform="xiaoniangao",
-        mode="author",
-        rule_dict={},
-        env="prod",
-        user_list=[{"link": 295640510, "uid": "12334"}],
-    )
-    XNGA.get_author_list()
+# if __name__ == "__main__":
+#     XNGA = XiaoNianGaoAuthor(
+#         platform="xiaoniangao",
+#         mode="author",
+#         rule_dict={},
+#         env="prod",
+#         user_list=[{"link": 295640510, "uid": "12334"}],
+#     )
+#     XNGA.get_author_list()

xiaoniangao/xiaoniangao_main/run_xng_author_v2.py

@@ -77,15 +77,15 @@ def main(my_platform, mode, topic_name, group_id, env):
                 user_list = MysqlHelper.get_values(
                     mode, my_platform, select_user_sql, env, action=""
                 )
-                our_uid_list = []
-                our_uid = random.choice(our_uid_list)
+                # our_uid_list = []
+                # our_uid = random.choice(our_uid_list)
                 AliyunLogger.logging(
                     code="1003",
                     platform=my_platform,
                     mode=mode,
                     env=env,
                     message="成功获取信息，启动爬虫，开始一轮抓取",
-                )
+               )
                 XNGAuthor = XiaoNianGaoAuthor(
                     platform=my_platform,
                     mode=mode,