Merge remote-tracking branch 'origin/master'

xigua/xigua_author/xigua_author.py

@@ -24,9 +24,9 @@ def tunnel_proxies():
     password = "5zqcjkmy"
     tunnel_proxies = {
         "http": "http://%(user)s:%(pwd)s@%(proxy)s/"
-        % {"user": username, "pwd": password, "proxy": tunnel},
+                % {"user": username, "pwd": password, "proxy": tunnel},
         "https": "http://%(user)s:%(pwd)s@%(proxy)s/"
-        % {"user": username, "pwd": password, "proxy": tunnel},
+                 % {"user": username, "pwd": password, "proxy": tunnel},
     }
 
     return tunnel_proxies
@@ -40,9 +40,9 @@ def random_signature():
     uppercase_num = random.randint(1, 26 - digits_num - 1)
     lowercase_num = 26 - (digits_num + uppercase_num)
     password = (
-        random.sample(src_digits, digits_num)
-        + random.sample(src_uppercase, uppercase_num)
-        + random.sample(src_lowercase, lowercase_num)
+            random.sample(src_digits, digits_num)
+            + random.sample(src_uppercase, uppercase_num)
+            + random.sample(src_lowercase, lowercase_num)
     )
     random.shuffle(password)
     new_password = "AAAAAAAAAA" + "".join(password)[10:-4] + "AAAB"
@@ -72,8 +72,8 @@ def get_video_url(video_info):
 
     elif "dash_120fps" in video_info["videoResource"]:
         if (
-            "video_list" in video_info["videoResource"]["dash_120fps"]
-            and "video_4" in video_info["videoResource"]["dash_120fps"]["video_list"]
+                "video_list" in video_info["videoResource"]["dash_120fps"]
+                and "video_4" in video_info["videoResource"]["dash_120fps"]["video_list"]
         ):
             video_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                 "video_4"
@@ -102,8 +102,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["dash_120fps"]
-            and "video_3" in video_info["videoResource"]["dash_120fps"]["video_list"]
+                "video_list" in video_info["videoResource"]["dash_120fps"]
+                and "video_3" in video_info["videoResource"]["dash_120fps"]["video_list"]
         ):
             video_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                 "video_3"
@@ -132,8 +132,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["dash_120fps"]
-            and "video_2" in video_info["videoResource"]["dash_120fps"]["video_list"]
+                "video_list" in video_info["videoResource"]["dash_120fps"]
+                and "video_2" in video_info["videoResource"]["dash_120fps"]["video_list"]
         ):
             video_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                 "video_2"
@@ -162,8 +162,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["dash_120fps"]
-            and "video_1" in video_info["videoResource"]["dash_120fps"]["video_list"]
+                "video_list" in video_info["videoResource"]["dash_120fps"]
+                and "video_1" in video_info["videoResource"]["dash_120fps"]["video_list"]
         ):
             video_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                 "video_1"
@@ -193,23 +193,23 @@ def get_video_url(video_info):
             video_url_dict["video_height"] = video_height
 
         elif (
-            "dynamic_video" in video_info["videoResource"]["dash_120fps"]
-            and "dynamic_video_list"
-            in video_info["videoResource"]["dash_120fps"]["dynamic_video"]
-            and "dynamic_audio_list"
-            in video_info["videoResource"]["dash_120fps"]["dynamic_video"]
-            and len(
-                video_info["videoResource"]["dash_120fps"]["dynamic_video"][
-                    "dynamic_video_list"
-                ]
-            )
-            != 0
-            and len(
-                video_info["videoResource"]["dash_120fps"]["dynamic_video"][
-                    "dynamic_audio_list"
-                ]
-            )
-            != 0
+                "dynamic_video" in video_info["videoResource"]["dash_120fps"]
+                and "dynamic_video_list"
+                in video_info["videoResource"]["dash_120fps"]["dynamic_video"]
+                and "dynamic_audio_list"
+                in video_info["videoResource"]["dash_120fps"]["dynamic_video"]
+                and len(
+            video_info["videoResource"]["dash_120fps"]["dynamic_video"][
+                "dynamic_video_list"
+            ]
+        )
+                != 0
+                and len(
+            video_info["videoResource"]["dash_120fps"]["dynamic_video"][
+                "dynamic_audio_list"
+            ]
+        )
+                != 0
         ):
             video_url = video_info["videoResource"]["dash_120fps"]["dynamic_video"][
                 "dynamic_video_list"
@@ -245,8 +245,8 @@ def get_video_url(video_info):
 
     elif "dash" in video_info["videoResource"]:
         if (
-            "video_list" in video_info["videoResource"]["dash"]
-            and "video_4" in video_info["videoResource"]["dash"]["video_list"]
+                "video_list" in video_info["videoResource"]["dash"]
+                and "video_4" in video_info["videoResource"]["dash"]["video_list"]
         ):
             video_url = video_info["videoResource"]["dash"]["video_list"]["video_4"][
                 "backup_url_1"
@@ -275,8 +275,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["dash"]
-            and "video_3" in video_info["videoResource"]["dash"]["video_list"]
+                "video_list" in video_info["videoResource"]["dash"]
+                and "video_3" in video_info["videoResource"]["dash"]["video_list"]
         ):
             video_url = video_info["videoResource"]["dash"]["video_list"]["video_3"][
                 "backup_url_1"
@@ -305,8 +305,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["dash"]
-            and "video_2" in video_info["videoResource"]["dash"]["video_list"]
+                "video_list" in video_info["videoResource"]["dash"]
+                and "video_2" in video_info["videoResource"]["dash"]["video_list"]
         ):
             video_url = video_info["videoResource"]["dash"]["video_list"]["video_2"][
                 "backup_url_1"
@@ -335,8 +335,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["dash"]
-            and "video_1" in video_info["videoResource"]["dash"]["video_list"]
+                "video_list" in video_info["videoResource"]["dash"]
+                and "video_1" in video_info["videoResource"]["dash"]["video_list"]
         ):
             video_url = video_info["videoResource"]["dash"]["video_list"]["video_1"][
                 "backup_url_1"
@@ -366,23 +366,23 @@ def get_video_url(video_info):
             video_url_dict["video_height"] = video_height
 
         elif (
-            "dynamic_video" in video_info["videoResource"]["dash"]
-            and "dynamic_video_list"
-            in video_info["videoResource"]["dash"]["dynamic_video"]
-            and "dynamic_audio_list"
-            in video_info["videoResource"]["dash"]["dynamic_video"]
-            and len(
-                video_info["videoResource"]["dash"]["dynamic_video"][
-                    "dynamic_video_list"
-                ]
-            )
-            != 0
-            and len(
-                video_info["videoResource"]["dash"]["dynamic_video"][
-                    "dynamic_audio_list"
-                ]
-            )
-            != 0
+                "dynamic_video" in video_info["videoResource"]["dash"]
+                and "dynamic_video_list"
+                in video_info["videoResource"]["dash"]["dynamic_video"]
+                and "dynamic_audio_list"
+                in video_info["videoResource"]["dash"]["dynamic_video"]
+                and len(
+            video_info["videoResource"]["dash"]["dynamic_video"][
+                "dynamic_video_list"
+            ]
+        )
+                != 0
+                and len(
+            video_info["videoResource"]["dash"]["dynamic_video"][
+                "dynamic_audio_list"
+            ]
+        )
+                != 0
         ):
             video_url = video_info["videoResource"]["dash"]["dynamic_video"][
                 "dynamic_video_list"
@@ -418,8 +418,8 @@ def get_video_url(video_info):
 
     elif "normal" in video_info["videoResource"]:
         if (
-            "video_list" in video_info["videoResource"]["normal"]
-            and "video_4" in video_info["videoResource"]["normal"]["video_list"]
+                "video_list" in video_info["videoResource"]["normal"]
+                and "video_4" in video_info["videoResource"]["normal"]["video_list"]
         ):
             video_url = video_info["videoResource"]["normal"]["video_list"]["video_4"][
                 "backup_url_1"
@@ -448,8 +448,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["normal"]
-            and "video_3" in video_info["videoResource"]["normal"]["video_list"]
+                "video_list" in video_info["videoResource"]["normal"]
+                and "video_3" in video_info["videoResource"]["normal"]["video_list"]
         ):
             video_url = video_info["videoResource"]["normal"]["video_list"]["video_3"][
                 "backup_url_1"
@@ -478,8 +478,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["normal"]
-            and "video_2" in video_info["videoResource"]["normal"]["video_list"]
+                "video_list" in video_info["videoResource"]["normal"]
+                and "video_2" in video_info["videoResource"]["normal"]["video_list"]
         ):
             video_url = video_info["videoResource"]["normal"]["video_list"]["video_2"][
                 "backup_url_1"
@@ -508,8 +508,8 @@ def get_video_url(video_info):
             video_url_dict["video_width"] = video_width
             video_url_dict["video_height"] = video_height
         elif (
-            "video_list" in video_info["videoResource"]["normal"]
-            and "video_1" in video_info["videoResource"]["normal"]["video_list"]
+                "video_list" in video_info["videoResource"]["normal"]
+                and "video_1" in video_info["videoResource"]["normal"]["video_list"]
         ):
             video_url = video_info["videoResource"]["normal"]["video_list"]["video_1"][
                 "backup_url_1"
@@ -539,23 +539,23 @@ def get_video_url(video_info):
             video_url_dict["video_height"] = video_height
 
         elif (
-            "dynamic_video" in video_info["videoResource"]["normal"]
-            and "dynamic_video_list"
-            in video_info["videoResource"]["normal"]["dynamic_video"]
-            and "dynamic_audio_list"
-            in video_info["videoResource"]["normal"]["dynamic_video"]
-            and len(
-                video_info["videoResource"]["normal"]["dynamic_video"][
-                    "dynamic_video_list"
-                ]
-            )
-            != 0
-            and len(
-                video_info["videoResource"]["normal"]["dynamic_video"][
-                    "dynamic_audio_list"
-                ]
-            )
-            != 0
+                "dynamic_video" in video_info["videoResource"]["normal"]
+                and "dynamic_video_list"
+                in video_info["videoResource"]["normal"]["dynamic_video"]
+                and "dynamic_audio_list"
+                in video_info["videoResource"]["normal"]["dynamic_video"]
+                and len(
+            video_info["videoResource"]["normal"]["dynamic_video"][
+                "dynamic_video_list"
+            ]
+        )
+                != 0
+                and len(
+            video_info["videoResource"]["normal"]["dynamic_video"][
+                "dynamic_audio_list"
+            ]
+        )
+                != 0
         ):
             video_url = video_info["videoResource"]["normal"]["dynamic_video"][
                 "dynamic_video_list"
@@ -634,9 +634,9 @@ def get_comment_cnt(item_id):
     )
     response.close()
     if (
-        response.status_code != 200
-        or "total_number" not in response.json()
-        or response.json() == {}
+            response.status_code != 200
+            or "total_number" not in response.json()
+            or response.json() == {}
     ):
         return 0
     return response.json().get("total_number", 0)
@@ -769,21 +769,22 @@ class XiGuaAuthor:
         video_dict["publish_time"] = video_dict["publish_time_str"]
         video_dict["strategy_type"] = self.mode
         video_dict["update_time_stamp"] = int(time.time())
-        if (
-                int(time.time()) - video_dict['publish_time_stamp']
-                > 3600 * 24 * int(self.rule_dict.get("period", {}).get("max", 1000))
-        ):
-            AliyunLogger.logging(
-                code="2004",
-                platform=self.platform,
-                mode=self.mode,
-                env=self.env,
-                data=video_dict,
-                message="发布时间超过{}天".format(
-                    int(self.rule_dict.get("period", {}).get("max", 1000))
-                ),
-            )
-            return False
+        if int(time.time()) - video_dict['publish_time_stamp'] > 3600 * 24 * int(self.rule_dict.get("period", {}).get("max", 1000)):
+            if not video_obj['is_top']:
+                """
+                非置顶数据发布时间超过才退出
+                """
+                AliyunLogger.logging(
+                    code="2004",
+                    platform=self.platform,
+                    mode=self.mode,
+                    env=self.env,
+                    data=video_dict,
+                    message="发布时间超过{}天".format(
+                        int(self.rule_dict.get("period", {}).get("max", 1000))
+                    ),
+                )
+                return False
 
         pipeline = PiaoQuanPipeline(
             platform=self.platform,
@@ -829,7 +830,7 @@ class XiGuaAuthor:
                     data=video_dict,
                     trace_id=trace_id,
                     message="成功发送 MQ 至 ETL",
-                    )
+                )
                 return True
             else:
                 AliyunLogger.logging(
@@ -853,22 +854,22 @@ class XiGuaAuthor:
         params = {
             "mixId": str(item_id),
             "msToken": "IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfC"
-            "NVVIOBNjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA",
+                       "NVVIOBNjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA",
             "X-Bogus": "DFSzswVupYTANCJOSBk0P53WxM-r",
             "_signature": "_02B4Z6wo0000119LvEwAAIDCuktNZ0y5wkdfS7jAALThuOR8D9yWNZ.EmWHKV0WSn6Px"
-            "fPsH9-BldyxVje0f49ryXgmn7Tzk-swEHNb15TiGqa6YF.cX0jW8Eds1TtJOIZyfc9s5emH7gdWN94",
+                          "fPsH9-BldyxVje0f49ryXgmn7Tzk-swEHNb15TiGqa6YF.cX0jW8Eds1TtJOIZyfc9s5emH7gdWN94",
         }
         cookies = {
             "ixigua-a-s": "1",
             "msToken": "IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfCNVVIOB"
-            "NjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA",
+                       "NjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA",
             "ttwid": "1%7C_yXQeHWwLZgCsgHClOwTCdYSOt_MjdOkgnPIkpi-Sr8%7C1661241238%7Cf57d0c5ef3f1d7"
-            "6e049fccdca1ac54887c34d1f8731c8e51a49780ff0ceab9f8",
+                     "6e049fccdca1ac54887c34d1f8731c8e51a49780ff0ceab9f8",
             "tt_scid": "QZ4l8KXDG0YAEaMCSbADdcybdKbUfG4BC6S4OBv9lpRS5VyqYLX2bIR8CTeZeGHR9ee3",
             "MONITOR_WEB_ID": "0a49204a-7af5-4e96-95f0-f4bafb7450ad",
             "__ac_nonce": "06304878000964fdad287",
             "__ac_signature": "_02B4Z6wo00f017Rcr3AAAIDCUVxeW1tOKEu0fKvAAI4cvoYzV-wBhq7B6D8k0no7lb"
-            "FlvYoinmtK6UXjRIYPXnahUlFTvmWVtb77jsMkKAXzAEsLE56m36RlvL7ky.M3Xn52r9t1IEb7IR3ke8",
+                              "FlvYoinmtK6UXjRIYPXnahUlFTvmWVtb77jsMkKAXzAEsLE56m36RlvL7ky.M3Xn52r9t1IEb7IR3ke8",
             "ttcid": "e56fabf6e85d4adf9e4d91902496a0e882",
             "_tea_utm_cache_1300": "undefined",
             "support_avif": "false",
@@ -885,9 +886,9 @@ class XiGuaAuthor:
             timeout=5,
         )
         if (
-            response.status_code != 200
-            or "data" not in response.json()
-            or response.json()["data"] == {}
+                response.status_code != 200
+                or "data" not in response.json()
+                or response.json()["data"] == {}
         ):
             AliyunLogger.logging(
                 code="2000",

xigua/xigua_author/xigua_author_test.py

@@ -898,8 +898,11 @@ class XiGuaAuthor:
             else:
                 feeds = response.json()["data"]["videoList"]
                 for video_obj in feeds:
-
+                    print(video_obj['is_top'])
+                    # print(json.dumps(video_obj, ensure_ascii=False, indent=4))
+                    # return
                     self.process_video_obj(video_obj, user_dict)
+
                     # try:
                     #     print("扫描到一条视频")
                     #     self.process_video_obj(video_obj, user_dict)
@@ -946,33 +949,17 @@ if __name__ == "__main__":
         {
             "uid": 6267140,
             "source": "xigua",
-            "link": "https://www.ixigua.com/home/2779177225827568",
-            "nick_name": "秋晴爱音乐",
-            "avatar_url": "",
-            "mode": "author",
-        },
-        {
-            "uid": 6267140,
-            "source": "xigua",
-            "link": "https://www.ixigua.com/home/2885546124776780",
-            "nick_name": "朗诵放歌的老山羊",
+            "link": "https://www.ixigua.com/home/113976532286319/?list_entrance=anyVideo",
+            "nick_name": "云姐犹记",
             "avatar_url": "",
             "mode": "author",
-        },
-        {
-            "uid": 6267140,
-            "source": "xigua",
-            "link": "https://www.ixigua.com/home/5880938217",
-            "nick_name": "天原声疗",
-            "avatar_url": "",
-            "mode": "author",
-        },
+        }
     ]
-    rule = {'period': {'min': 30, 'max': 30}, 'duration': {'min': 20, 'max': 0}, 'play_cnt': {'min': 100, 'max': 0}}
+    # rule = {'period': {'min': 30, 'max': 30}, 'duration': {'min': 20, 'max': 0}, 'play_cnt': {'min': 100, 'max': 0}}
     XGA = XiGuaAuthor(
         platform="xigua",
         mode="author",
-        rule_dict=rule,
+        rule_dict={},
         env="prod",
         user_list=user_list
     )