提交祝万物复苏代码，

analysis/analysis.py

@@ -41,7 +41,7 @@ class Analysis(object):
 class Drawer(object):
     def __init__(self, json_obj):
         self.ori_data = json_obj
-        self.x_list = ["2023-11-{:02}".format(i) for i in range(1, 22)]
+        self.x_list = ["2023-11-{:02}".format(i) for i in range(1, 24)]
 
     def draw_line(self):
         line = Line()
@@ -76,5 +76,6 @@ if __name__ == '__main__':
     A = Analysis()
     A.analysis_videos()
     result_data = A.out_put
+    print(json.dumps(result_data['gongzhonghao'], ensure_ascii=False, indent=4))
     D = Drawer(result_data)
     D.draw_line()

common/__init__.py

@@ -1,3 +1,4 @@
 from .aliyun_log import AliyunLogger
 from .redirect_url import get_redirect_url
-from .pipeline import PiaoQuanPipeline
+from .pipeline import PiaoQuanPipeline
+from .proxy import tunnel_proxies

common/proxy.py

@@ -0,0 +1,11 @@
+def tunnel_proxies():
+    # 隧道域名:端口号
+    tunnel = "q796.kdltps.com:15818"
+    # 用户名密码方式
+    username = "t17772369458618"
+    password = "5zqcjkmy"
+    proxies = {
+        "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
+        "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
+    }
+    return proxies

zhuwanwufusu/crypt/__init__.py

@@ -0,0 +1 @@
+from .crypt import AESCipher

zhuwanwufusu/crypt/crypt.py

@@ -0,0 +1,26 @@
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.backends import default_backend
+
+
+class AESCipher:
+    def __init__(self):
+        self.key = b'50102fa64073ad76'  # 用适当的方式转换或直接定义为字节串
+        self.iv = b'173d023138824bb0'  # 同上
+
+    def aes_encrypt(self, data):
+        cipher = Cipher(algorithms.AES(self.key), modes.CBC(self.iv), backend=default_backend())
+        encryptor = cipher.encryptor()
+        ct = encryptor.update(self._pad(data).encode()) + encryptor.finalize()
+        return ct.hex().upper()
+
+    def aes_decrypt(self, data):
+        cipher = Cipher(algorithms.AES(self.key), modes.CBC(self.iv), backend=default_backend())
+        decryptor = cipher.decryptor()
+        decrypted_data = decryptor.update(bytes.fromhex(data)) + decryptor.finalize()
+        return self._unpad(decrypted_data).decode()
+
+    def _pad(self, s):
+        return s + (16 - len(s) % 16) * chr(16 - len(s) % 16)
+
+    def _unpad(self, s):
+        return s[:-ord(s[len(s) - 1:])]

zhuwanwufusu/zhuwanwufusu_recommend/zwwfs_recommend.py

@@ -0,0 +1,142 @@
+import os
+import re
+import base64
+import json
+import random
+import sys
+import time
+import uuid
+
+import requests
+
+sys.path.append(os.getcwd())
+from common.video_item import VideoItem
+from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies
+from common.mq import MQ
+from common.scheduling_db import MysqlHelper
+from zhuwanwufusu.crypt import AESCipher as AES
+
+
+class ZhuWanWuFuSuRecommend(object):
+    def __init__(self, platform, mode, rule_dict, user_dict, env):
+        self.platform = platform
+        self.mode = mode
+        self.rule_dict = rule_dict
+        self.user_dict = user_dict
+        self.env = env
+        self.download_cnt = 0
+        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
+        self.expire_flag = False
+        self.cryptor = AES()
+
+    def get_recommend_list(self):
+        url = "https://api.lidongze.cn/jeecg-boot/ugc/getVideoListsEn2"
+        headers = {
+            'Host': 'api.lidongze.cn',
+            'xweb_xhr': '1',
+            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.4(0x13080410)XWEB/31009',
+            'token': '',
+            'content-type': 'application/json',
+            'accept': '*/*',
+            'referer': 'https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html',
+            'accept-language': 'en-US,en;q=0.9'
+        }
+        page_index = 1
+        total_page = 2
+        while page_index <= total_page:
+            try:
+                query = {
+                    "pageNo": page_index,
+                    "pageSize": 10,
+                    "groupId": "1650323161797439489",  # 推荐流的 ID
+                    "vn": 1,
+                    "gx": 1,
+                    "appid": "wx0afdc2669ed8df2f",
+                    "type": 0
+                }
+                params = {
+                    "v": self.cryptor.aes_encrypt(data=json.dumps(query))
+                }
+                response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
+                result = json.loads(self.cryptor.aes_decrypt(response.text))
+                total_page = result['list']['pages']
+                page_index = result['list']['current'] + 1
+                for index, video_obj in enumerate(result['list']['records'], 1):
+                    try:
+                        AliyunLogger.logging(
+                            code="1001",
+                            platform=self.platform,
+                            mode=self.mode,
+                            env=self.env,
+                            message="扫描到一条视频",
+                            data=video_obj
+                        )
+                        self.process_video_obj(video_obj)
+                    except Exception as e:
+                        AliyunLogger.logging(
+                            code="3000",
+                            platform=self.platform,
+                            mode=self.mode,
+                            env=self.env,
+                            message="抓取单条视频失败, 该视频位于第{}页第{}条报错原因是{}".format(page_index, index, e)
+                        )
+            except Exception as e:
+                AliyunLogger.logging(
+                    code="3000",
+                    platform=self.platform,
+                    mode=self.mode,
+                    env=self.env,
+                    message="抓取第{}页的时候失败, 报错原因是{}".format(page_index, e)
+                )
+
+    def process_video_obj(self, video_obj):
+        trace_id = self.platform + str(uuid.uuid1())
+        play_cnt = int(video_obj['playnum'].replace("万+", "0000")) if "万+" in video_obj['playnum'] else int(
+            video_obj['playnum'])
+        item = VideoItem()
+        item.add_video_info("video_id", video_obj['id'])
+        item.add_video_info("video_title", video_obj['vname'])
+        item.add_video_info("play_cnt", play_cnt)
+        item.add_video_info("publish_time_stamp", int(time.time()))
+        item.add_video_info("out_user_id", video_obj['authid'])
+        item.add_video_info("cover_url", video_obj['shareimg'])
+        item.add_video_info("like_cnt", int(video_obj['likenum']))
+        item.add_video_info("video_url", video_obj['videoaddr'])
+        item.add_video_info("out_video_id", video_obj['id'])
+        item.add_video_info("platform", self.platform)
+        item.add_video_info("strategy", self.mode)
+        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
+        item.add_video_info("user_id", self.user_dict['uid'])
+        item.add_video_info("user_name", self.user_dict['nick_name'])
+
+        mq_obj = item.produce_item()
+        pipeline = PiaoQuanPipeline(
+            platform=self.platform,
+            mode=self.mode,
+            rule_dict=self.rule_dict,
+            env=self.env,
+            item=mq_obj,
+            trace_id=trace_id,
+        )
+        if pipeline.process_item():
+            self.download_cnt += 1
+            self.mq.send_msg(mq_obj)
+            AliyunLogger.logging(
+                code="1002",
+                platform=self.platform,
+                mode=self.mode,
+                env=self.env,
+                message="成功发送至 ETL",
+                data=mq_obj
+            )
+
+
+if __name__ == '__main__':
+    Z = ZhuWanWuFuSuRecommend(
+        platform="zhuwanwufusu",
+        mode="recommend",
+        rule_dict={},
+        user_dict={"uid": 123456, "nick_name": "luojunhuishuaige"},
+        env="prod"
+    )
+    Z.get_recommend_list()

zhuwanwufusu/zhuwanwufusu_recommend/zwwfs_recommend_test.py

@@ -0,0 +1,109 @@
+import os
+import re
+import base64
+import json
+import random
+import sys
+import time
+import uuid
+
+import requests
+
+sys.path.append(os.getcwd())
+from common.video_item import VideoItem
+from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies
+from common.mq import MQ
+from common.scheduling_db import MysqlHelper
+from zhuwanwufusu.crypt import AESCipher as AES
+
+
+class ZhuWanWuFuSuRecommend(object):
+    def __init__(self, platform, mode, rule_dict, user_dict, env):
+        self.platform = platform
+        self.mode = mode
+        self.rule_dict = rule_dict
+        self.user_dict = user_dict
+        self.env = env
+        self.download_cnt = 0
+        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
+        self.expire_flag = False
+        self.cryptor = AES()
+
+    def get_recommend_list(self):
+        url = "https://api.lidongze.cn/jeecg-boot/ugc/getVideoListsEn2"
+        headers = {
+            'Host': 'api.lidongze.cn',
+            'xweb_xhr': '1',
+            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.4(0x13080410)XWEB/31009',
+            'token': '',
+            'content-type': 'application/json',
+            'accept': '*/*',
+            'referer': 'https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html',
+            'accept-language': 'en-US,en;q=0.9'
+        }
+        page_index = 1
+        total_page = 2
+        while page_index <= total_page:
+            query = {
+                "pageNo": page_index,
+                "pageSize": 10,
+                "groupId": "1650323161797439489",  # 推荐流的 ID
+                "vn": 1,
+                "gx": 1,
+                "appid": "wx0afdc2669ed8df2f",
+                "type": 0
+            }
+            params = {
+                "v": self.cryptor.aes_encrypt(data=json.dumps(query))
+            }
+            response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
+            result = json.loads(self.cryptor.aes_decrypt(response.text))
+            total_page = result['list']['pages']
+            page_index = result['list']['current'] + 1
+            for index, video_obj in enumerate(result['list']['records']):
+                self.process_video_obj(video_obj)
+
+    def process_video_obj(self, video_obj):
+        trace_id = self.platform + str(uuid.uuid1())
+        play_cnt = int(video_obj['playnum'].replace("万+", "0000")) if "万+" in video_obj['playnum'] else int(
+            video_obj['playnum'])
+        item = VideoItem()
+        item.add_video_info("video_id", video_obj['id'])
+        item.add_video_info("video_title", video_obj['vname'])
+        item.add_video_info("play_cnt", play_cnt)
+        item.add_video_info("publish_time_stamp", int(time.time()))
+        item.add_video_info("out_user_id", video_obj['authid'])
+        item.add_video_info("cover_url", video_obj['shareimg'])
+        item.add_video_info("like_cnt", int(video_obj['likenum']))
+        item.add_video_info("video_url", video_obj['videoaddr'])
+        item.add_video_info("out_video_id", video_obj['id'])
+        item.add_video_info("platform", self.platform)
+        item.add_video_info("strategy", self.mode)
+        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
+        item.add_video_info("user_id", self.user_dict['uid'])
+        item.add_video_info("user_name", self.user_dict['nick_name'])
+
+        mq_obj = item.produce_item()
+        pipeline = PiaoQuanPipeline(
+            platform=self.platform,
+            mode=self.mode,
+            rule_dict=self.rule_dict,
+            env=self.env,
+            item=mq_obj,
+            trace_id=trace_id,
+        )
+        if pipeline.process_item():
+            print(json.dumps(mq_obj, ensure_ascii=False, indent=4))
+            self.download_cnt += 1
+            print(self.download_cnt)
+
+
+if __name__ == '__main__':
+    Z = ZhuWanWuFuSuRecommend(
+        platform="zhuwanwufusu",
+        mode="recommend",
+        rule_dict={},
+        user_dict={"uid": 123456, "nick_name": "luojunhuishuaige"},
+        env="dev"
+    )
+    Z.get_recommend_list()