祝万物复苏 2 上线

application/config/topic_group_queue.py

@@ -6,7 +6,8 @@ class TopicGroup(object):
         self.spider_list = [
             # ("test", "recommend", "test"),
             ("zhsdm", "recommend", "zhuhaoshiduomo"),
-            ("fxs", "recommend", "fuxiaoshun")
+            ("fxs", "recommend", "fuxiaoshun"),
+            ("zwwfsn", "recommend", "zhuwanwufusunew")
         ]
 
     def produce(self):

spider/crawler_online/__init__.py

@@ -1,2 +1,3 @@
 from .zhuhaoshiduomo import ZhuHaoShiDuoMoRecommend
+from .zhuwanwufusu_2 import ZhuWanWuFuSuRecommend
 from .fuxiaoshun import FuXiaoShunRecommend

spider/crawler_online/gongzhonghao_author.py

@@ -3,6 +3,47 @@
 @file: gongzhonghao.py
 @time: 2024/01/05
 """
+import os
+import re
+import sys
+import html
+import json
+import time
+import uuid
+import random
+import requests
+import datetime
+
+sys.path.append(os.getcwd())
+
+from application.items import VideoItem
+from application.pipeline import PiaoQuanPipeline
+from application.common.messageQueue import MQ
+from application.common.proxies import tunnel_proxies
+from application.common.log import AliyunLogger
+from application.common.mysql import MysqlHelper
+from application.common.feishu import Feishu
+from application.functions.read_mysql_config import get_config_from_mysql
+
+
+def get_video_url(article_url):
+    """
+    :param article_url:
+    :return:
+    """
+    # 替换为目标网页的 URL
+    response = requests.get(article_url)
+    html_text = response.text
+    # 正则表达式提取
+    w = re.search(
+        r"mp_video_trans_info.*url:\s*\(\'(.*?)\'\)\.replace", html_text, re.S | re.M
+    ).group(1)
+    url = html.unescape(
+        re.sub(
+            r"\\x\d+", lambda x: bytes.fromhex(x.group().replace("\\x", "")).decode(), w
+        )
+    )
+    return url
 
 
 class OfficialAccountAuthor(object):
@@ -16,16 +57,257 @@ class OfficialAccountAuthor(object):
         self.user_list = user_list
         self.rule_dict = rule_dict
         self.env = env
+        self.mysql = MysqlHelper(mode=self.mode, platform=self)
+        self.aliyun_log = AliyunLogger(self.platform, self.mode)
+        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
 
-    def get_video_list(self):
+    def get_video_list(self, account_name):
         """
         获取视频列表
         :return:
+        todo: 修改一下获取 token 的逻辑，增加 token 的可用性
+        """
+        # 获取 token and cookie
+        fake_id = self.fake_id_manage(account_name)
+        begin = 0
+        while True:
+            token_dict = self.get_token(1)
+            url = "https://mp.weixin.qq.com/cgi-bin/appmsg"
+            headers = {
+                "accept": "*/*",
+                "accept-encoding": "gzip, deflate, br",
+                "accept-language": "zh-CN,zh;q=0.9",
+                "referer": "https://mp.weixin.qq.com/cgi-bin/appmsg?"
+                "t=media/appmsg_edit_v2&action=edit&isNew=1"
+                "&type=77&createType=5&token="
+                + str(token_dict["token"])
+                + "&lang=zh_CN",
+                "sec-ch-ua": '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
+                "sec-ch-ua-mobile": "?0",
+                "sec-ch-ua-platform": '"Windows"',
+                "sec-fetch-dest": "empty",
+                "sec-fetch-mode": "cors",
+                "sec-fetch-site": "same-origin",
+                "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                " (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+                "x-requested-with": "XMLHttpRequest",
+                "cookie": token_dict["cookie"],
+            }
+            params = {
+                "action": "list_ex",
+                "begin": str(begin),
+                "count": "5",
+                "fakeid": fake_id,
+                "type": "9",
+                "query": "",
+                "token": str(token_dict["token"]),
+                "lang": "zh_CN",
+                "f": "json",
+                "ajax": "1",
+            }
+            response = requests.get(url=url, params=params, headers=headers)
+            if response.status_code == 200:
+                result = response.json()
+                if result["base_resp"]["err_msg"] in [
+                    "invalid session",
+                    "freq control",
+                ]:
+                    self.aliyun_log.logging(
+                        code="2000",
+                        message=f"status_code:{response.status_code}, get_fakeid:{response.text}\n",
+                    )
+                    if 20 >= datetime.datetime.now().hour >= 10:
+                        Feishu.bot(
+                            self.mode,
+                            self.platform,
+                            f"{token_dict['title']}\n操作人:{token_dict['operator']}\n更换日期:{token_dict['update_time']} \n过期啦，请扫码更换token\nhttps://mp.weixin.qq.com/",
+                        )
+                    time.sleep(60 * 15)
+                    continue
+                if result["base_resp"]["err_msg"] == "ok" and len(result["list"]) == 0:
+                    print("No more data")
+                if len(result["app_msg_list"]) == 0:
+                    self.aliyun_log.logging(
+                        code="2000",
+                        message="没有更多视频了\n",
+                    )
+                    return
+                else:
+                    begin += 5
+                    app_msg_list = result["app_msg_list"]
+                    for article in app_msg_list:
+                        try:
+                            self.process_video(article, account_name, fake_id)
+                        except Exception as e:
+                            self.aliyun_log.logging(
+                                code="3000",
+                                message="代码报错， 报错信息是{}".format(e),
+                                data=article,
+                                account=account_name,
+                            )
+
+    def process_video(self, article, account_name, fake_id):
+        """
+        处理视频信息
+        :param fake_id: 公众号唯一 id
+        :param account_name: 公众号的名称
+        :param article: 微信公众号的链接
+        :return: None
+        """
+        trace_id = self.platform + str(uuid.uuid1())
+        create_time = article.get("create_time", 0)
+        update_time = article.get("update_time", 0)
+        publish_time_stamp = int(create_time)
+        update_time_stamp = int(update_time)
+        publish_time_str = time.strftime(
+            "%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp)
+        )
+        article_url = article.get("link", "")
+        video_dict = {
+            "video_id": article.get("aid", ""),
+            "video_title": article.get("title", "")
+            .replace(" ", "")
+            .replace('"', "")
+            .replace("'", ""),
+            "publish_time_stamp": publish_time_stamp,
+            "publish_time_str": publish_time_str,
+            "user_name": account_name,
+            "play_cnt": 0,
+            "comment_cnt": 0,
+            "like_cnt": 0,
+            "share_cnt": 0,
+            "user_id": fake_id,
+            "avatar_url": "",
+            "cover_url": article.get("cover", ""),
+            "article_url": article.get("link", ""),
+            "session": f"gongzhonghao-author1-{int(time.time())}",
+        }
+        self.aliyun_log.logging(
+            code="1001", message="扫描到一条视频", data=article, account=account_name
+        )
+        if (
+            int(time.time()) - publish_time_stamp
+            > 3600 * 24 * int(self.rule_dict.get("period", {}).get("max", 1000))
+        ) and (
+            int(time.time()) - update_time_stamp
+            > 3600 * 24 * int(self.rule_dict.get("period", {}).get("max", 1000))
+        ):
+            self.aliyun_log.logging(
+                code="2004",
+                trace_id=trace_id,
+                data=video_dict,
+                message="发布时间超过{}天".format(
+                    int(self.rule_dict.get("period", {}).get("max", 1000))
+                ),
+                account=account_name,
+            )
+            return
+        # 标题敏感词过滤
+        elif (
+            any(
+                str(word) if str(word) in video_dict["video_title"] else False
+                for word in get_config_from_mysql(
+                    log_type=self.mode,
+                    source=self.platform,
+                    env=self.env,
+                    text="filter",
+                )
+            )
+            is True
+        ):
+            self.aliyun_log.logging(
+                code="2003",
+                trace_id=trace_id,
+                data=video_dict,
+                account=account_name,
+                message="标题已中过滤词\n",
+            )
+        # 已下载判断
+        elif (
+            self.repeat_video(
+                video_dict["video_id"],
+            )
+            != 0
+        ):
+            self.aliyun_log.logging(
+                code="2002",
+                trace_id=trace_id,
+                data=video_dict,
+                account=account_name,
+                message="视频已下载",
+            )
+        else:
+            video_dict["out_user_id"] = video_dict["user_id"]
+            video_dict["platform"] = self.platform
+            video_dict["strategy"] = self.mode
+            video_dict["out_video_id"] = video_dict["video_id"]
+            video_dict["width"] = 0
+            video_dict["height"] = 0
+            video_dict["crawler_rule"] = json.dumps(self.rule_dict)
+            video_dict["user_id"] = fake_id  # 站内 UID？爬虫获取不到了（随机发布到原 5 个账号中）
+            video_dict["publish_time"] = video_dict["publish_time_str"]
+            video_dict["video_url"] = get_video_url(article_url)
+            self.mq.send_msg(video_dict)
+            self.aliyun_log.logging(
+                code="1002",
+                trace_id=trace_id,
+                data=video_dict,
+                account=account_name,
+                message="成功发送 MQ 至 ETL",
+            )
+            time.sleep(random.randint(1, 8))
+
+    def repeat_video(self, video_id):
+        """
+        :param video_id: video_id
+        :return:
         """
-        return 0
+        sql = f""" select * from crawler_video where platform = "{self.platform}" and out_video_id="{video_id}" ; """
+        repeat_video = self.mysql.select(sql)
+        return len(repeat_video)
 
-    def fake_id_manage(self):
+    def fake_id_manage(self, account_name):
         """
+        根据公众号的名字去查询 fake_id， 若 fake_id 存在，则返回，若不存在则插入
+        account_name: 公众号的名字，user_dict['link']
         获取fake_id
         :return:
         """
+        select_sql = f"""select name, name_id from accounts where name = "{account_name}" and platform = "{self.platform}" and useful = 1 limit 1"""
+        account_info = self.mysql.select(sql=select_sql)
+        if account_info:
+            name, name_id = account_info[0]
+            return name_id
+        else:
+            user_info = self.get_user_fake_id(account_name)
+            if user_info:
+                fake_id = user_info["user_id"]
+                insert_sql = f"""INSERT INTO accounts (name, name_id, platform, useful) values ("{account_name}", "{fake_id}", "{self.platform}", 1 )"""
+                self.mysql.update(sql=insert_sql)
+                return fake_id
+
+    def get_token(self, token_index):
+        """
+        获取 公众号的 token
+        :param token_index:
+        :return:
+        """
+
+        select_sql = f""" select * from crawler_config where source="{self.platform}" and title LIKE "%公众号_{token_index}%";"""
+        configs = self.mysql.select(select_sql)
+        if len(configs) == 0:
+            Feishu.bot(self.mode, self.platform, f"公众号_{token_index}:未配置token")
+            time.sleep(60)
+            return None
+        token_dict = {
+            "token_id": configs[0]["id"],
+            "title": configs[0]["title"].strip(),
+            "token": dict(eval(configs[0]["config"]))["token"].strip(),
+            "cookie": dict(eval(configs[0]["config"]))["cookie"].strip(),
+            "update_time": time.strftime(
+                "%Y-%m-%d %H:%M:%S",
+                time.localtime(int(configs[0]["update_time"] / 1000)),
+            ),
+            "operator": configs[0]["operator"].strip(),
+        }
+        return token_dict

spider/crawler_online/zhuwanwufusu_2.py

@@ -0,0 +1,394 @@
+import os
+import json
+import random
+import sys
+import time
+import uuid
+import requests
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.backends import default_backend
+
+sys.path.append(os.getcwd())
+
+from application.items import VideoItem
+from application.pipeline import PiaoQuanPipeline
+from application.common.messageQueue import MQ
+from application.common.proxies import tunnel_proxies
+from application.common.log import AliyunLogger
+from application.common.mysql import MysqlHelper
+
+
+class AES(object):
+    """
+    祝万物复苏
+    """
+
+    def __init__(self):
+        self.key = b"50102fa64073ad76"  # 用适当的方式转换或直接定义为字节串
+        self.iv = b"173d023138824bb0"  # 同上
+
+    def aes_encrypt(self, data):
+        """
+        aes 加密
+        :param data:
+        :return:
+        """
+        cipher = Cipher(
+            algorithms.AES(self.key), modes.CBC(self.iv), backend=default_backend()
+        )
+        encryptor = cipher.encryptor()
+        ct = encryptor.update(self._pad(data).encode()) + encryptor.finalize()
+        return ct.hex().upper()
+
+    def aes_decrypt(self, data):
+        """
+        解密
+        :param data:
+        :return:
+        """
+        cipher = Cipher(
+            algorithms.AES(self.key), modes.CBC(self.iv), backend=default_backend()
+        )
+        decryptor = cipher.decryptor()
+        decrypted_data = decryptor.update(bytes.fromhex(data)) + decryptor.finalize()
+        return self._unpad(decrypted_data).decode()
+
+    def _pad(self, s):
+        """
+        填充
+        :param s:
+        :return:
+        """
+        return s + (16 - len(s) % 16) * chr(16 - len(s) % 16)
+
+    def _unpad(self, s):
+        """
+        非填充
+        :param s:
+        :return:
+        """
+        return s[: -ord(s[len(s) - 1:])]
+
+
+class ZhuWanWuFuSuRecommend(object):
+    """
+    祝好事多磨小程序爬虫
+    """
+
+    def __init__(self, platform, mode, rule_dict, user_list, env="prod"):
+        self.limit_flag = False
+        self.platform = platform
+        self.mode = mode
+        self.rule_dict = rule_dict
+        self.user_list = user_list
+        self.env = env
+        self.download_cnt = 0
+        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
+        self.expire_flag = False
+        self.cryptor = AES()
+        self.aliyun_log = AliyunLogger(mode=self.mode, platform=self.platform)
+        self.mysql = MysqlHelper(mode=self.mode, platform=self)
+
+    def get_recommend_list(self):
+        """
+        获取推荐页视频
+        """
+        url = "https://api.lidongze.cn/jeecg-boot/ugc/getVideoListsEn2"
+        headers = {
+            "Host": "api.lidongze.cn",
+            "xweb_xhr": "1",
+            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.4(0x13080410)XWEB/31009",
+            "token": "",
+            "content-type": "application/json",
+            "accept": "*/*",
+            "referer": "https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html",
+            "accept-language": "en-US,en;q=0.9",
+        }
+        page_index = 1
+        total_page = 2
+        while page_index <= total_page:
+            try:
+                query = {
+                    "pageNo": page_index,
+                    "pageSize": 10,
+                    "groupId": "1650323161797439489",  # 推荐流的 ID
+                    "vn": 1,
+                    "gx": 1,
+                    "appid": "wx0afdc2669ed8df2f",
+                    "type": 0,
+                }
+                params = {"v": self.cryptor.aes_encrypt(data=json.dumps(query))}
+                response = requests.get(
+                    url, headers=headers, params=params, proxies=tunnel_proxies()
+                )
+                result = json.loads(self.cryptor.aes_decrypt(response.text))
+                total_page = result["list"]["pages"]
+                page_index = result["list"]["current"] + 1
+                for index, video_obj in enumerate(result["list"]["records"], 1):
+                    try:
+                        self.aliyun_log.logging(
+                            code="1001", message="扫描到一条视频", data=video_obj
+                        )
+                        self.process_video_obj(video_obj)
+                        # print(json.dumps(video_obj, ensure_ascii=False, indent=4))
+                    except Exception as e:
+                        self.aliyun_log.logging(
+                            code="3000",
+                            message="抓取单条视频失败, 该视频位于第{}页第{}条报错原因是{}".format(
+                                page_index, index, e
+                            ),
+                        )
+            except Exception as e:
+                self.aliyun_log.logging(
+                    code="3000", message="抓取第{}页的时候失败, 报错原因是{}".format(page_index, e)
+                )
+            time.sleep(random.randint(5, 10))
+
+    def get_user_videos(self, user_id):
+        """
+        在抓取完推荐页之后，去抓每一个用户的主页视频
+        """
+        url = "https://api.lidongze.cn/jeecg-boot/ugc/getAuthVideoList"
+        headers = {
+            "Host": "api.lidongze.cn",
+            "xweb_xhr": "1",
+            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.4(0x13080410)XWEB/31009",
+            "token": "",
+            "content-type": "application/json",
+            "accept": "*/*",
+            "referer": "https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html",
+            "accept-language": "en-US,en;q=0.9",
+        }
+        page_index = 1
+        total_page = 1
+        while page_index <= total_page:
+            query = {"pageNo": page_index, "pageSize": 10, "authid": user_id}
+            params = {"v": self.cryptor.aes_encrypt(data=json.dumps(query))}
+            response = requests.request(
+                "GET", url, headers=headers, params=params, proxies=tunnel_proxies()
+            )
+            result = json.loads(self.cryptor.aes_decrypt(response.text))
+            total_page = result["list"]["pages"]
+            page_index = result["list"]["current"] + 1
+            for index, video_temp in enumerate(result["list"]["records"]):
+                video_id = video_temp["id"]
+                detail_query = {"videoId": video_id}
+                detail_params = {
+                    "v": self.cryptor.aes_encrypt(data=json.dumps(detail_query))
+                }
+                url = "https://api.lidongze.cn/jeecg-boot/ugc/getVideosDataEn"
+                headers = {
+                    "Host": "api.lidongze.cn",
+                    "xweb_xhr": "1",
+                    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.4(0x13080410)XWEB/31009",
+                    "token": "",
+                    "content-type": "application/json",
+                    "accept": "*/*",
+                    "referer": "https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html",
+                    "accept-language": "en-US,en;q=0.9",
+                }
+                detail_response = requests.request(
+                    "GET",
+                    url,
+                    headers=headers,
+                    params=detail_params,
+                    proxies=tunnel_proxies(),
+                )
+                detail_video = json.loads(
+                    self.cryptor.aes_decrypt(detail_response.text)
+                )
+                if detail_video["success"]:
+                    try:
+                        self.aliyun_log.logging(
+                            code="1001", message="扫描到一条视频", data=detail_video["data"]
+                        )
+                        self.process_video_obj(detail_video["data"])
+                    except Exception as e:
+                        self.aliyun_log.logging(
+                            code="3000",
+                            message="抓取单条视频失败, 该视频位于第{}条报错原因是{}".format(index, e),
+                        )
+
+    def process_video_obj(self, video_obj):
+        """
+        处理视频
+        :param video_obj:
+        """
+        time.sleep(random.randint(3, 8))
+        trace_id = self.platform + str(uuid.uuid1())
+        if video_obj.get("playnum"):
+            play_cnt = (
+                int(video_obj["playnum"].replace("万+", "0000"))
+                if "万+" in video_obj["playnum"]
+                else int(video_obj["playnum"])
+            )
+        else:
+            play_cnt = 0
+        our_user = random.choice(self.user_list)
+        item = VideoItem()
+        item.add_video_info("video_id", video_obj["id"])
+        item.add_video_info("video_title", video_obj["vname"])
+        item.add_video_info("play_cnt", play_cnt)
+        item.add_video_info("publish_time_stamp", int(time.time()))
+        item.add_video_info("out_user_id", video_obj["authid"])
+        item.add_video_info("cover_url", video_obj["shareimg"])
+        item.add_video_info("like_cnt", int(video_obj["likenum"]))
+        item.add_video_info("video_url", video_obj["videoaddr"])
+        item.add_video_info("out_video_id", video_obj["id"])
+        item.add_video_info("platform", self.platform)
+        item.add_video_info("strategy", self.mode)
+        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
+        item.add_video_info("user_id", our_user["uid"])
+        item.add_video_info("user_name", our_user["nick_name"])
+        # 把扫描到的账号存到 accounts 表中
+        self.manage_auth_id(
+            out_user_id=video_obj["authid"], out_user_name=video_obj["authname"]
+        )
+        mq_obj = item.produce_item()
+        pipeline = PiaoQuanPipeline(
+            platform=self.platform,
+            mode=self.mode,
+            rule_dict=self.rule_dict,
+            env=self.env,
+            item=mq_obj,
+            trace_id=trace_id,
+        )
+        if pipeline.process_item():
+            self.download_cnt += 1
+            self.mq.send_msg(mq_obj)
+            self.aliyun_log.logging(code="1002", message="成功发送至 ETL", data=mq_obj)
+            if self.download_cnt >= int(
+                    self.rule_dict.get("videos_cnt", {}).get("min", 200)
+            ):
+                self.limit_flag = True
+
+    def manage_auth_id(self, out_user_id, out_user_name):
+        """
+        out_user_id: 外站视频的用户 id
+        out_user_name: 外站视频用户名字
+        逻辑： 对新扫描到的视频的用户 id 进行判断，若用户 id 不存在，则把视频 id 存到表中,
+              如果用户 id 存在，则判断用户是否修改名字，若名字修改则更新名字
+        """
+        select_user_sql = f"""select name, name_id from accounts where name_id = "{out_user_id}" and platform = "{self.platform}" and useful = 1 limit 1"""
+        out_user_info = self.mysql.select(sql=select_user_sql)
+        if out_user_info:
+            name, name_id = out_user_info[0]
+            if name == out_user_name:
+                return
+            else:
+                update_sql = f"""update accounts set name = "{out_user_name}" where name_id = "{out_user_id}";"""
+                self.mysql.update(sql=update_sql)
+        else:
+            insert_sql = f"""INSERT INTO accounts (name, name_id, platform, useful) values ("{out_user_name}", "{out_user_id}", "{self.platform}", 1 )"""
+            self.mysql.update(sql=insert_sql)
+
+    def get_user_list(self):
+        """
+        获取用户列表
+        :return:
+        """
+        select_user_sql = f"""select name_id from accounts where platform = "{self.platform}" and useful = 1"""
+        out_user_info = self.mysql.select(
+            sql=select_user_sql,
+        )
+        if out_user_info:
+            result = []
+            for i in out_user_info:
+                result.append(i[0])
+            return result
+        else:
+            return []
+
+    def get_detail_video_list(self):
+        """
+        获取推荐列视频
+        :return:
+        """
+        url = "https://api.lidongze.cn/jeecg-boot/ugc/getDetailVideoListsEn2"
+        headers = {
+            "Host": "api.lidongze.cn",
+            "xweb_xhr": "1",
+            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.5(0x13080510)XWEB/1100",
+            "token": "",
+            "referer": "https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html",
+            "accept-language": "en-US,en;q=0.9",
+        }
+        page_index = 1
+        total_page = 2
+        while page_index <= total_page:
+            try:
+                if self.limit_flag:
+                    self.aliyun_log.logging(code="2000", message="本轮已经抓取足够数量的视频")
+                    return
+                else:
+                    query = {
+                        "groupId": "1650323161797439489",
+                        "pageNo": page_index,
+                        "pageSize": 10,
+                        "appid": "wx0afdc2669ed8df2f",
+                        "type": 3,
+                        "hxid": "1556555457243828666",
+                    }
+                    params = {"v": self.cryptor.aes_encrypt(data=json.dumps(query))}
+                    response = requests.request(
+                        "GET", url, headers=headers, params=params
+                    )
+                    result = json.loads(self.cryptor.aes_decrypt(response.text))
+                    total_page = result["list"]["pages"]
+                    page_index = result["list"]["current"] + 1
+                    for index, video_obj in enumerate(result["list"]["records"], 1):
+                        try:
+                            self.aliyun_log.logging(
+                                code="1001", message="扫描到一条视频", data=video_obj
+                            )
+                            self.process_video_obj(video_obj)
+                        except Exception as e:
+                            self.aliyun_log.logging(
+                                code="3000",
+                                message="抓取单条视频失败, 该视频位于第{}页第{}条报错原因是{}".format(
+                                    page_index, index, e
+                                ),
+                            )
+            except Exception as e:
+                self.aliyun_log.logging(
+                    code="3000", message="抓取第{}页的时候失败, 报错原因是{}".format(page_index, e)
+                )
+            time.sleep(random.randint(5, 10))
+
+    def run(self):
+        """
+        先抓取推荐列表的视频， 等待 2 分钟后抓取 detail 页面，等待 5 分钟后，抓取账号视频
+        """
+        self.get_recommend_list()
+        if self.limit_flag:
+            return
+        time.sleep(2 * 60)
+        self.get_detail_video_list()
+        if self.limit_flag:
+            return
+        time.sleep(5 * 60)
+        self.mode = "author"
+        user_list = self.get_user_list()
+        if user_list:
+            for index, user_id in enumerate(user_list):
+                try:
+                    if self.limit_flag:
+                        self.aliyun_log.logging(code="2000", message="本轮已经抓取足够数量的视频")
+                        return
+                    self.get_user_videos(user_id=user_id)
+                except Exception as e:
+                    self.aliyun_log.logging(
+                        code="3000",
+                        message="抓取账号视频出现异常，账号 id 是{}, 报错原因是{}".format(user_id, e),
+                    )
+
+
+# if __name__ == '__main__':
+#     Z = ZhuWanWuFuSuRecommend(
+#         platform="zhuwanwufusu2",
+#         mode="recommend",
+#         rule_dict={},
+#         user_list=[{'uid': "123456", 'nick_name': "luojunhui"}],
+#
+#     )
+#     Z.get_recommend_list()

spider/spider_map.py

@@ -8,8 +8,8 @@ from spider.crawler_online import *
 
 spider_map = {
     # 祝万物复苏
-    "zhuwanwufusu": {
-        "recommend": "ZhuWanWuFuSuRecommend"
+    "zhuwanwufusunew": {
+        "recommend": ZhuWanWuFuSuRecommend
     },
     # 祝好事多磨
     "zhuhaoshiduomo": {