新增吉祥祝福为你传递好运旗下小程序

jixiangzhufuweinichuandihaoyun/jixiangzhufuweinichuandihaoyun_main/run_jxzfwncdhy_recommend.py

@@ -12,6 +12,8 @@ from common.common import Common
 from common.public import task_fun_mq, get_consumer, ack_message
 from common.scheduling_db import MysqlHelper
 from jixiangzhufuweinichuandihaoyun.jixiangzhufuweinichuandihaoyun_recommend.jixiangzhufuweinichuandihaoyun_recommend import Jxzfwncdhyspcheduling
+from jixiangzhufuweinichuandihaoyun.jixiangzhufuweinichuandihaoyun_recommend.zhufudaigeinimenxiqiyangyang_recommend import Zfdgnmxqyyspcheduling
+from jixiangzhufuweinichuandihaoyun.jixiangzhufuweinichuandihaoyun_recommend.zhufusonggeiniwendingjixiang_recommend import Zfsgnwdjxspcheduling
 def main(log_type, crawler, topic_name, group_id, env):
     consumer = get_consumer(topic_name, group_id)
     # 长轮询表示如果Topic没有消息，则客户端请求会在服务端挂起3秒，3秒内如果有消息可以消费则立即返回响应。
@@ -106,8 +108,26 @@ def main(log_type, crawler, topic_name, group_id, env):
                     env=env,
                     our_uid=our_uid
                 )
-
                 HH.get_videoList()
+
+                YY = Zfdgnmxqyyspcheduling(
+                    log_type=log_type,
+                    crawler=crawler,
+                    rule_dict=rule_dict,
+                    env=env,
+                    our_uid=our_uid
+                )
+
+                YY.get_videoList()
+                XX = Zfsgnwdjxspcheduling(
+                    log_type=log_type,
+                    crawler=crawler,
+                    rule_dict=rule_dict,
+                    env=env,
+                    our_uid=our_uid
+                )
+
+                XX.get_videoList()
                 Common.logger(log_type, crawler).info("抓取一轮结束\n")
                 Common.logging(log_type, crawler, env, "抓取一轮结束\n")
 

jixiangzhufuweinichuandihaoyun/jixiangzhufuweinichuandihaoyun_recommend/jixiangzhufuweinichuandihaoyun_recommend.py

@@ -1,115 +1,104 @@
-# -*- coding: utf-8 -*-
-# @Author: zhangyong
-# @Time: 2023/12/04
 import json
 import os
 import random
 import sys
 import time
-import requests
-from hashlib import md5
-from datetime import datetime
+import uuid
 
-from common import get_redirect_url
-from common.mq import MQ
+import requests
 
 sys.path.append(os.getcwd())
-from common.common import Common
-from common.scheduling_db import MysqlHelper
-from common.public import get_config_from_mysql, download_rule
-
-proxies = {"http": None, "https": None}
-
-
-def clean_title(strings):
-    return (
-        strings.strip()
-        .replace("\n", "")
-        .replace("/", "")
-        .replace("\r", "")
-        .replace("#", "")
-        .replace(".", "。")
-        .replace("\\", "")
-        .replace("&NBSP", "")
-        .replace(":", "")
-        .replace("*", "")
-        .replace("？", "")
-        .replace("?", "")
-        .replace('"', "")
-        .replace("<", "")
-        .replace(">", "")
-        .replace("|", "")
-        .replace(" ", "")
-        .replace('"', "")
-        .replace("'", "")
-    )
+from common.video_item import VideoItem
+from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies, get_redirect_url
+from common.mq import MQ
 
 
-class Jxzfwncdhyspcheduling:
-    def __init__(self, log_type, crawler, rule_dict, env, our_uid):
-        self.platform = "吉祥祝福为你传递好运"
-        self.log_type = log_type
-        self.crawler = crawler
+class Jxzfwncdhyspcheduling(object):
+    def __init__(self, platform, mode, rule_dict, user_list, env):
+        self.platform = platform
+        self.mode = mode
         self.rule_dict = rule_dict
+        self.user_list = user_list
         self.env = env
-        self.our_uid = our_uid
+        self.download_cnt = 0
         self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
-        self.download_count = 0
-
-    def repeat_video(self, video_id):
-        sql = f""" select * from crawler_video where platform in ("{self.crawler}","{self.platform}") and out_video_id="{video_id}"; """
-        repeat_video = MysqlHelper.get_values(
-            self.log_type, self.crawler, sql, self.env
-        )
-        return len(repeat_video)
+        self.limit_flag = False
 
-    # 获取视频id_list
     def get_videoList(self):
-        for i in range(1, 10):
-            time.sleep(random.randint(5, 10))
-            url = "https://api.xiahong.top/index.php?s=mobile/Home/getHomeList&page={}&appid=wx7457ce7bf3cdbdbf&version=1.9.1&env_version=release&scene=1008".format(i)
 
-            headers = {
-                'Host': 'api.xiahong.top',
-                'Content-Type': 'application/json',
-                'Accept-Language': 'zh-cn',
-                'Accept': '*/*',
-                'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
-                'Referer': 'https://servicewechat.com/wxd074e82284aa59cd/2/page-frame.html',
-                'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI0MzU3NTYsIm5iZiI6MTcwMjQzNTc1NiwiZXhwIjoxNzAyNDQyOTU2LCJkYXRhIjp7InVzZXJfaWQiOiIyNDQ3NzI0OTcifX0.zZVfWBw1r8yR7GLO3Ff1NkJVwtmQ1v9LUzynOV4GlbI',
-                'ik': 'b326b5062b2f0e69046810717534cb09'
-            }
-            response = requests.post(url, headers=headers)
-            if "data" not in response.text or response.status_code != 200:
-                Common.logger(self.log_type, self.crawler).info(
-                    f"get_videoList:{response.text}\n"
-                )
-                Common.logging(
-                    self.log_type,
-                    self.crawler,
-                    self.env,
-                    f"get_videoList:{response.text}\n",
-                )
-                return
-            elif len(response.json()["data"]["list"]) == 0:
-                Common.logger(self.log_type, self.crawler).info(f"没有更多数据啦~\n")
-                Common.logging(self.log_type, self.crawler, self.env, f"没有更多数据啦~\n")
-                return
-            else:
-                data_list = response.json()["data"]["list"]
-                for video_obj in data_list:
-                    try:
-                        self.process_video_obj(video_obj)
-                    except Exception as e:
-                        Common.logger(self.log_type, self.crawler).error(f"抓取单条视频异常:{e}\n")
-                        Common.logging(
-                            self.log_type, self.crawler, self.env, f"抓取单条视频异常:{e}\n"
+        headers = {
+            'Host': 'api.xiahong.top',
+            'Content-Type': 'application/json',
+            'Accept-Language': 'zh-cn',
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
+            'Referer': 'https://servicewechat.com/wxd074e82284aa59cd/2/page-frame.html',
+            'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI0NTMwNjIsIm5iZiI6MTcwMjQ1MzA2MiwiZXhwIjoxNzAyNDYwMjYyLCJkYXRhIjp7InVzZXJfaWQiOiIyNDUwOTAxNDQifX0.4LM2vKA5gYmGtjcdGHJ19neHfgKQes6X68Uo-CWLsQM',
+            'ik': 'b326b5062b2f0e69046810717534cb09'
+        }
+        page = 1
+        while True:
+            time.sleep(random.randint(1, 10))
+            try:
+                if self.limit_flag:
+                    message = "本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt)
+                    AliyunLogger.logging(
+                        code="2000",
+                        platform=self.platform,
+                        env=self.env,
+                        mode=self.mode,
+                        message=message
+                    )
+                    return
+                else:
+                    url = "https://api.xiahong.top/index.php?s=mobile/Home/getHomeList&page={}&appid=wx7457ce7bf3cdbdbf&version=1.9.1&env_version=release&scene=1008".format(
+                        page)
+                    response = requests.post(url, headers=headers, proxies=tunnel_proxies())
+                    video_list = response.json()['data']['list']
+                    page += 1
+                    if video_list:
+                        for index, video_obj in enumerate(video_list, 1):
+                            try:
+                                AliyunLogger.logging(
+                                    code="1001",
+                                    platform=self.platform,
+                                    mode=self.mode,
+                                    env=self.env,
+                                    message="扫描到一条视频",
+                                    data=video_obj
+                                )
+                                self.process_video_obj(video_obj)
+                            except Exception as e:
+                                AliyunLogger.logging(
+                                    code="3000",
+                                    platform=self.platform,
+                                    mode=self.mode,
+                                    env=self.env,
+                                    message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e)
+                                )
+                    else:
+                        AliyunLogger.logging(
+                            code="2000",
+                            platform=self.platform,
+                            mode=self.mode,
+                            env=self.env,
+                            message="视频已经抓完",
                         )
+                        return
+            except Exception as e:
+                AliyunLogger.logging(
+                    code="3000",
+                    platform=self.platform,
+                    mode=self.mode,
+                    env=self.env,
+                    message="抓取推荐页的时候出现错误， 报错信息是{}".format(e)
+                )
 
-    def get_video_list(self, video_id):
+    def process_video_obj(self, video_obj):
+        trace_id = self.platform + str(uuid.uuid1())
+        video_id = video_obj['id']
         url = "https://api.xiahong.top/index.php?s=mobile/Video/getVideoInfo&vid={}&appid=wx7457ce7bf3cdbdbf&version=1.9.1&scene=1089".format(
             video_id)
-
         headers = {
             'Host': 'api.xiahong.top',
             'Content-Type': 'application/json',
@@ -117,128 +106,59 @@ class Jxzfwncdhyspcheduling:
             'Accept': '*/*',
             'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
             'Referer': 'https://servicewechat.com/wxd074e82284aa59cd/2/page-frame.html',
-            'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI0MzU3NTYsIm5iZiI6MTcwMjQzNTc1NiwiZXhwIjoxNzAyNDQyOTU2LCJkYXRhIjp7InVzZXJfaWQiOiIyNDQ3NzI0OTcifX0.zZVfWBw1r8yR7GLO3Ff1NkJVwtmQ1v9LUzynOV4GlbI',
+            'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI0NTMwNjIsIm5iZiI6MTcwMjQ1MzA2MiwiZXhwIjoxNzAyNDYwMjYyLCJkYXRhIjp7InVzZXJfaWQiOiIyNDUwOTAxNDQifX0.4LM2vKA5gYmGtjcdGHJ19neHfgKQes6X68Uo-CWLsQM',
             'ik': 'b326b5062b2f0e69046810717534cb09'
         }
-        response = requests.post(url, headers=headers)
-        if "data" not in response.text or response.status_code != 200:
-            Common.logger(self.log_type, self.crawler).info(
-                f"get_videoList:{response.text}\n"
-            )
-            Common.logging(
-                self.log_type,
-                self.crawler,
-                self.env,
-                f"get_videoList:{response.text}\n",
-            )
-            return
-        elif len(response.json()["data"]["video_info"]) == 0:
-            Common.logger(self.log_type, self.crawler).info(f"详情页数据为空~\n")
-            Common.logging(self.log_type, self.crawler, self.env, f"详情页数据为空~\n")
-            return
-        else:
-            data_list = response.json()["data"]["video_info"]
-            return data_list
-
-
-    def process_video_obj(self, video_obj):
-        video_id = video_obj.get("id", 0)
-        get_video_list = self.get_video_list(video_id)
-        video_title = clean_title(get_video_list.get("title", "no title"))
-        video_time = video_obj.get("v_time", 0)
-        publish_time_stamp = int(time.time())
-        publish_time_str = time.strftime(
-            "%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp)
-        )
-        user_name = ""
-        video_dict = {
-            "video_title": video_title,
-            "video_id": video_id,
-            "duration": video_time,
-            "play_cnt": video_obj.get("visited", 0),
-            "like_cnt": 0,
-            "comment_cnt": 0,
-            "share_cnt": video_obj.get("shared", 0),
-            "user_name": user_name,
-            "publish_time_stamp": publish_time_stamp,
-            "publish_time_str": publish_time_str,
-            "video_width": 0,
-            "video_height": 0,
-            "profile_id": 0,
-            "profile_mid": 0,
-            # "cover_url": "",
-            "session": f"zhufuzanmenzhonglaonianzhishipin-{int(time.time())}",
-        }
-        for k, v in video_dict.items():
-            Common.logger(self.log_type, self.crawler).info(f"{k}:{v}")
-        Common.logging(
-            self.log_type, self.crawler, self.env, f"{video_dict}"
+        response = requests.post(url, headers=headers, proxies=tunnel_proxies())
+        video_info = response.json()["data"]["video_info"]
+        video_url = video_info['video_url']
+        video_url = get_redirect_url(video_url)
+
+        our_user = random.choice(self.user_list)
+        item = VideoItem()
+        item.add_video_info("user_id", our_user["uid"])
+        item.add_video_info("user_name", our_user["nick_name"])
+        item.add_video_info("video_id", video_id)
+        item.add_video_info("video_title", video_info["title"])
+        item.add_video_info("video_url", video_url)
+        item.add_video_info("cover_url", video_obj["images"])
+        item.add_video_info("avatar_url", video_obj["images"])
+        item.add_video_info("out_video_id", video_id)
+        item.add_video_info("platform", self.platform)
+        item.add_video_info("strategy", self.mode)
+        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
+        mq_obj = item.produce_item()
+        pipeline = PiaoQuanPipeline(
+            platform=self.platform,
+            mode=self.mode,
+            rule_dict=self.rule_dict,
+            env=self.env,
+            item=mq_obj,
+            trace_id=trace_id,
         )
-        # 过滤无效视频
-        if video_title == "" or video_dict["video_id"] == "":
-            Common.logger(self.log_type, self.crawler).info("无效视频\n")
-            Common.logging(self.log_type, self.crawler, self.env, "无效视频\n")
-            # 抓取基础规则过滤
-        elif (
-                download_rule(
-                    log_type=self.log_type,
-                    crawler=self.crawler,
-                    video_dict=video_dict,
-                    rule_dict=self.rule_dict,
-                )
-                is False
-        ):
-            Common.logger(self.log_type, self.crawler).info("不满足抓取规则\n")
-            Common.logging(
-                self.log_type, self.crawler, self.env, "不满足抓取规则\n"
+        if pipeline.process_item():
+            self.download_cnt += 1
+            self.mq.send_msg(mq_obj)
+            AliyunLogger.logging(
+                code="1002",
+                platform=self.platform,
+                mode=self.mode,
+                env=self.env,
+                message="成功发送至 ETL",
+                data=mq_obj,
+                trace_id=trace_id
             )
-        elif (
-                any(
-                    str(word)
-                    if str(word) in video_dict["video_title"]
-                    else False
-                    for word in get_config_from_mysql(
-                        log_type=self.log_type,
-                        source=self.crawler,
-                        env=self.env,
-                        text="filter",
-                        action="",
-                    )
-                )
-                is True
-        ):
-            Common.logger(self.log_type, self.crawler).info("已中过滤词\n")
-            Common.logging(self.log_type, self.crawler, self.env, "已中过滤词\n")
-        elif self.repeat_video(video_dict["video_id"]) != 0:
-            Common.logger(self.log_type, self.crawler).info("视频已下载\n")
-            Common.logging(self.log_type, self.crawler, self.env, "视频已下载\n")
-        else:
-            video_url = get_video_list['video_url']
-            video_url = get_redirect_url(video_url)
-
-            video_dict["out_user_id"] = video_dict["profile_id"]
-            video_dict["platform"] = self.crawler
-            video_dict["strategy"] = self.log_type
-            video_dict["out_video_id"] = str(video_dict["video_id"])
-            video_dict["width"] = video_dict["video_width"]
-            video_dict["height"] = video_dict["video_height"]
-            video_dict["crawler_rule"] = json.dumps(self.rule_dict)
-            video_dict["user_id"] = self.our_uid
-            video_dict["publish_time"] = video_dict["publish_time_str"]
-            video_dict["video_url"] = video_url
-            video_dict["avatar_url"] = video_obj['images']
-            video_dict["cover_url"] = video_obj['images']
-            self.download_count += 1
-            self.mq.send_msg(video_dict)
-
+            if self.download_cnt >= int(
+                self.rule_dict.get("videos_cnt", {}).get("min", 200)
+            ):
+                self.limit_flag = True
 
 if __name__ == "__main__":
     ZL = Jxzfwncdhyspcheduling(
-        log_type="recommend",
-        crawler="Jxzfwncdhy",
+        platform="Jxzfwncdhy",
+        mode="recommend",
+        env="dev",
         rule_dict={},
-        our_uid="zhangyong",
-        env="dev"
+        user_list=[{'nick_name': "Ivring", 'uid': "1997"}, {'nick_name': "paul", 'uid': "1998"}]
     )
-    ZL.get_videoList()
-
+    ZL.get_videoList()

jixiangzhufuweinichuandihaoyun/jixiangzhufuweinichuandihaoyun_recommend/zhufudaigeinimenxiqiyangyang_recommend.py

@@ -0,0 +1,164 @@
+import json
+import os
+import random
+import sys
+import time
+import uuid
+
+import requests
+
+sys.path.append(os.getcwd())
+from common.video_item import VideoItem
+from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies, get_redirect_url
+from common.mq import MQ
+
+
+class Zfdgnmxqyyspcheduling(object):
+    def __init__(self, platform, mode, rule_dict, user_list, env):
+        self.platform = platform
+        self.mode = mode
+        self.rule_dict = rule_dict
+        self.user_list = user_list
+        self.env = env
+        self.download_cnt = 0
+        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
+        self.limit_flag = False
+
+    def get_videoList(self):
+
+        headers = {
+            'Host': 'api.xiahong.top',
+            'Content-Type': 'application/json',
+            'Accept-Language': 'zh-cn',
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
+            'Referer': 'https://servicewechat.com/wxd0e272b032f12c4b/2/page-frame.html',
+            'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI0NTMwNjIsIm5iZiI6MTcwMjQ1MzA2MiwiZXhwIjoxNzAyNDYwMjYyLCJkYXRhIjp7InVzZXJfaWQiOiIyNDUwOTAxNDQifX0.4LM2vKA5gYmGtjcdGHJ19neHfgKQes6X68Uo-CWLsQM',
+            'ik': 'b326b5062b2f0e69046810717534cb09'
+        }
+        page = 1
+        while True:
+            time.sleep(random.randint(1, 10))
+            try:
+                if self.limit_flag:
+                    message = "本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt)
+                    AliyunLogger.logging(
+                        code="2000",
+                        platform=self.platform,
+                        env=self.env,
+                        mode=self.mode,
+                        message=message
+                    )
+                    return
+                else:
+                    url = "https://api.xiahong.top/index.php?s=mobile/Home/getHomeList&page={}&appid=wxd0e272b032f12c4b&version=1.9.1&env_version=release&scene=1053".format(
+                        page)
+                    response = requests.post(url, headers=headers, proxies=tunnel_proxies())
+                    video_list = response.json()['data']['list']
+                    page += 1
+                    if video_list:
+                        for index, video_obj in enumerate(video_list, 1):
+                            try:
+                                AliyunLogger.logging(
+                                    code="1001",
+                                    platform=self.platform,
+                                    mode=self.mode,
+                                    env=self.env,
+                                    message="扫描到一条视频",
+                                    data=video_obj
+                                )
+                                self.process_video_obj(video_obj)
+                            except Exception as e:
+                                AliyunLogger.logging(
+                                    code="3000",
+                                    platform=self.platform,
+                                    mode=self.mode,
+                                    env=self.env,
+                                    message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e)
+                                )
+                    else:
+                        AliyunLogger.logging(
+                            code="2000",
+                            platform=self.platform,
+                            mode=self.mode,
+                            env=self.env,
+                            message="视频已经抓完",
+                        )
+                        return
+            except Exception as e:
+                AliyunLogger.logging(
+                    code="3000",
+                    platform=self.platform,
+                    mode=self.mode,
+                    env=self.env,
+                    message="抓取推荐页的时候出现错误， 报错信息是{}".format(e)
+                )
+
+    def process_video_obj(self, video_obj):
+        trace_id = self.platform + str(uuid.uuid1())
+        video_id = video_obj['id']
+        url = "https://api.xiahong.top/index.php?s=mobile/Video/getVideoInfo&vid={}&appid=wxd0e272b032f12c4b&version=1.9.1&scene=1053".format(
+            video_id)
+        headers = {
+            'Host': 'api.xiahong.top',
+            'Content-Type': 'application/json',
+            'Accept-Language': 'zh-cn',
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
+            'Referer': 'https://servicewechat.com/wxd0e272b032f12c4b/2/page-frame.html',
+            'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI0NTMwNjIsIm5iZiI6MTcwMjQ1MzA2MiwiZXhwIjoxNzAyNDYwMjYyLCJkYXRhIjp7InVzZXJfaWQiOiIyNDUwOTAxNDQifX0.4LM2vKA5gYmGtjcdGHJ19neHfgKQes6X68Uo-CWLsQM',
+            'ik': 'b326b5062b2f0e69046810717534cb09'
+        }
+        response = requests.post(url, headers=headers, proxies=tunnel_proxies())
+        video_info = response.json()["data"]["video_info"]
+        video_url = video_info['video_url']
+        video_url = get_redirect_url(video_url)
+
+        our_user = random.choice(self.user_list)
+        item = VideoItem()
+        item.add_video_info("user_id", our_user["uid"])
+        item.add_video_info("user_name", our_user["nick_name"])
+        item.add_video_info("video_id", video_id)
+        item.add_video_info("video_title", video_info["title"])
+        item.add_video_info("video_url", video_url)
+        item.add_video_info("cover_url", video_obj["images"])
+        item.add_video_info("avatar_url", video_obj["images"])
+        item.add_video_info("out_video_id", video_id)
+        item.add_video_info("platform", self.platform)
+        item.add_video_info("strategy", self.mode)
+        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
+        mq_obj = item.produce_item()
+        pipeline = PiaoQuanPipeline(
+            platform=self.platform,
+            mode=self.mode,
+            rule_dict=self.rule_dict,
+            env=self.env,
+            item=mq_obj,
+            trace_id=trace_id,
+        )
+        if pipeline.process_item():
+            self.download_cnt += 1
+            self.mq.send_msg(mq_obj)
+            AliyunLogger.logging(
+                code="1002",
+                platform=self.platform,
+                mode=self.mode,
+                env=self.env,
+                message="成功发送至 ETL",
+                data=mq_obj,
+                trace_id=trace_id
+            )
+            if self.download_cnt >= int(
+                self.rule_dict.get("videos_cnt", {}).get("min", 200)
+            ):
+                self.limit_flag = True
+
+if __name__ == "__main__":
+    ZL = Zfdgnmxqyyspcheduling(
+        platform="Jxzfwncdhy",
+        mode="recommend",
+        env="dev",
+        rule_dict={},
+        user_list=[{'nick_name': "Ivring", 'uid': "1997"}, {'nick_name': "paul", 'uid': "1998"}]
+    )
+    ZL.get_videoList()

jixiangzhufuweinichuandihaoyun/jixiangzhufuweinichuandihaoyun_recommend/zhufusonggeiniwendingjixiang_recommend.py

@@ -0,0 +1,165 @@
+import json
+import os
+import random
+import sys
+import time
+import uuid
+
+import requests
+
+sys.path.append(os.getcwd())
+from common.video_item import VideoItem
+from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies, get_redirect_url
+from common.mq import MQ
+
+
+class Zfsgnwdjxspcheduling(object):
+    def __init__(self, platform, mode, rule_dict, user_list, env):
+        self.platform = platform
+        self.mode = mode
+        self.rule_dict = rule_dict
+        self.user_list = user_list
+        self.env = env
+        self.download_cnt = 0
+        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
+        self.limit_flag = False
+
+    def get_videoList(self):
+
+        headers = {
+            'Host': 'api.xiahong.top',
+            'Content-Type': 'application/json',
+            'Accept-Language': 'zh-cn',
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
+            'Referer': 'https://servicewechat.com/wx6634bd205a275f0d/2/page-frame.html',
+            'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI0NTMwNjIsIm5iZiI6MTcwMjQ1MzA2MiwiZXhwIjoxNzAyNDYwMjYyLCJkYXRhIjp7InVzZXJfaWQiOiIyNDUwOTAxNDQifX0.4LM2vKA5gYmGtjcdGHJ19neHfgKQes6X68Uo-CWLsQM',
+            'ik': 'b326b5062b2f0e69046810717534cb09'
+        }
+        page = 1
+        while True:
+            time.sleep(random.randint(1, 10))
+            try:
+                if self.limit_flag:
+                    message = "本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt)
+                    AliyunLogger.logging(
+                        code="2000",
+                        platform=self.platform,
+                        env=self.env,
+                        mode=self.mode,
+                        message=message
+                    )
+                    return
+                else:
+                    url = "https://api.xiahong.top/index.php?s=mobile/Home/getHomeList&page={}&appid=wx6634bd205a275f0d&version=1.9.1&env_version=release&scene=1053".format(
+                        page)
+
+                    response = requests.post(url, headers=headers, proxies=tunnel_proxies())
+                    video_list = response.json()['data']['list']
+                    page += 1
+                    if video_list:
+                        for index, video_obj in enumerate(video_list, 1):
+                            try:
+                                AliyunLogger.logging(
+                                    code="1001",
+                                    platform=self.platform,
+                                    mode=self.mode,
+                                    env=self.env,
+                                    message="扫描到一条视频",
+                                    data=video_obj
+                                )
+                                self.process_video_obj(video_obj)
+                            except Exception as e:
+                                AliyunLogger.logging(
+                                    code="3000",
+                                    platform=self.platform,
+                                    mode=self.mode,
+                                    env=self.env,
+                                    message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e)
+                                )
+                    else:
+                        AliyunLogger.logging(
+                            code="2000",
+                            platform=self.platform,
+                            mode=self.mode,
+                            env=self.env,
+                            message="视频已经抓完",
+                        )
+                        return
+            except Exception as e:
+                AliyunLogger.logging(
+                    code="3000",
+                    platform=self.platform,
+                    mode=self.mode,
+                    env=self.env,
+                    message="抓取推荐页的时候出现错误， 报错信息是{}".format(e)
+                )
+
+    def process_video_obj(self, video_obj):
+        trace_id = self.platform + str(uuid.uuid1())
+        video_id = video_obj['id']
+        url = "https://api.xiahong.top/index.php?s=mobile/Video/getVideoInfo&vid={}&appid=wx6634bd205a275f0d&version=1.9.1&scene=1053".format(
+            video_id)
+        headers = {
+            'Host': 'api.xiahong.top',
+            'Content-Type': 'application/json',
+            'Accept-Language': 'zh-cn',
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E217 MicroMessenger/6.8.0(0x16080000) NetType/WIFI Language/en Branch/Br_trunk MiniProgramEnv/Mac',
+            'Referer': 'https://servicewechat.com/wx6634bd205a275f0d/2/page-frame.html',
+            'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MDI0NTMwNjIsIm5iZiI6MTcwMjQ1MzA2MiwiZXhwIjoxNzAyNDYwMjYyLCJkYXRhIjp7InVzZXJfaWQiOiIyNDUwOTAxNDQifX0.4LM2vKA5gYmGtjcdGHJ19neHfgKQes6X68Uo-CWLsQM',
+            'ik': 'b326b5062b2f0e69046810717534cb09'
+        }
+        response = requests.post(url, headers=headers, proxies=tunnel_proxies())
+        video_info = response.json()["data"]["video_info"]
+        video_url = video_info['video_url']
+        video_url = get_redirect_url(video_url)
+
+        our_user = random.choice(self.user_list)
+        item = VideoItem()
+        item.add_video_info("user_id", our_user["uid"])
+        item.add_video_info("user_name", our_user["nick_name"])
+        item.add_video_info("video_id", video_id)
+        item.add_video_info("video_title", video_info["title"])
+        item.add_video_info("video_url", video_url)
+        item.add_video_info("cover_url", video_obj["images"])
+        item.add_video_info("avatar_url", video_obj["images"])
+        item.add_video_info("out_video_id", video_id)
+        item.add_video_info("platform", self.platform)
+        item.add_video_info("strategy", self.mode)
+        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
+        mq_obj = item.produce_item()
+        pipeline = PiaoQuanPipeline(
+            platform=self.platform,
+            mode=self.mode,
+            rule_dict=self.rule_dict,
+            env=self.env,
+            item=mq_obj,
+            trace_id=trace_id,
+        )
+        if pipeline.process_item():
+            self.download_cnt += 1
+            self.mq.send_msg(mq_obj)
+            AliyunLogger.logging(
+                code="1002",
+                platform=self.platform,
+                mode=self.mode,
+                env=self.env,
+                message="成功发送至 ETL",
+                data=mq_obj,
+                trace_id=trace_id
+            )
+            if self.download_cnt >= int(
+                self.rule_dict.get("videos_cnt", {}).get("min", 200)
+            ):
+                self.limit_flag = True
+
+if __name__ == "__main__":
+    ZL = Zfsgnwdjxspcheduling(
+        platform="Jxzfwncdhy",
+        mode="recommend",
+        env="dev",
+        rule_dict={},
+        user_list=[{'nick_name': "Ivring", 'uid': "1997"}, {'nick_name': "paul", 'uid': "1998"}]
+    )
+    ZL.get_videoList()