节日祝福矩阵-上线

application/config/topic_group_queue.py

@@ -15,7 +15,8 @@ class TopicGroup(object):
             ("jxxf", 'recommend', 'jixiangxingfu'),
             ("xnght", 'recommend', 'xiaoniangaohuati'),
             ('ynfqcz', 'recommend', 'yuannifuqichangzai'),
-            ('zhdsn', 'recommend', 'zuihaodesongni')
+            ('zhdsn', 'recommend', 'zuihaodesongni'),
+            ('jrzf', 'recommend', 'jierizhufu')
         ]
 
     def produce(self):

spider/crawler_online/__init__.py

@@ -9,4 +9,5 @@ from .jiajiezhufuxishiduoduo import XiShiDuoDuoRecommend
 from .jixiangxingfu import JXXFRecommend
 from .xiaoniangaohuati import XNGHTecommend
 from .yuannifuqichangzai import YuanNiFuQiChangZai
-from .zuihaodesongni import ZuiHaoDeSongNi
+from .zuihaodesongni import ZuiHaoDeSongNi
+from .jierizhufu import JieRiZhuFu

spider/crawler_online/jierizhufu.py

@@ -0,0 +1,165 @@
+"""
+@author luojunhui
+节日祝福矩阵
+节日祝福你祥贵
+"""
+import os
+import sys
+import time
+import uuid
+import random
+import datetime
+import requests
+
+sys.path.append(os.getcwd())
+
+from application.items import VideoItem
+from application.pipeline import PiaoQuanPipeline
+from application.common.messageQueue import MQ
+from application.common.proxies import tunnel_proxies
+from application.common.log import AliyunLogger
+
+
+class JieRiZhuFu(object):
+    """
+    节日祝福矩阵——推荐爬虫
+    """
+
+    def __init__(self, platform, mode, rule_dict, user_list, env="prod"):
+        self.platform = platform
+        self.mode = mode
+        self.rule_dict = rule_dict
+        self.user_list = user_list
+        self.env = env
+        self.download_cnt = 0
+        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
+        self.expire_flag = False
+        self.aliyun_log = AliyunLogger(platform=self.platform, mode=self.mode)
+
+    def process_video_obj(self, video_obj):
+        """
+        处理每一个视频内容
+        :return: None
+        """
+        trace_id = self.platform + str(uuid.uuid1())
+        our_user = random.choice(self.user_list)
+        publish_time_stamp = int(time.time())
+        publish_time_str = datetime.datetime.fromtimestamp(publish_time_stamp).strftime(
+            "%Y-%m-%d %H:%M:%S"
+        )
+        item = VideoItem()
+        item.add_video_info("user_id", our_user["uid"])
+        item.add_video_info("user_name", our_user["nick_name"])
+        item.add_video_info("video_id", video_obj["id"])
+        item.add_video_info("video_title", video_obj["title"])
+        item.add_video_info("publish_time_str", publish_time_str)
+        item.add_video_info("publish_time_stamp", int(publish_time_stamp))
+        item.add_video_info("video_url", video_obj["video_url"])
+        item.add_video_info("cover_url", video_obj["video_cover"])
+        item.add_video_info("out_video_id", video_obj["id"])
+        item.add_video_info("play_cnt", video_obj['visited'])
+        item.add_video_info("platform", self.platform)
+        item.add_video_info("share_cnt", video_obj['shared'])
+        item.add_video_info("strategy", self.mode)
+        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
+        mq_obj = item.produce_item()
+        pipeline = PiaoQuanPipeline(
+            platform=self.platform,
+            mode=self.mode,
+            rule_dict=self.rule_dict,
+            env=self.env,
+            item=mq_obj,
+            trace_id=trace_id,
+        )
+        if pipeline.process_item():
+            self.download_cnt += 1
+            # print(json.dumps(mq_obj, ensure_ascii=False, indent=4))
+            self.mq.send_msg(mq_obj)
+            time.sleep(random.randint(1, 5))
+            self.aliyun_log.logging(
+                code="1002",
+                message="成功发送至 ETL",
+                data=mq_obj,
+            )
+            if self.download_cnt >= int(
+                self.rule_dict.get("videos_cnt", {}).get("min", 200)
+            ):
+                self.expire_flag = True
+
+    def get_recommend_list(self, page_index):
+        """
+        获取推荐页面的video_list
+        :param page_index: 页码
+        :return: None
+        """
+        if self.expire_flag:
+            self.aliyun_log.logging(
+                code="2000",
+                message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
+            )
+            return
+        url = "https://api.bestcaiwei.top/index.php"
+        params = {
+            "s": "mobile/Video/getList",
+            "cid": 1,
+            "page": page_index,
+            "api_version": 4,
+            "appid": "wx02bca7643610fee6",
+            "version": "1.9.5",
+            "env_version": "release",
+            "scene": 1089
+        }
+        headers = {
+            'Host': 'api.bestcaiwei.top',
+            'ik': 'b326b5062b2f0e69046810717534cb09',
+            'xweb_xhr': '1',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.6(0x13080610) XWEB/1156',
+            'token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE3MTAzMDgxMDIsIm5iZiI6MTcxMDMwODEwMiwiZXhwIjoxNzEwMzExNzAyLCJkYXRhIjp7InVzZXJfaWQiOjQ0MTEzNTY2NH19.x6zQx2Ha_7C6g1UZKlxFPLU-JvJU4gaZ132_nzwLGEY',
+            'Content-Type': 'application/json',
+            'Accept': '*/*',
+            'Sec-Fetch-Site': 'cross-site',
+            'Sec-Fetch-Mode': 'cors',
+            'Sec-Fetch-Dest': 'empty',
+            'Referer': 'https://servicewechat.com/wx02bca7643610fee6/3/page-frame.html',
+            'Accept-Language': 'en-US,en;q=0.9'
+        }
+        response = requests.request(
+            "GET", url=url, headers=headers, params=params, proxies=tunnel_proxies()
+        )
+        data = response.json()
+        for index, video_obj in enumerate(data["data"]["list"], 1):
+            try:
+                self.aliyun_log.logging(
+                    code="1001",
+                    message="扫描到一条视频",
+                    data=video_obj,
+                )
+                self.process_video_obj(video_obj)
+            except Exception as e:
+                self.aliyun_log.logging(
+                    code="3000",
+                    message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e),
+                )
+
+    def run(self):
+        """
+        执行代码
+        :return: None
+        """
+        for page in range(1, 10):
+            if self.expire_flag:
+                self.aliyun_log.logging(
+                    code="2000",
+                    message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
+                )
+                return
+            else:
+                # self.get_recommend_list(page_index=page)
+                try:
+                    self.get_recommend_list(page_index=page)
+                except Exception as e:
+                    self.aliyun_log.logging(
+                        code="3000",
+                        message="抓取第{}页时候出现错误， 报错信息是{}".format(page, e),
+                    )
+

spider/spider_map.py

@@ -54,6 +54,10 @@ spider_map = {
     # 福气旺系列， 最好的送你
     "zuihaodesongni": {
         "recommend": ZuiHaoDeSongNi
+    },
+    # 节日祝福类
+    "jierizhufu": {
+        "recommend": JieRiZhuFu
     }
 
 }