福小顺 调试

application/common/feishu/feishu.py

@@ -46,6 +46,8 @@ class Feishu:
     crawler_zhihu = 'https://w42nne6hzg.feishu.cn/sheets/shtcnkGPBmGsjaqapgzouuj8MXe?'
     # 吉祥幸福
     crawler_jixiangxingfu = 'https://w42nne6hzg.feishu.cn/sheets/shtcnSx4nafMbLTq7xl7RHBwHBf?'
+    # 福小顺
+    crawler_fuxiaoshun = 'https://w42nne6hzg.feishu.cn/sheets/CoXEsl6MDhMaKKt6GUBcvLwsnWb?'
     # 众妙音信
     crawler_zmyx = 'https://w42nne6hzg.feishu.cn/sheets/shtcnbZIxstPeM0xshW07b26sve?'
     # 岁岁年年迎福气
@@ -107,6 +109,8 @@ class Feishu:
             return 'shtcnkGPBmGsjaqapgzouuj8MXe'
         elif crawler == 'jixiangxingfu':
             return 'shtcnSx4nafMbLTq7xl7RHBwHBf'
+        elif crawler == 'fuxiaoshun':
+            return 'CoXEsl6MDhMaKKt6GUBcvLwsnWb'
         elif crawler == 'zhongmiaoyinxin':
             return 'shtcnbZIxstPeM0xshW07b26sve'
         elif crawler == 'suisuiniannianyingfuqi':

spider/crawler_online/fuxiaoshun.py

@@ -13,15 +13,19 @@ import asyncio
 import aiohttp
 import datetime
 from base64 import b64decode
+from datetime import datetime
+
+import requests
 from Crypto.Cipher import AES
 from Crypto.Util.Padding import unpad
 
+
 sys.path.append(os.getcwd())
+from application.common import Feishu
 
 from application.items import VideoItem
 from application.pipeline import PiaoQuanPipeline
 from application.common.messageQueue import MQ
-from application.common.proxies import tunnel_proxies
 from application.common.log import AliyunLogger
 
 
@@ -60,14 +64,14 @@ class FuXiaoShunRecommend(object):
         self.expire_flag = False
         self.aliyun_log = AliyunLogger(platform=self.platform, mode=self.mode)
 
-    async def process_video_obj(self, video_obj):
+    def process_video_obj(self, video_obj):
         """
         处理每一个视频内容
         :return: None
         """
         trace_id = self.platform + str(uuid.uuid1())
         our_user = random.choice(self.user_list)
-        publish_time_stamp = datetime.datetime.strptime(
+        publish_time_stamp = datetime.strptime(
             video_obj["create_at"], "%Y-%m-%d %H:%M:%S"
         ).timestamp()
         item = VideoItem()
@@ -89,6 +93,7 @@ class FuXiaoShunRecommend(object):
         item.add_video_info("strategy", self.mode)
         item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
         mq_obj = item.produce_item()
+
         pipeline = PiaoQuanPipeline(
             platform=self.platform,
             mode=self.mode,
@@ -99,8 +104,22 @@ class FuXiaoShunRecommend(object):
         )
         if pipeline.process_item():
             self.download_cnt += 1
+            # 获取当前时间
+            current_time = datetime.now()
+            formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
+            values = [[
+                video_obj["id"],
+                formatted_time,
+                video_obj["name"],
+                video_obj["cover"] + "&vframe/png/offset/1/w/200",
+                video_obj["cover"],
+                video_obj["num_like"],
+                video_obj["num_read"]
+            ]]
+            Feishu.insert_columns(self.platform, 'fuxiaoshun', "0e1e47", "ROWS", 1, 2)
+            time.sleep(0.5)
+            Feishu.update_values(self.platform, 'fuxiaoshun', "0e1e47", "A2:Z2", values)
             self.mq.send_msg(mq_obj)
-            # print(json.dumps(mq_obj, ensure_ascii=False, indent=4))
             self.aliyun_log.logging(
                 code="1002",
                 message="成功发送至 ETL",
@@ -111,10 +130,9 @@ class FuXiaoShunRecommend(object):
             ):
                 self.expire_flag = True
 
-    async def get_recommend_list(self, session, page_index):
+    def get_recommend_list(self, page_index):
         """
         获取推荐页面的video_list
-        :param session: aiohttp 的session
         :param page_index: 页码
         :return: None
         """
@@ -124,8 +142,18 @@ class FuXiaoShunRecommend(object):
                 message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
             )
             return
+        host_referer_mapping = {
+            'quan.nnjuxing.cn': 'https://servicewechat.com/wxbb18ecb64efe217a/2/page-frame.html',  # 福小全
+            'nian.nnjuxing.cn': 'https://servicewechat.com/wx3e31d735ebb23d29/3/page-frame.html',  # 福小年
+            'shun.nnjuxing.cn': 'https://servicewechat.com/wx5b89401c90c9f367/3/page-frame.html'   # 福小顺
+        }
+        # 随机选择一个 host
+        random_host = random.choice(list(host_referer_mapping.keys()))
+
+        # 根据选择的 host 获取对应的 referer
+        random_referer = host_referer_mapping[random_host]
         headers = {
-            "Host": "shun.nnjuxing.cn",
+            "Host": random_host,
             "xweb_xhr": "1",
             "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.6(0x13080610) XWEB/1156",
             "content-type": "application/json",
@@ -133,71 +161,63 @@ class FuXiaoShunRecommend(object):
             "sec-fetch-site": "cross-site",
             "sec-fetch-mode": "cors",
             "sec-fetch-dest": "empty",
-            "referer": "https://servicewechat.com/wx5b89401c90c9f367/3/page-frame.html",
+            "referer": random_referer,
             "accept-language": "en-US,en;q=0.9",
         }
-        url = "https://shun.nnjuxing.cn/videos/api.videos/getItem"
-        params = {"mark": "", "page": page_index}
-        await asyncio.sleep(5)
-        async with session.get(
-            url, headers=headers, params=params, proxy=tunnel_proxies()['https']
-        ) as response:
-            cryp_data = await response.json()
-            data = json.loads(fxs_decrypt(cryp_data["data"]))
-            for index, video_obj in enumerate(data["list"], 1):
-                try:
-                    self.aliyun_log.logging(
-                        code="1001",
-                        message="扫描到一条视频",
-                        data=video_obj,
-                    )
-                    await self.process_video_obj(video_obj)
-                except Exception as e:
-                    self.aliyun_log.logging(
-                        code="3000",
-                        message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e),
-                    )
+        url = f"https://{random_host}/videos/api.videos/getItem?page={page_index}"
+
+        payload = {}
+        time.sleep(10)
+        response = requests.request("GET", url, headers=headers, data=payload)
+        cryp_data = response.json()
+        data = json.loads(fxs_decrypt(cryp_data["data"]))
+        for index, video_obj in enumerate(data["list"], 1):
+            try:
+                self.aliyun_log.logging(
+                    code="1001",
+                    message="扫描到一条视频",
+                    data=video_obj,
+                )
+                self.process_video_obj(video_obj)
+            except Exception as e:
+                self.aliyun_log.logging(
+                    code="3000",
+                    message="抓取第{}条的时候出现问题， 报错信息是{}".format(index, e),
+                )
 
-    async def run(self):
+    def run(self):
         """
         执行代码
         :return: None
         """
-        async with aiohttp.ClientSession() as session:
-            # for i in range(0, 100, 20):
-            #     if self.expire_flag:
-            #         return
-            #     tasks = [self.get_recommend_list(session, index) for index in range(i, min(i + 20, 200))]
-            #     await asyncio.gather(*tasks)
-            #     done, pending = await asyncio.wait(
-            #         tasks, return_when=asyncio.FIRST_COMPLETED
-            #     )
-            #     # 取消所有剩余的任务
-            #     for task in pending:
-            #         task.cancel()
-
-            # tasks = [self.get_recommend_list(session, index) for index in range(1, 100)]
-            # await asyncio.gather(*tasks)
-            # done, pending = await asyncio.wait(
-            #     tasks, return_when=asyncio.FIRST_COMPLETED
-            # )
-            # # 取消所有剩余的任务
-            # for task in pending:
-            #     task.cancel()
-            for page in range(1, 200):
-                if self.expire_flag:
+        for page in range(1, 200):
+            if self.expire_flag:
+                self.aliyun_log.logging(
+                    code="2000",
+                    message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
+                )
+                message = "本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt)
+                print(message)
+                return
+            else:
+                try:
+                    self.get_recommend_list(page_index=page)
+                except Exception as e:
                     self.aliyun_log.logging(
-                        code="2000",
-                        message="本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt),
+                        code="3000",
+                        message="抓取第{}页时候出现错误， 报错信息是{}".format(page, e),
                     )
-                    message = "本轮已经抓取到足够的数据，自动退出\t{}".format(self.download_cnt)
-                    print(message)
-                    return
-                else:
-                    try:
-                        await self.get_recommend_list(session, page_index=page)
-                    except Exception as e:
-                        self.aliyun_log.logging(
-                            code="3000",
-                            message="抓取第{}页时候出现错误， 报错信息是{}".format(page, e),
-                        )
+
+
+if __name__ == '__main__':
+    J = FuXiaoShunRecommend(
+        platform="fuxiaoshun",
+        mode="recommend",
+        rule_dict={},
+        user_list=[{'uid': "123456", 'nick_name': "xiaoxiao"}],
+
+    )
+    J.run()
+
+
+