修改万物复苏代码

zhuwanwufusu/zhuwanwufusu_recommend/zwwfs_recommend.py

@@ -1,6 +1,4 @@
 import os
-import re
-import base64
 import json
 import random
 import sys
@@ -13,7 +11,7 @@ sys.path.append(os.getcwd())
 from common.video_item import VideoItem
 from common import PiaoQuanPipeline, AliyunLogger, tunnel_proxies
 from common.mq import MQ
-from common.scheduling_db import MysqlHelper
+from common.db import MysqlHelper
 from zhuwanwufusu.crypt import AESCipher as AES
 
 
@@ -88,11 +86,88 @@ class ZhuWanWuFuSuRecommend(object):
                     env=self.env,
                     message="抓取第{}页的时候失败, 报错原因是{}".format(page_index, e)
                 )
+            time.sleep(random.randint(5, 10))
+
+    def get_user_videos(self, user_id):
+        """
+        在抓取完推荐页之后，去抓每一个用户的主页视频
+        """
+        url = "https://api.lidongze.cn/jeecg-boot/ugc/getAuthVideoList"
+        headers = {
+            'Host': 'api.lidongze.cn',
+            'xweb_xhr': '1',
+            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.4(0x13080410)XWEB/31009',
+            'token': '',
+            'content-type': 'application/json',
+            'accept': '*/*',
+            'referer': 'https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html',
+            'accept-language': 'en-US,en;q=0.9'
+        }
+        page_index = 1
+        total_page = 1
+        while page_index <= total_page:
+            query = {
+                "pageNo": page_index,
+                "pageSize": 10,
+                "authid": user_id
+            }
+            params = {
+                "v": self.cryptor.aes_encrypt(data=json.dumps(query))
+            }
+            response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
+            result = json.loads(self.cryptor.aes_decrypt(response.text))
+            total_page = result['list']['pages']
+            page_index = result['list']['current'] + 1
+            for index, video_temp in enumerate(result['list']['records']):
+                video_id = video_temp['id']
+                detail_query = {
+                    "videoId": video_id
+                }
+                detail_params = {
+                    "v": self.cryptor.aes_encrypt(data=json.dumps(detail_query))
+                }
+                url = "https://api.lidongze.cn/jeecg-boot/ugc/getVideosDataEn"
+                headers = {
+                    'Host': 'api.lidongze.cn',
+                    'xweb_xhr': '1',
+                    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.4(0x13080410)XWEB/31009',
+                    'token': '',
+                    'content-type': 'application/json',
+                    'accept': '*/*',
+                    'referer': 'https://servicewechat.com/wx0afdc2669ed8df2f/3/page-frame.html',
+                    'accept-language': 'en-US,en;q=0.9'
+                }
+                detail_response = requests.request("GET", url, headers=headers, params=detail_params,
+                                                   proxies=tunnel_proxies())
+                detail_video = json.loads(self.cryptor.aes_decrypt(detail_response.text))
+                if detail_video['success']:
+                    # print(json.dumps(detail_video, ensure_ascii=False, indent=4))
+                    try:
+                        AliyunLogger.logging(
+                            code="1001",
+                            platform=self.platform,
+                            mode=self.mode,
+                            env=self.env,
+                            message="扫描到一条视频",
+                            data=detail_video['data']
+                        )
+                        self.process_video_obj(detail_video['data'])
+                    except Exception as e:
+                        AliyunLogger.logging(
+                            code="3000",
+                            platform=self.platform,
+                            mode=self.mode,
+                            env=self.env,
+                            message="抓取单条视频失败, 该视频位于第{}条报错原因是{}".format(index, e)
+                        )
 
     def process_video_obj(self, video_obj):
         trace_id = self.platform + str(uuid.uuid1())
-        play_cnt = int(video_obj['playnum'].replace("万+", "0000")) if "万+" in video_obj['playnum'] else int(
-            video_obj['playnum'])
+        if video_obj.get("playnum"):
+            play_cnt = int(video_obj['playnum'].replace("万+", "0000")) if "万+" in video_obj['playnum'] else int(
+                video_obj['playnum'])
+        else:
+            play_cnt = 0
         item = VideoItem()
         item.add_video_info("video_id", video_obj['id'])
         item.add_video_info("video_title", video_obj['vname'])
@@ -108,7 +183,8 @@ class ZhuWanWuFuSuRecommend(object):
         item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
         item.add_video_info("user_id", self.user_dict['uid'])
         item.add_video_info("user_name", self.user_dict['nick_name'])
-
+        # 把扫描到的账号存到 accounts 表中
+        self.manage_auth_id(out_user_id=video_obj['authid'], out_user_name=video_obj['authname'])
         mq_obj = item.produce_item()
         pipeline = PiaoQuanPipeline(
             platform=self.platform,
@@ -120,7 +196,8 @@ class ZhuWanWuFuSuRecommend(object):
         )
         if pipeline.process_item():
             self.download_cnt += 1
-            self.mq.send_msg(mq_obj)
+            print(mq_obj)
+            # self.mq.send_msg(mq_obj)
             AliyunLogger.logging(
                 code="1002",
                 platform=self.platform,
@@ -130,13 +207,82 @@ class ZhuWanWuFuSuRecommend(object):
                 data=mq_obj
             )
 
+    def manage_auth_id(self, out_user_id, out_user_name):
+        """
+        out_user_id: 外站视频的用户 id
+        out_user_name: 外站视频用户名字
+        逻辑： 对新扫描到的视频的用户 id 进行判断，若用户 id 不存在，则把视频 id 存到表中,
+              如果用户 id 存在，则判断用户是否修改名字，若名字修改则更新名字
+        """
+        select_user_sql = f"""select name, name_id from accounts where name_id = "{out_user_id}" and platform = "{self.platform}" and useful = 1 limit 1"""
+        out_user_info = MysqlHelper.get_values(
+            log_type=self.mode,
+            crawler=self.platform,
+            sql=select_user_sql,
+            env=self.env,
+            machine="",
+        )
+        if out_user_info:
+            name, name_id = out_user_info[0]
+            if name == out_user_name:
+                return
+            else:
+                update_sql = f"""update accounts set name = "{out_user_name}" where name_id = "{out_user_id}";"""
+                MysqlHelper.update_values(
+                    log_type=self.mode,
+                    crawler=self.platform,
+                    sql=update_sql,
+                    env=self.env,
+                    machine=""
+                )
+        else:
+            insert_sql = f"""INSERT INTO accounts (name, name_id, platform, useful) values ("{out_user_name}", "{out_user_id}", "{self.platform}", 1 )"""
+            MysqlHelper.update_values(
+                log_type=self.mode,
+                crawler=self.platform,
+                sql=insert_sql,
+                env=self.env,
+                machine="",
+            )
+
+    def get_user_list(self):
+        select_user_sql = f"""select name_id from accounts where platform = "{self.platform}" and useful = 1"""
+        out_user_info = MysqlHelper.get_values(
+            log_type=self.mode,
+            crawler=self.platform,
+            sql=select_user_sql,
+            env=self.env,
+            machine="",
+        )
+        if out_user_info:
+            result = []
+            for i in out_user_info:
+                result.append(i[0])
+            return result
+        else:
+            return []
+
+    def schedule(self):
+        """
+        先抓取推荐列表的视频， 等待 5 分钟后，抓取账号视频
+        """
+        self.get_recommend_list()
+        time.sleep(5 * 60)
+        self.mode = "author"
+        user_list = self.get_user_list()
+        if user_list:
+            for index, user_id in enumerate(user_list):
+                try:
+                    self.get_user_videos(user_id=user_id)
+                except Exception as e:
+                    AliyunLogger.logging(
+                        code="3000",
+                        platform=self.platform,
+                        mode=self.mode,
+                        env=self.env,
+                        message="抓取账号视频出现异常，账号 id 是{}, 报错原因是{}".format(user_id, e)
+                    )
+
 
 if __name__ == '__main__':
-    Z = ZhuWanWuFuSuRecommend(
-        platform="zhuwanwufusu",
-        mode="recommend",
-        rule_dict={},
-        user_dict={"uid": 123456, "nick_name": "luojunhuishuaige"},
-        env="prod"
-    )
-    Z.get_recommend_list()
+    pass

zhuwanwufusu/zhuwanwufusu_recommend/zwwfs_recommend_test.py

@@ -64,6 +64,7 @@ class ZhuWanWuFuSuRecommend(object):
                 self.process_video_obj(video_obj)
 
     def process_video_obj(self, video_obj):
+        print(json.dumps(video_obj, ensure_ascii=False, indent=4))
         trace_id = self.platform + str(uuid.uuid1())
         play_cnt = int(video_obj['playnum'].replace("万+", "0000")) if "万+" in video_obj['playnum'] else int(
             video_obj['playnum'])