automatic_crawler/spider/ad_click/piaoquan_vlog.py

"""
Created on Mon Feb 19 2024
"""
import os
import sys
import json
import random
import time
import requests
import datetime

sys.path.append(os.getcwd())

from application.common.proxies import tunnel_proxies
from application.common.log import AliyunLogger
from application.common.feishu import FeishuInsert


class PiaoQuanVlog(object):
    """
    票圈 vlog 推荐监测
    会影响推荐的参数
        算法： app_type, mid， timestamp, machine_Info, abtest, extParam, pageSource， IP
    """

    def __init__(self):
        self.url = "https://vlogapi.piaoquantv.com/longvideoapi/video/distribute/category/videoList/v2"
        self.headers = {
            'Host': 'vlogapi.piaoquantv.com',
            'xweb_xhr': '1',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.6(0x13080610) XWEB/1156',
            'content-type': 'application/x-www-form-urlencoded',
            'accept': '*/*',
            'sec-fetch-site': 'cross-site',
            'sec-fetch-mode': 'cors',
            'sec-fetch-dest': 'empty',
            'referer': 'https://servicewechat.com/wx89e7eb06478361d7/735/page-frame.html',
            'accept-language': 'en-US,en;q=0.9'
        }
        self.aliyun_log = AliyunLogger(platform="piaoquanVlog", mode="recommend")
        self.feishu = FeishuInsert(document_token="PN9usqN4ehDIFxtdsuqcH546nwe")
        self.sheet_id = "cc9ace"

    def send_request(self, page_num):
        """
        :param page_num: 第几页
        """
        payload = {
            "categoryJson": '{"categoryId":55}',
            "pageNo": page_num,
            "pageSize": 4,
            "sortField": 0,
            "pageSource": "vlog-pages/category",
            "sharePageVideoId": None,
            "shareId": "",
            "careModelStatus": 1,
            "token": "eacc679e9719307517de3d945479a545add3d48a",
            "loginUid": 64168088,
            "platform": "mac",
            "versionCode": 760,
            "machineCode": "weixin_openid_o0w175U16THwQmviRDIdGFmFgc6U",
            "appType": 0,
            "realAppType": 0,
            "system": "Mac OS X 14.2.1",
            "pageCategoryId": 55,
            "rootPageSource": "",
            "shareDepth": "",
            "rootPageCategoryId": "",
            "appId": "wx89e7eb06478361d7",
            "clientTimestamp": int(time.time() * 1000),
            "machineInfo": '{"sdkVersion":"3.3.4","brand":"apple","language":"zh_CN","model":"Mac14,2","platform":"mac","system":"Mac OS X 14.2.1","weChatVersion":"3.8.6","screenHeight":736,"screenWidth":414,"pixelRatio":2,"windowHeight":736,"windowWidth":414,"softVersion":"4.1.760"}',
            "networkType": "wifi",
            "network": "wifi",
            "sessionId": "1708320545204-7cfd21f2-2762-a2a8-3331-7fa04e844ab0",
            "subSessionId": "1708320545204-7cfd21f2-2762-a2a8-3331-7fa04e844ab0",
            "returnId": "",
            "jumpHomeVideoId": "",
            "senceType": 1089,
            "hotSenceType": 1089,
            "abExpInfo": '{"ab_test004":[{"abExpCode":"126","configValue":""},{"abExpCode":"211","configValue":""}],"ab_test005":[],"ab_test006":[{"abExpCode":"310","configValue":""},{"abExpCode":"321","configValue":""},{"abExpCode":"331","configValue":""},{"abExpCode":"356","configValue":"{\\"playProgress\\": 10, \\"delayHide\\": 8}"},{"abExpCode":"371","configValue":"{\\"playIcon\\": \\"http://weapppiccdn.yishihui.com/wxicon/common/icon_play_btn_font.png?v=2\\", \\"width\\": 125}"}],"ab_test001":[{"abExpCode":"223","configValue":""},{"abExpCode":"201","configValue":""},{"abExpCode":"410","configValue":"{\\"layerStyle\\": 1, \\"oneDayShowCount\\": 3, \\"everyRecommendVideo\\": 5, \\"playProgress\\": 90, \\"closePosition\\": \\"top\\", \\"guideDialogText\\": [\\"您可以点击‘关注票圈公众号’\\", \\"每日最新资讯不错过\\"], \\"guideButtonText\\": \\"关注票圈公众号\\", \\"topImage\\": \\"https://weapppiccdn.yishihui.com/wxicon/common/img_cgi_image3.png\\", \\"jumpUrl\\": \\"https://mp.weixin.qq.com/s?__biz=MzIxMjg2MzE2Mg==&mid=2247483675&idx=1&sn=0338228015ba7a5b0a1937b14e610efc&chksm=97bed0cea0c959d81d90a5d9ce82502ca24fa418df70d6e619a88d4e007a8b14b2b3b3e62386#rd\\", \\"gzhId\\": 105}"}]}',
            "extParams": '{"eventIds":"22040202,ab100,ab100,ab100,ab100,ab100,ab100,ab100,ab100,ab100,ab100","eventInfos":{"ab_test001":"ab100","ab_test002":"ab100","ab_test003":"ab100","ab_test004":"ab100","ab_test005":"ab100","ab_test006":"ab100","ab_test007":"ab100","ab_test008":"ab100","ab_test009":"ab100","ab_test010":"ab100"}}'
        }
        basic_response = requests.request("POST", self.url, headers=self.headers, data=payload,
                                          ).json()
        self.process_video_list(basic_response['data'])

    def process_video_list(self, video_list):
        """
        处理返回的视频列表
        :param video_list: 请求到的 video_list
        :return: None
        """
        for video_obj in video_list:
            # print(json.dumps(video_obj, ensure_ascii=False, indent=4))
            video_item = {
                "id": video_obj['id'],
                "status": video_obj['status'],
                "uid": video_obj['uid'],
                "playCount": video_obj['playCount'],
                "title": video_obj['title'],
                "titleId": video_obj['titleId'],
                "shareCount": video_obj['shareCountFriend'],
                "favorCount": video_obj['favoriteds'],
                "publish_date": video_obj['gmtCreateDescr']
            }
            time.sleep(12)
            self.aliyun_log.logging(
                code="7001",
                message="监控到一条视频",
                data=video_item
            )
            line = [video_item["id"], video_item["status"], video_item["uid"], video_item["title"], video_item["titleId"],video_item['playCount'], video_item["shareCount"], video_item["favorCount"], datetime.datetime.now().__str__(), video_item['publish_date']]
            self.feishu.insert_value(
                sheet_id=self.sheet_id,
                values=[line],
                ranges="A2:K2"
            )

    def run(self):
        """
        一天抓取 24h， 每个小时的 0-15min 不抓取，每一个小时抓取条数为 110，每天抓取条数为 24 * 110 = 2640  条
        110 / 4 =～ 28， 每一小时大抓取 28 页
        :return: None
        """
        while True:
            # 每一小时执行一次
            current_time = datetime.datetime.now()
            if 0 <= current_time.minute < 15:
                # 计算需要等待的秒数，直到15分钟过去
                wait_time = (15 - current_time.minute) * 60 - current_time.second
                time.sleep(wait_time)
            else:
                # 平均 96 秒抓一页，即 96秒抓 4 条，每条视频之间等待时间是 24s
                # 一共抓取 28 页
                for index in range(1, 29):
                    try:
                        self.send_request(index)
                        # 随机休息 1 - 50  秒
                        time.sleep(random.randint(4 * 10, 4 * 14))
                    except Exception as e:
                        self.aliyun_log.logging(
                            code="3000",
                            message="扫描第{}页失败， 原因是{}".format(index, e)
                        )


if __name__ == '__main__':
    P = PiaoQuanVlog()
    P.run()