piaoquan_crawler/haokanshipin/haokanshipin_author/hksp_author.py

import os
import re
import base64
import json
import random
import sys
import datetime
import time
import uuid

import requests
from fake_useragent import FakeUserAgent

sys.path.append(os.getcwd())
from common.video_item import VideoItem
from common import PiaoQuanPipeline, AliyunLogger
from common.mq import MQ


class HaoKaoVideoAccount:
    def __init__(self, platform, mode, rule_dict, user_dict, env):
        self.account_id = user_dict["link"]
        self.platform = platform
        self.mode = mode
        self.rule_dict = rule_dict
        self.user_dict = user_dict
        self.env = env
        self.download_cnt = 0
        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
        self.url_key = "guanghui456"

    # 解密
    def decrypt(self, ori_str):
        # base64解码
        base64_bytes = base64.b64decode(ori_str)
        crypt_str = base64_bytes.decode("utf-8")
        # 异或解密
        n = ""
        for i in range(len(crypt_str)):
            o = ord(crypt_str[i])
            s = ord(self.url_key[i % len(self.url_key)])
            n += chr(o ^ s)
        return n

    # 获取视频列表
    def get_video_list(self):
        page_limit = 10
        url = "https://haokan.baidu.com/web/author/listall"
        params = {
            "app_id": self.account_id,  # 账号id
            "ctime": "",  # 翻页指示，用时间戳表示，为空表示从头开始
            "rn": page_limit,  # 每一页的页数，默认是 10
            "_api": 1
        }
        headers = {
            'Accept': '*/*',
            'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Content-Type': 'application/x-www-form-urlencoded',
            'Cookie': 'BIDUPSID=504D4A3A8D0584CA8C3BE27ACFED5323; PSTM=1695297510; BAIDUID=504D4A3A8D0584CA7398158209FA507F:FG=1; BAIDUID_BFESS=504D4A3A8D0584CA7398158209FA507F:FG=1; H_WISE_SIDS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; H_WISE_SIDS_BFESS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%7D; H_PS_PSSID=39624_39663_39684_39690_39676_39678_39713; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; hkpcSearch=%u674E%u6709%u8D22; BA_HECTOR=0h240k018l2ka0a000a00g0g1il6sn21q; ZFY=PnfJdvQrpMNOIoZw1LsOKMDW8:BWcrmDvtL60fANYxmQ:C; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1699950106,1700038092; PC_TAB_LOG=video_details_page; COMMON_LID=c414c76cb1b11dfb04b7062a5ae09ce2; ab_sr=1.0.1_ZGEwNWY3OTFhZmNhNzZiN2YyZjljYjI3ZmE2YzRiZDA5NjY3YTI0OTdiMWM0ZWZhMjU3YWMzMThmZjU5YzQ5MDM0NDA4N2FmZWM5M2I0ZmQzZmE0ZWI2MjJkMjE0MDQ1ZTIyYmU5OGQ3OWY3ZDNlYTI2NmFiMGZlYTliODFmZmJiY2U0MDRmMzkwZWMzOTQxMTc0MDU3NmQ2ZWVlNjM3ZA==; reptileData=%7B%22data%22%3A%22636c55e0319da5169a60acec4a264a35c10862f8abfe2f2cc32c55eb6b0ab4de0efdfa115ea522d6d4d361dea07feae27710e59370b70671d347daddec662182e4c514cbcba8dae1736cf735f29e19b9a42be523d0836d04fa3b15dd57de52f4%22%2C%22key_id%22%3A%2230%22%2C%22sign%22%3A%22551bc5f4%22%7D; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1700038278; RT="z=1&dm=baidu.com&si=bd56d547-8a2c-433d-9eee-70f64ccf646a&ss=lozishrc&sl=9&tt=c2s&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=6z26"',
            'Referer': 'https://haokan.baidu.com/author/1774716548084013',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
        }
        response = requests.request("GET", url, headers=headers, params=params)
        result = response.json()
        print(json.dumps(result, ensure_ascii=False, indent=4))

    # 获取小视频列表
    def get_tiny_video_list(self):
        url = "https://haokan.baidu.com/web/author/listall"
        params = {
            "app_id": self.account_id,
            "ctime": "",
            "video_type": "haokan|tabhubVideo",
            "rn": 20
        }
        headers = {
            'Accept': '*/*',
            'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Content-Type': 'application/x-www-form-urlencoded',
            'Cookie': 'BIDUPSID=504D4A3A8D0584CA8C3BE27ACFED5323; PSTM=1695297510; BAIDUID=504D4A3A8D0584CA7398158209FA507F:FG=1; BAIDUID_BFESS=504D4A3A8D0584CA7398158209FA507F:FG=1; H_WISE_SIDS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; H_WISE_SIDS_BFESS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%7D; H_PS_PSSID=39624_39663_39684_39690_39676_39678_39713; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; hkpcSearch=%u674E%u6709%u8D22; BA_HECTOR=0h240k018l2ka0a000a00g0g1il6sn21q; ZFY=PnfJdvQrpMNOIoZw1LsOKMDW8:BWcrmDvtL60fANYxmQ:C; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1699950106,1700038092; PC_TAB_LOG=video_details_page; COMMON_LID=c414c76cb1b11dfb04b7062a5ae09ce2; BDRCVFR[X_XKQks0S63]=mk3SLVN4HKm; ab_sr=1.0.1_ODM2M2M4NzY2MWE3Yjg0MGQ4NDk2YTQ4ZTRlMWVlYjdiY2JmMzQ5ZjU1MjAxODUxZTQ0NTg4YjJjNzNmYTU2MzZiMjI2Y2EwZTU2OGIwYjdmMDc3NmRhMGJjODZkZmIyN2U0YWFjMjUzZWEwNTRlZWQ1N2U0MzkxY2YwMzk4Zjk1N2NiYWM1OGZlN2M0NWU4ZWJiZjFmNWE5YWU3YjFmMA==; reptileData=%7B%22data%22%3A%22636c55e0319da5169a60acec4a264a35c10862f8abfe2f2cc32c55eb6b0ab4de0efdfa115ea522d6d4d361dea07feae27710e59370b70671d347daddec6621825f6963cc8f86c5fee1d0664c82edf0ae5c838cf8bde5188e6a7757f1998a7c48%22%2C%22key_id%22%3A%2230%22%2C%22sign%22%3A%22f302bd06%22%7D; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1700046227; RT="z=1&dm=baidu.com&si=bd56d547-8a2c-433d-9eee-70f64ccf646a&ss=loznmnju&sl=1&tt=1w3&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=2oq"',
            'Referer': 'https://haokan.baidu.com/author/1702511677416581',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
        }
        response = requests.request("GET", url, headers=headers, params=params)
        result = response.json()
        print(json.dumps(result, ensure_ascii=False, indent=4))

    # 处理视频列表（建议可以公共化）
    def process_video_list(self, result):
        if result['errmsg'] == "成功":
            video_list = result['data']['results']
            for index, video_obj in enumerate(video_list):
                try:
                    AliyunLogger.logging(
                        code="1001",
                        platform=self.platform,
                        mode=self.mode,
                        env=self.env,
                        message="扫描到一条视频",
                        data=video_obj
                    )
                    self.get_video_info(video_obj)
                except Exception as e:
                    AliyunLogger.logging(
                        code="3000",
                        platform=self.platform,
                        mode=self.mode,
                        env=self.env,
                        message="抓取单条视频异常，报错是:{}".format(e)
                    )
        else:
            AliyunLogger.logging(
                code="3000",
                platform=self.platform,
                mode=self.mode,
                env=self.env,
                message="请求视频链接列表失败，需要 review 代码验证"
            )

    # 处理单条视频信息
    def get_video_info(self, video_obj):
        url = "https://haokan.baidu.com/v?vid={}&collection_id=".format(video_obj['content']['vid'])
        header = {
            "User-Agent": FakeUserAgent().random()
        }
        response_text = requests.get(url, headers=header).text
        encrypted_data = json.loads(
            re.search(r'window\.__PRELOADED_STATE__\s*=\s*(.*?);', response_text).group(1).strip())
        encrypted_video_obj = encrypted_data['encrptedVideoMeta']
        d_obj = json.loads(self.decrypt(encrypted_video_obj))
        AliyunLogger.logging(
            code="1001",
            platform=self.platform,
            mode=self.mode,
            env=self.env,
            data=d_obj,
            message="扫描到一条视频"
        )
        trace_id = self.platform + str(uuid.uuid1())
        item = VideoItem()
        item.add_video_info("video_id", d_obj['id'])
        item.add_video_info("video_title", d_obj['title'])
        item.add_video_info("playcnt", d_obj['playcnt'])
        item.add_video_info("publish_time_stamp", d_obj['publish_time'])
        item.add_video_info("duration", d_obj['duration'])
        item.add_video_info("video_url", d_obj['playurl'])
        item.add_video_info("like_cnt", d_obj['like'])
        item.add_video_info("comment_cnt", d_obj['comment'])
        item.add_video_info("cover_url", video_obj['content']['cover_src'])
        # 准备发往 MQ 的消息
        mq_obj = item.produce_item()
        # 筛选规则的 pipeline
        pipeline = PiaoQuanPipeline(
            platform=self.platform,
            mode=self.mode,
            rule_dict=self.rule_dict,
            env=self.env,
            item=mq_obj,
            trace_id=trace_id
        )
        if pipeline.process_item():
            self.mq.send_msg(mq_obj)
            AliyunLogger.logging(
                code="1002",
                platform=self.platform,
                mode=self.mode,
                env=self.env,
                message="成功发送至 ETL",
                data=mq_obj,
                trace_id=trace_id
            )


if __name__ == "__main__":
    T = HaoKaoVideoAccount(
        platform="baidu",
        mode="author",
        rule_dict={},
        user_dict={"link": 1657075178605219},
        env="prod"
    )
    T.get_tiny_video_list()