piaoquan_crawler/haokanshipin/haokanshipin_author/hksp_test.py

import os
import re
import base64
import json
import random
import sys
import time
import uuid

import requests

sys.path.append(os.getcwd())
from common.video_item import VideoItem
from common.pipeline import PiaoQuanPipelineTest
from common.mq import MQ
from common.scheduling_db import MysqlHelper


def tunnel_proxies():
    # 隧道域名:端口号
    tunnel = "q796.kdltps.com:15818"
    # 用户名密码方式
    username = "t17772369458618"
    password = "5zqcjkmy"
    proxies = {
        "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
        "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
    }
    return proxies


class HaoKanVideoAccount(object):
    def __init__(self, platform, mode, rule_dict, user_dict, env):
        self.account_id = user_dict["link"].split("/")[-1]
        self.platform = platform
        self.mode = mode
        self.rule_dict = rule_dict
        self.user_dict = user_dict
        self.env = env
        self.download_cnt = 0
        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
        self.url_key = "guanghui456"
        self.expire_flag = False

    def decrypt(self, ori_str):
        # base64解码
        base64_bytes = base64.b64decode(ori_str)
        crypt_str = base64_bytes.decode("utf-8")
        # 异或解密
        n = ""
        for i in range(len(crypt_str)):
            o = ord(crypt_str[i])
            s = ord(self.url_key[i % len(self.url_key)])
            n += chr(o ^ s)
        return n

    def get_video_list(self):
        page_limit = 10
        url = "https://haokan.baidu.com/web/author/listall"
        headers = {
            "Accept": "*/*",
            "Accept-Language": "en,zh-CN;q=0.9,zh;q=0.8",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded",
            "Cookie": 'BIDUPSID=504D4A3A8D0584CA8C3BE27ACFED5323; PSTM=1695297510; BAIDUID=504D4A3A8D0584CA7398158209FA507F:FG=1; BAIDUID_BFESS=504D4A3A8D0584CA7398158209FA507F:FG=1; H_WISE_SIDS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; H_WISE_SIDS_BFESS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%7D; H_PS_PSSID=39624_39663_39684_39690_39676_39678_39713; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; hkpcSearch=%u674E%u6709%u8D22; BA_HECTOR=0h240k018l2ka0a000a00g0g1il6sn21q; ZFY=PnfJdvQrpMNOIoZw1LsOKMDW8:BWcrmDvtL60fANYxmQ:C; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1699950106,1700038092; PC_TAB_LOG=video_details_page; COMMON_LID=c414c76cb1b11dfb04b7062a5ae09ce2; ab_sr=1.0.1_ZGEwNWY3OTFhZmNhNzZiN2YyZjljYjI3ZmE2YzRiZDA5NjY3YTI0OTdiMWM0ZWZhMjU3YWMzMThmZjU5YzQ5MDM0NDA4N2FmZWM5M2I0ZmQzZmE0ZWI2MjJkMjE0MDQ1ZTIyYmU5OGQ3OWY3ZDNlYTI2NmFiMGZlYTliODFmZmJiY2U0MDRmMzkwZWMzOTQxMTc0MDU3NmQ2ZWVlNjM3ZA==; reptileData=%7B%22data%22%3A%22636c55e0319da5169a60acec4a264a35c10862f8abfe2f2cc32c55eb6b0ab4de0efdfa115ea522d6d4d361dea07feae27710e59370b70671d347daddec662182e4c514cbcba8dae1736cf735f29e19b9a42be523d0836d04fa3b15dd57de52f4%22%2C%22key_id%22%3A%2230%22%2C%22sign%22%3A%22551bc5f4%22%7D; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1700038278; RT="z=1&dm=baidu.com&si=bd56d547-8a2c-433d-9eee-70f64ccf646a&ss=lozishrc&sl=9&tt=c2s&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=6z26"',
            "Referer": "https://haokan.baidu.com/author/1774716548084013",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
        }
        p_flag = ""
        while True:
            params = {
                "app_id": self.account_id,  # 账号id
                "ctime": p_flag,  # 翻页指示，用时间戳表示，为空表示从头开始
                "rn": page_limit,  # 每一页的页数，默认是 10
                "_api": 1,
            }
            response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
            result = response.json()
            p_flag = result['data']['ctime']
            self.process_video_list(result)
            if self.expire_flag:
                return
            if result['data']['has_more'] == 0:
                return

    def get_tiny_video_list(self):
        url = "https://haokan.baidu.com/web/author/listall"
        headers = {
            "Accept": "*/*",
            "Accept-Language": "en,zh-CN;q=0.9,zh;q=0.8",
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded",
            "Cookie": 'BIDUPSID=504D4A3A8D0584CA8C3BE27ACFED5323; PSTM=1695297510; BAIDUID=504D4A3A8D0584CA7398158209FA507F:FG=1; BAIDUID_BFESS=504D4A3A8D0584CA7398158209FA507F:FG=1; H_WISE_SIDS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; H_WISE_SIDS_BFESS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%7D; H_PS_PSSID=39624_39663_39684_39690_39676_39678_39713; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; hkpcSearch=%u674E%u6709%u8D22; BA_HECTOR=0h240k018l2ka0a000a00g0g1il6sn21q; ZFY=PnfJdvQrpMNOIoZw1LsOKMDW8:BWcrmDvtL60fANYxmQ:C; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1699950106,1700038092; PC_TAB_LOG=video_details_page; COMMON_LID=c414c76cb1b11dfb04b7062a5ae09ce2; BDRCVFR[X_XKQks0S63]=mk3SLVN4HKm; ab_sr=1.0.1_ODM2M2M4NzY2MWE3Yjg0MGQ4NDk2YTQ4ZTRlMWVlYjdiY2JmMzQ5ZjU1MjAxODUxZTQ0NTg4YjJjNzNmYTU2MzZiMjI2Y2EwZTU2OGIwYjdmMDc3NmRhMGJjODZkZmIyN2U0YWFjMjUzZWEwNTRlZWQ1N2U0MzkxY2YwMzk4Zjk1N2NiYWM1OGZlN2M0NWU4ZWJiZjFmNWE5YWU3YjFmMA==; reptileData=%7B%22data%22%3A%22636c55e0319da5169a60acec4a264a35c10862f8abfe2f2cc32c55eb6b0ab4de0efdfa115ea522d6d4d361dea07feae27710e59370b70671d347daddec6621825f6963cc8f86c5fee1d0664c82edf0ae5c838cf8bde5188e6a7757f1998a7c48%22%2C%22key_id%22%3A%2230%22%2C%22sign%22%3A%22f302bd06%22%7D; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1700046227; RT="z=1&dm=baidu.com&si=bd56d547-8a2c-433d-9eee-70f64ccf646a&ss=loznmnju&sl=1&tt=1w3&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=2oq"',           "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
        }
        cursor = ""
        while True:
            params = {
                "app_id": self.account_id,
                "ctime": cursor,
                "video_type": "haokan|tabhubVideo",
                "rn": 20,
            }
            response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies())
            result = response.json()
            cursor = result['data']['ctime']
            self.process_video_list(result)
            if self.expire_flag:
                return
            if result['data']['has_more'] == 0:
                print("No more videos")
                return

    def process_video_list(self, result):
        if result["errmsg"] == "成功":
            video_list = result["data"]["results"]
            for index, video_obj in enumerate(video_list):
                # self.get_video_info(video_obj)
                try:
                    if self.expire_flag:
                        return
                    else:
                        print("扫描到一条视频")
                        self.get_video_info(video_obj)
                except Exception as e:
                    print("抓取单条视频异常，报错是:{}".format(e))
        else:
            print("请求视频链接列表失败，需要 review 代码验证")

    def get_video_info(self, video_obj):
        url = "https://haokan.baidu.com/v?vid={}&collection_id=".format(
            video_obj["content"]["vid"]
        )
        header = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"
        }
        response_text = requests.get(url, headers=header).text
        encrypted_data = json.loads(
            re.search(r"window\.__PRELOADED_STATE__\s*=\s*(.*?);", response_text)
            .group(1)
            .strip()
        )
        encrypted_video_obj = encrypted_data["encrptedVideoMeta"]
        d_obj = json.loads(self.decrypt(encrypted_video_obj))
        trace_id = self.platform + str(uuid.uuid1())
        item = VideoItem()
        item.add_video_info("video_id", d_obj["id"])
        item.add_video_info("video_title", d_obj["title"])
        item.add_video_info("play_cnt", d_obj["playcnt"])
        item.add_video_info("publish_time_stamp", d_obj["publish_time"])
        item.add_video_info("duration", d_obj["duration"])
        item.add_video_info("video_url", d_obj["playurl"])
        item.add_video_info("like_cnt", d_obj["like"])
        item.add_video_info("comment_cnt", d_obj["comment"])
        item.add_video_info("cover_url", video_obj["content"]["cover_src"])
        item.add_video_info("user_id", self.user_dict['uid'])
        item.add_video_info("user_name", self.user_dict['nick_name'])
        item.add_video_info("out_video_id", d_obj["id"])
        item.add_video_info("platform", self.platform)
        item.add_video_info("strategy", self.mode)
        item.add_video_info("session", "{}-{}".format(self.platform, int(time.time())))
        # 准备发往 MQ 的消息
        mq_obj = item.produce_item()
        # 筛选规则的 pipeline
        pipeline = PiaoQuanPipelineTest(
            platform=self.platform,
            mode=self.mode,
            rule_dict=self.rule_dict,
            env=self.env,
            item=mq_obj,
            trace_id=trace_id,
        )
        if not pipeline.publish_time_flag():
            self.expire_flag = True
            print("过期, 退出")
            return
        if pipeline.process_item():
            # self.mq.send_msg(mq_obj)
            print(json.dumps(mq_obj, ensure_ascii=False, indent=4))
            print("成功发送至 ETL")

    def schedule(self):
        """
        small: 只抓取小视频
        big: 只抓取视频
        both: 抓取小视频和视频， 两者之间休息 5 分钟
        """
        flag = self.user_dict['link'].split("_")[0]
        match flag:
            case "big":
                self.get_video_list()
            case "small":
                self.get_tiny_video_list()
            case "both":
                self.get_tiny_video_list()
                time.sleep(300)
                self.get_video_list()


if __name__ == "__main__":
    select_user_sql = f"""select * from crawler_user_v3 where source = 'haokanshipin';"""
    user_list = MysqlHelper.get_values("author", "haokanshipin", select_user_sql, "prod", action="")
    # print(json.dumps(user_list[1], ensure_ascii=False, indent=4))
    T = HaoKanVideoAccount(
        platform="haokanshipin",
        mode="author",
        rule_dict={"period": {"min": 10, "max": 10}},
        user_dict=user_list[1],
        env="prod",
    )
    print(user_list[1])
    T.schedule()