import os import re import base64 import json import random import sys import time import uuid import requests sys.path.append(os.getcwd()) from common.video_item import VideoItem from common.pipeline import PiaoQuanPipelineTest from common.mq import MQ from common.scheduling_db import MysqlHelper def tunnel_proxies(): # 隧道域名:端口号 tunnel = "q796.kdltps.com:15818" # 用户名密码方式 username = "t17772369458618" password = "5zqcjkmy" proxies = { "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}, "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel} } return proxies class HaoKanVideoAccount(object): def __init__(self, platform, mode, rule_dict, user_dict, env): self.account_id = user_dict["link"].split("/")[-1] self.platform = platform self.mode = mode self.rule_dict = rule_dict self.user_dict = user_dict self.env = env self.download_cnt = 0 self.mq = MQ(topic_name="topic_crawler_etl_" + self.env) self.url_key = "guanghui456" self.expire_flag = False def decrypt(self, ori_str): # base64解码 base64_bytes = base64.b64decode(ori_str) crypt_str = base64_bytes.decode("utf-8") # 异或解密 n = "" for i in range(len(crypt_str)): o = ord(crypt_str[i]) s = ord(self.url_key[i % len(self.url_key)]) n += chr(o ^ s) return n def get_video_list(self): page_limit = 10 url = "https://haokan.baidu.com/web/author/listall" headers = { "Accept": "*/*", "Accept-Language": "en,zh-CN;q=0.9,zh;q=0.8", "Cache-Control": "no-cache", "Connection": "keep-alive", "Content-Type": "application/x-www-form-urlencoded", "Cookie": 'BIDUPSID=504D4A3A8D0584CA8C3BE27ACFED5323; PSTM=1695297510; BAIDUID=504D4A3A8D0584CA7398158209FA507F:FG=1; BAIDUID_BFESS=504D4A3A8D0584CA7398158209FA507F:FG=1; H_WISE_SIDS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; H_WISE_SIDS_BFESS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%7D; H_PS_PSSID=39624_39663_39684_39690_39676_39678_39713; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; hkpcSearch=%u674E%u6709%u8D22; BA_HECTOR=0h240k018l2ka0a000a00g0g1il6sn21q; ZFY=PnfJdvQrpMNOIoZw1LsOKMDW8:BWcrmDvtL60fANYxmQ:C; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1699950106,1700038092; PC_TAB_LOG=video_details_page; COMMON_LID=c414c76cb1b11dfb04b7062a5ae09ce2; ab_sr=1.0.1_ZGEwNWY3OTFhZmNhNzZiN2YyZjljYjI3ZmE2YzRiZDA5NjY3YTI0OTdiMWM0ZWZhMjU3YWMzMThmZjU5YzQ5MDM0NDA4N2FmZWM5M2I0ZmQzZmE0ZWI2MjJkMjE0MDQ1ZTIyYmU5OGQ3OWY3ZDNlYTI2NmFiMGZlYTliODFmZmJiY2U0MDRmMzkwZWMzOTQxMTc0MDU3NmQ2ZWVlNjM3ZA==; reptileData=%7B%22data%22%3A%22636c55e0319da5169a60acec4a264a35c10862f8abfe2f2cc32c55eb6b0ab4de0efdfa115ea522d6d4d361dea07feae27710e59370b70671d347daddec662182e4c514cbcba8dae1736cf735f29e19b9a42be523d0836d04fa3b15dd57de52f4%22%2C%22key_id%22%3A%2230%22%2C%22sign%22%3A%22551bc5f4%22%7D; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1700038278; RT="z=1&dm=baidu.com&si=bd56d547-8a2c-433d-9eee-70f64ccf646a&ss=lozishrc&sl=9&tt=c2s&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=6z26"', "Referer": "https://haokan.baidu.com/author/1774716548084013", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", } p_flag = "" while True: params = { "app_id": self.account_id, # 账号id "ctime": p_flag, # 翻页指示，用时间戳表示，为空表示从头开始 "rn": page_limit, # 每一页的页数，默认是 10 "_api": 1, } response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies()) result = response.json() p_flag = result['data']['ctime'] self.process_video_list(result) if self.expire_flag: return if result['data']['has_more'] == 0: return def get_tiny_video_list(self): url = "https://haokan.baidu.com/web/author/listall" headers = { "Accept": "*/*", "Accept-Language": "en,zh-CN;q=0.9,zh;q=0.8", "Cache-Control": "no-cache", "Connection": "keep-alive", "Content-Type": "application/x-www-form-urlencoded", "Cookie": 'BIDUPSID=504D4A3A8D0584CA8C3BE27ACFED5323; PSTM=1695297510; BAIDUID=504D4A3A8D0584CA7398158209FA507F:FG=1; BAIDUID_BFESS=504D4A3A8D0584CA7398158209FA507F:FG=1; H_WISE_SIDS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; H_WISE_SIDS_BFESS=213352_214793_110085_244721_236312_265883_265985_269905_271172_270102_234295_234207_272282_263618_272473_260335_273141_273244_273397_273481_275098_275007_275853_276196_275170_271562_253022_275870_277354_251972_277631_277642_277635_277611_275732_276665_275209_277554_259642_278057_278166_278163_278300_274784_275167_278263_272560_278573_278575_277542_278790_278388_256739_278920_279021_279045_278237_279267_276573_279367_279385_278392_274947_276269_278946_279086_279610_279605_279680_276983_279877_279307_279695_279945_279703_279975_279998_278249_278213_280132_280209_277699_280161_280227_274286_280405_280368_278674_280485_280541_270366_278414_276929_275856_280614_256223_280488_280636_276438_280560_277759_279896_280768_280809_279850_280771_280107_280583; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218b98064c148fe-08a95b8d70a5fe-17525634-1901520-18b98064c151479%22%7D; H_PS_PSSID=39624_39663_39684_39690_39676_39678_39713; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; hkpcSearch=%u674E%u6709%u8D22; BA_HECTOR=0h240k018l2ka0a000a00g0g1il6sn21q; ZFY=PnfJdvQrpMNOIoZw1LsOKMDW8:BWcrmDvtL60fANYxmQ:C; Hm_lvt_4aadd610dfd2f5972f1efee2653a2bc5=1699950106,1700038092; PC_TAB_LOG=video_details_page; COMMON_LID=c414c76cb1b11dfb04b7062a5ae09ce2; BDRCVFR[X_XKQks0S63]=mk3SLVN4HKm; ab_sr=1.0.1_ODM2M2M4NzY2MWE3Yjg0MGQ4NDk2YTQ4ZTRlMWVlYjdiY2JmMzQ5ZjU1MjAxODUxZTQ0NTg4YjJjNzNmYTU2MzZiMjI2Y2EwZTU2OGIwYjdmMDc3NmRhMGJjODZkZmIyN2U0YWFjMjUzZWEwNTRlZWQ1N2U0MzkxY2YwMzk4Zjk1N2NiYWM1OGZlN2M0NWU4ZWJiZjFmNWE5YWU3YjFmMA==; reptileData=%7B%22data%22%3A%22636c55e0319da5169a60acec4a264a35c10862f8abfe2f2cc32c55eb6b0ab4de0efdfa115ea522d6d4d361dea07feae27710e59370b70671d347daddec6621825f6963cc8f86c5fee1d0664c82edf0ae5c838cf8bde5188e6a7757f1998a7c48%22%2C%22key_id%22%3A%2230%22%2C%22sign%22%3A%22f302bd06%22%7D; Hm_lpvt_4aadd610dfd2f5972f1efee2653a2bc5=1700046227; RT="z=1&dm=baidu.com&si=bd56d547-8a2c-433d-9eee-70f64ccf646a&ss=loznmnju&sl=1&tt=1w3&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=2oq"', "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", } cursor = "" while True: params = { "app_id": self.account_id, "ctime": cursor, "video_type": "haokan|tabhubVideo", "rn": 20, } response = requests.request("GET", url, headers=headers, params=params, proxies=tunnel_proxies()) result = response.json() cursor = result['data']['ctime'] self.process_video_list(result) if self.expire_flag: return if result['data']['has_more'] == 0: print("No more videos") return def process_video_list(self, result): if result["errmsg"] == "成功": video_list = result["data"]["results"] for index, video_obj in enumerate(video_list): # self.get_video_info(video_obj) try: if self.expire_flag: return else: print("扫描到一条视频") self.get_video_info(video_obj) except Exception as e: print("抓取单条视频异常，报错是:{}".format(e)) else: print("请求视频链接列表失败，需要 review 代码验证") def get_video_info(self, video_obj): url = "https://haokan.baidu.com/v?vid={}&collection_id=".format( video_obj["content"]["vid"] ) header = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" } response_text = requests.get(url, headers=header).text encrypted_data = json.loads( re.search(r"window\.__PRELOADED_STATE__\s*=\s*(.*?);", response_text) .group(1) .strip() ) encrypted_video_obj = encrypted_data["encrptedVideoMeta"] d_obj = json.loads(self.decrypt(encrypted_video_obj)) trace_id = self.platform + str(uuid.uuid1()) item = VideoItem() item.add_video_info("video_id", d_obj["id"]) item.add_video_info("video_title", d_obj["title"]) item.add_video_info("play_cnt", d_obj["playcnt"]) item.add_video_info("publish_time_stamp", d_obj["publish_time"]) item.add_video_info("duration", d_obj["duration"]) item.add_video_info("video_url", d_obj["playurl"]) item.add_video_info("like_cnt", d_obj["like"]) item.add_video_info("comment_cnt", d_obj["comment"]) item.add_video_info("cover_url", video_obj["content"]["cover_src"]) item.add_video_info("user_id", self.user_dict['uid']) item.add_video_info("user_name", self.user_dict['nick_name']) item.add_video_info("out_video_id", d_obj["id"]) item.add_video_info("platform", self.platform) item.add_video_info("strategy", self.mode) item.add_video_info("session", "{}-{}".format(self.platform, int(time.time()))) # 准备发往 MQ 的消息 mq_obj = item.produce_item() # 筛选规则的 pipeline pipeline = PiaoQuanPipelineTest( platform=self.platform, mode=self.mode, rule_dict=self.rule_dict, env=self.env, item=mq_obj, trace_id=trace_id, ) if not pipeline.publish_time_flag(): self.expire_flag = True print("过期, 退出") return if pipeline.process_item(): # self.mq.send_msg(mq_obj) print(json.dumps(mq_obj, ensure_ascii=False, indent=4)) print("成功发送至 ETL") def schedule(self): """ small: 只抓取小视频 big: 只抓取视频 both: 抓取小视频和视频， 两者之间休息 5 分钟 """ flag = self.user_dict['link'].split("_")[0] match flag: case "big": self.get_video_list() case "small": self.get_tiny_video_list() case "both": self.get_tiny_video_list() time.sleep(300) self.get_video_list() if __name__ == "__main__": select_user_sql = f"""select * from crawler_user_v3 where source = 'haokanshipin';""" user_list = MysqlHelper.get_values("author", "haokanshipin", select_user_sql, "prod", action="") # print(json.dumps(user_list[1], ensure_ascii=False, indent=4)) T = HaoKanVideoAccount( platform="haokanshipin", mode="author", rule_dict={"period": {"min": 10, "max": 10}}, user_dict=user_list[1], env="prod", ) print(user_list[1]) T.schedule()