gzh_spider_scrapy/gzh_spider/xigua_video/XiguaRecommend.py

import random
import string
import urllib3
import requests
import re
from requests.adapters import HTTPAdapter
import base64

"""
西瓜推荐的方法
"""


def tunnel_proxies():
    # 隧道域名:端口号
    tunnel = "q796.kdltps.com:15818"

    # 用户名密码方式
    username = "t17772369458618"
    password = "5zqcjkmy"
    tunnel_proxies = {
        "http": "http://%(user)s:%(pwd)s@%(proxy)s/"
                % {"user": username, "pwd": password, "proxy": tunnel},
        "https": "http://%(user)s:%(pwd)s@%(proxy)s/"
                 % {"user": username, "pwd": password, "proxy": tunnel},
    }

    # 白名单方式（需提前设置白名单）
    # proxies = {
    #     "http": "http://%(proxy)s/" % {"proxy": tunnel},
    #     "https": "http://%(proxy)s/" % {"proxy": tunnel}
    # }

    # 要访问的目标网页
    # target_url = "https://www.kuaishou.com/profile/3xk9tkk6kkwkf7g"
    # target_url = "https://dev.kdlapi.com/testproxy"

    # # 使用隧道域名发送请求
    # response = requests.get(target_url, proxies=proxies)
    # print(response.text)

    return tunnel_proxies  # {'http': 'http://t17772369458618:5zqcjkmy@q796.kdltps.com:15818/', 'https': 'http://t17772369458618:5zqcjkmy@q796.kdltps.com:15818/'}


def random_signature():
    """
    # 生成随机签名
    :return: str
    """
    src_digits = string.digits  # string_数字
    src_uppercase = string.ascii_uppercase  # string_大写字母
    src_lowercase = string.ascii_lowercase  # string_小写字母
    digits_num = random.randint(1, 6)
    uppercase_num = random.randint(1, 26 - digits_num - 1)
    lowercase_num = 26 - (digits_num + uppercase_num)
    password = (
            random.sample(src_digits, digits_num)
            + random.sample(src_uppercase, uppercase_num)
            + random.sample(src_lowercase, lowercase_num)
    )
    random.shuffle(password)
    new_password = "AAAAAAAAAA" + "".join(password)[10:-4] + "AAAB"
    new_password_start = new_password[0:18]
    new_password_end = new_password[-7:]
    if new_password[18] == "8":
        new_password = new_password_start + "w" + new_password_end
    elif new_password[18] == "9":
        new_password = new_password_start + "x" + new_password_end
    elif new_password[18] == "-":
        new_password = new_password_start + "y" + new_password_end
    elif new_password[18] == ".":
        new_password = new_password_start + "z" + new_password_end
    else:
        new_password = new_password_start + "y" + new_password_end
    return new_password


def get_video_url(video_info):
    video_url_dict = {}

    video_resource = video_info.get("videoResource", {})
    dash_120fps = video_resource.get("dash_120fps", {})
    normal = video_resource.get("normal", {})

    # 从dash_120fps和normal字典中获取video_list字典
    video_list = dash_120fps.get("video_list", {}) or normal.get("video_list", {})
    # 获取video_list字典中的video_4、video_3、video_2或video_1的值。如果找到非空视频URL，则将其赋值给变量video_url。否则，将赋值为空字符串。
    video = (
            video_list.get("video_4")
            or video_list.get("video_3")
            or video_list.get("video_2")
            or video_list.get("video_1")
    )

    video_url = video.get("backup_url_1", "") if video else ""
    audio_url = video.get("backup_url_1", "") if video else ""
    video_width = video.get("vwidth", 0) if video else 0
    video_height = video.get("vheight", 0) if video else 0

    video_url = re.sub(r"[^a-zA-Z0-9+/=]", "", video_url)  # 从视频URL中删除特殊字符
    audio_url = re.sub(r"[^a-zA-Z0-9+/=]", "", audio_url)  # 从音频URL中删除特殊字符

    video_url = base64.b64decode(video_url).decode("utf8")  # 解码视频URL
    audio_url = base64.b64decode(audio_url).decode("utf8")  # 解码音频URL

    video_url_dict["video_url"] = video_url
    video_url_dict["audio_url"] = audio_url
    video_url_dict["video_width"] = video_width
    video_url_dict["video_height"] = video_height

    return video_url_dict


class XiguaRecommend:
    def __init__(self):
        self.platform = "xigua"

    def get_comment_cnt(self, item_id):
        """
        获取评论数量
        :param item_id:
        :return:
        """
        url = "https://www.ixigua.com/tlb/comment/article/v5/tab_comments/?"
        params = {
            "tab_index": "0",
            "count": "10",
            "offset": "10",
            "group_id": str(item_id),
            "item_id": str(item_id),
            "aid": "1768",
            "msToken": "50-JJObWB07HfHs-BMJWT1eIDX3G-6lPSF_i-QwxBIXE9VVa-iN0jbEXR5pG2DKjXBmP299n6ZTuXzY-GAy968CCvouSAYIS4GzvGQT3pNlKNejr5G4-1g==",
            "X-Bogus": "DFSzswVOyGtANVeWtCLMqR/F6q9U",
            "_signature": random_signature(),
        }
        headers = {
            "authority": "www.ixigua.com",
            "accept": "application/json, text/plain, */*",
            "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
            "cache-control": "no-cache",
            "cookie": "MONITOR_WEB_ID=67cb5099-a022-4ec3-bb8e-c4de6ba51dd0; passport_csrf_token=72b2574f3c99f8ba670e42df430218fd; passport_csrf_token_default=72b2574f3c99f8ba670e42df430218fd; sid_guard=c7472b508ea631823ba765a60cf8757f%7C1680867422%7C3024002%7CFri%2C+12-May-2023+11%3A37%3A04+GMT; uid_tt=c13f47d51767f616befe32fb3e9f485a; uid_tt_ss=c13f47d51767f616befe32fb3e9f485a; sid_tt=c7472b508ea631823ba765a60cf8757f; sessionid=c7472b508ea631823ba765a60cf8757f; sessionid_ss=c7472b508ea631823ba765a60cf8757f; sid_ucp_v1=1.0.0-KGUzNWYxNmRkZGJiZjgxY2MzZWNkMTEzMTkwYjY1Yjg5OTY5NzVlNmMKFQiu3d-eqQIQ3oDAoQYYGCAMOAhACxoCaGwiIGM3NDcyYjUwOGVhNjMxODIzYmE3NjVhNjBjZjg3NTdm; ssid_ucp_v1=1.0.0-KGUzNWYxNmRkZGJiZjgxY2MzZWNkMTEzMTkwYjY1Yjg5OTY5NzVlNmMKFQiu3d-eqQIQ3oDAoQYYGCAMOAhACxoCaGwiIGM3NDcyYjUwOGVhNjMxODIzYmE3NjVhNjBjZjg3NTdm; odin_tt=b893608d4dde2e1e8df8cd5d97a0e2fbeafc4ca762ac72ebef6e6c97e2ed19859bb01d46b4190ddd6dd17d7f9678e1de; SEARCH_CARD_MODE=7168304743566296612_0; support_webp=true; support_avif=false; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; tt_scid=7Pux7s634-z8DYvCM20y7KigwH5u7Rh6D9C-RROpnT.aGMEcz6Vsxp.oai47wJqa4f86; ttwid=1%7CHHtv2QqpSGuSu8r-zXF1QoWsvjmNi1SJrqOrZzg-UCY%7C1683858689%7Ca5223fe1500578e01e138a0d71d6444692018296c4c24f5885af174a65873c95; ixigua-a-s=3; msToken=50-JJObWB07HfHs-BMJWT1eIDX3G-6lPSF_i-QwxBIXE9VVa-iN0jbEXR5pG2DKjXBmP299n6ZTuXzY-GAy968CCvouSAYIS4GzvGQT3pNlKNejr5G4-1g==; __ac_nonce=0645dcbf0005064517440; __ac_signature=_02B4Z6wo00f01FEGmAwAAIDBKchzCGqn-MBRJpyAAHAjieFC5GEg6gGiwz.I4PRrJl7f0GcixFrExKmgt6QI1i1S-dQyofPEj2ugWTCnmKUdJQv-wYuDofeKNe8VtMtZq2aKewyUGeKU-5Ud21; ixigua-a-s=3",
            "pragma": "no-cache",
            "referer": f"https://www.ixigua.com/{item_id}?logTag=3c5aa86a8600b9ab8540",
            "sec-ch-ua": '"Microsoft Edge";v="113", "Chromium";v="113", "Not-A.Brand";v="24"',
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": '"macOS"',
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-origin",
            "tt-anti-token": "cBITBHvmYjEygzv-f9c78c1297722cf1f559c74b084e4525ce4900bdcf9e8588f20cc7c2e3234422",
            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35",
            "x-secsdk-csrf-token": "000100000001f8e733cf37f0cd255a51aea9a81ff7bc0c09490cfe41ad827c3c5c18ec809279175e4d9f5553d8a5",
        }
        urllib3.disable_warnings()
        s = requests.session()
        # max_retries=3 重试3次
        s.mount("http://", HTTPAdapter(max_retries=3))
        s.mount("https://", HTTPAdapter(max_retries=3))
        response = s.get(
            url=url,
            headers=headers,
            params=params,
            verify=False,
            proxies=tunnel_proxies(),
            timeout=5,
        )
        response.close()
        if (
                response.status_code != 200
                or "total_number" not in response.json()
                or response.json() == {}
        ):
            return 0
        return response.json().get("total_number", 0)