automatic_crawler/spider/crawler_search/xigua_search.py

"""
西瓜视频搜索爬虫
"""
import os
import sys
import json
import time
import uuid
import random
import base64
import asyncio
import aiohttp
import urllib.parse

import requests
from lxml import etree

sys.path.append(os.getcwd())

from application.items import VideoItem
from application.pipeline import PiaoQuanPipeline
from application.common.messageQueue import MQ
from application.common.proxies import tunnel_proxies
from application.common.log import AliyunLogger


def get_video_url(video_info):
    """
    获取视频链接信息
    :param video_info:
    :return:
    """
    video_url_dict = {}
    # video_url
    if "videoResource" not in video_info:
        video_url_dict["video_url"] = ""
        video_url_dict["audio_url"] = ""
        video_url_dict["video_width"] = 0
        video_url_dict["video_height"] = 0

    elif "dash_120fps" in video_info["videoResource"]:
        if (
            "video_list" in video_info["videoResource"]["dash_120fps"]
            and "video_4" in video_info["videoResource"]["dash_120fps"]["video_list"]
        ):
            video_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_4"
            ]["backup_url_1"]
            audio_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_4"
            ]["backup_url_1"]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_4"
            ]["vwidth"]
            video_height = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_4"
            ]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["dash_120fps"]
            and "video_3" in video_info["videoResource"]["dash_120fps"]["video_list"]
        ):
            video_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_3"
            ]["backup_url_1"]
            audio_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_3"
            ]["backup_url_1"]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_3"
            ]["vwidth"]
            video_height = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_3"
            ]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["dash_120fps"]
            and "video_2" in video_info["videoResource"]["dash_120fps"]["video_list"]
        ):
            video_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_2"
            ]["backup_url_1"]
            audio_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_2"
            ]["backup_url_1"]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_2"
            ]["vwidth"]
            video_height = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_2"
            ]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["dash_120fps"]
            and "video_1" in video_info["videoResource"]["dash_120fps"]["video_list"]
        ):
            video_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_1"
            ]["backup_url_1"]
            audio_url = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_1"
            ]["backup_url_1"]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_1"
            ]["vwidth"]
            video_height = video_info["videoResource"]["dash_120fps"]["video_list"][
                "video_1"
            ]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height

        elif (
            "dynamic_video" in video_info["videoResource"]["dash_120fps"]
            and "dynamic_video_list"
            in video_info["videoResource"]["dash_120fps"]["dynamic_video"]
            and "dynamic_audio_list"
            in video_info["videoResource"]["dash_120fps"]["dynamic_video"]
            and len(
                video_info["videoResource"]["dash_120fps"]["dynamic_video"][
                    "dynamic_video_list"
                ]
            )
            != 0
            and len(
                video_info["videoResource"]["dash_120fps"]["dynamic_video"][
                    "dynamic_audio_list"
                ]
            )
            != 0
        ):
            video_url = video_info["videoResource"]["dash_120fps"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["backup_url_1"]
            audio_url = video_info["videoResource"]["dash_120fps"]["dynamic_video"][
                "dynamic_audio_list"
            ][-1]["backup_url_1"]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash_120fps"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["vwidth"]
            video_height = video_info["videoResource"]["dash_120fps"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        else:
            video_url_dict["video_url"] = ""
            video_url_dict["audio_url"] = ""
            video_url_dict["video_width"] = 0
            video_url_dict["video_height"] = 0

    elif "dash" in video_info["videoResource"]:
        if (
            "video_list" in video_info["videoResource"]["dash"]
            and "video_4" in video_info["videoResource"]["dash"]["video_list"]
        ):
            video_url = video_info["videoResource"]["dash"]["video_list"]["video_4"][
                "backup_url_1"
            ]
            audio_url = video_info["videoResource"]["dash"]["video_list"]["video_4"][
                "backup_url_1"
            ]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash"]["video_list"]["video_4"][
                "vwidth"
            ]
            video_height = video_info["videoResource"]["dash"]["video_list"]["video_4"][
                "vheight"
            ]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["dash"]
            and "video_3" in video_info["videoResource"]["dash"]["video_list"]
        ):
            video_url = video_info["videoResource"]["dash"]["video_list"]["video_3"][
                "backup_url_1"
            ]
            audio_url = video_info["videoResource"]["dash"]["video_list"]["video_3"][
                "backup_url_1"
            ]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash"]["video_list"]["video_3"][
                "vwidth"
            ]
            video_height = video_info["videoResource"]["dash"]["video_list"]["video_3"][
                "vheight"
            ]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["dash"]
            and "video_2" in video_info["videoResource"]["dash"]["video_list"]
        ):
            video_url = video_info["videoResource"]["dash"]["video_list"]["video_2"][
                "backup_url_1"
            ]
            audio_url = video_info["videoResource"]["dash"]["video_list"]["video_2"][
                "backup_url_1"
            ]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash"]["video_list"]["video_2"][
                "vwidth"
            ]
            video_height = video_info["videoResource"]["dash"]["video_list"]["video_2"][
                "vheight"
            ]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["dash"]
            and "video_1" in video_info["videoResource"]["dash"]["video_list"]
        ):
            video_url = video_info["videoResource"]["dash"]["video_list"]["video_1"][
                "backup_url_1"
            ]
            audio_url = video_info["videoResource"]["dash"]["video_list"]["video_1"][
                "backup_url_1"
            ]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash"]["video_list"]["video_1"][
                "vwidth"
            ]
            video_height = video_info["videoResource"]["dash"]["video_list"]["video_1"][
                "vheight"
            ]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height

        elif (
            "dynamic_video" in video_info["videoResource"]["dash"]
            and "dynamic_video_list"
            in video_info["videoResource"]["dash"]["dynamic_video"]
            and "dynamic_audio_list"
            in video_info["videoResource"]["dash"]["dynamic_video"]
            and len(
                video_info["videoResource"]["dash"]["dynamic_video"][
                    "dynamic_video_list"
                ]
            )
            != 0
            and len(
                video_info["videoResource"]["dash"]["dynamic_video"][
                    "dynamic_audio_list"
                ]
            )
            != 0
        ):
            video_url = video_info["videoResource"]["dash"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["backup_url_1"]
            audio_url = video_info["videoResource"]["dash"]["dynamic_video"][
                "dynamic_audio_list"
            ][-1]["backup_url_1"]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["dash"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["vwidth"]
            video_height = video_info["videoResource"]["dash"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        else:
            video_url_dict["video_url"] = ""
            video_url_dict["audio_url"] = ""
            video_url_dict["video_width"] = 0
            video_url_dict["video_height"] = 0

    elif "normal" in video_info["videoResource"]:
        if (
            "video_list" in video_info["videoResource"]["normal"]
            and "video_4" in video_info["videoResource"]["normal"]["video_list"]
        ):
            video_url = video_info["videoResource"]["normal"]["video_list"]["video_4"][
                "backup_url_1"
            ]
            audio_url = video_info["videoResource"]["normal"]["video_list"]["video_4"][
                "backup_url_1"
            ]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["normal"]["video_list"][
                "video_4"
            ]["vwidth"]
            video_height = video_info["videoResource"]["normal"]["video_list"][
                "video_4"
            ]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["normal"]
            and "video_3" in video_info["videoResource"]["normal"]["video_list"]
        ):
            video_url = video_info["videoResource"]["normal"]["video_list"]["video_3"][
                "backup_url_1"
            ]
            audio_url = video_info["videoResource"]["normal"]["video_list"]["video_3"][
                "backup_url_1"
            ]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["normal"]["video_list"][
                "video_3"
            ]["vwidth"]
            video_height = video_info["videoResource"]["normal"]["video_list"][
                "video_3"
            ]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["normal"]
            and "video_2" in video_info["videoResource"]["normal"]["video_list"]
        ):
            video_url = video_info["videoResource"]["normal"]["video_list"]["video_2"][
                "backup_url_1"
            ]
            audio_url = video_info["videoResource"]["normal"]["video_list"]["video_2"][
                "backup_url_1"
            ]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["normal"]["video_list"][
                "video_2"
            ]["vwidth"]
            video_height = video_info["videoResource"]["normal"]["video_list"][
                "video_2"
            ]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        elif (
            "video_list" in video_info["videoResource"]["normal"]
            and "video_1" in video_info["videoResource"]["normal"]["video_list"]
        ):
            video_url = video_info["videoResource"]["normal"]["video_list"]["video_1"][
                "backup_url_1"
            ]
            audio_url = video_info["videoResource"]["normal"]["video_list"]["video_1"][
                "backup_url_1"
            ]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["normal"]["video_list"][
                "video_1"
            ]["vwidth"]
            video_height = video_info["videoResource"]["normal"]["video_list"][
                "video_1"
            ]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height

        elif (
            "dynamic_video" in video_info["videoResource"]["normal"]
            and "dynamic_video_list"
            in video_info["videoResource"]["normal"]["dynamic_video"]
            and "dynamic_audio_list"
            in video_info["videoResource"]["normal"]["dynamic_video"]
            and len(
                video_info["videoResource"]["normal"]["dynamic_video"][
                    "dynamic_video_list"
                ]
            )
            != 0
            and len(
                video_info["videoResource"]["normal"]["dynamic_video"][
                    "dynamic_audio_list"
                ]
            )
            != 0
        ):
            video_url = video_info["videoResource"]["normal"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["backup_url_1"]
            audio_url = video_info["videoResource"]["normal"]["dynamic_video"][
                "dynamic_audio_list"
            ][-1]["backup_url_1"]
            if len(video_url) % 3 == 1:
                video_url += "=="
            elif len(video_url) % 3 == 2:
                video_url += "="
            elif len(audio_url) % 3 == 1:
                audio_url += "=="
            elif len(audio_url) % 3 == 2:
                audio_url += "="
            video_url = base64.b64decode(video_url).decode("utf8")
            audio_url = base64.b64decode(audio_url).decode("utf8")
            video_width = video_info["videoResource"]["normal"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["vwidth"]
            video_height = video_info["videoResource"]["normal"]["dynamic_video"][
                "dynamic_video_list"
            ][-1]["vheight"]
            video_url_dict["video_url"] = video_url
            video_url_dict["audio_url"] = audio_url
            video_url_dict["video_width"] = video_width
            video_url_dict["video_height"] = video_height
        else:
            video_url_dict["video_url"] = ""
            video_url_dict["audio_url"] = ""
            video_url_dict["video_width"] = 0
            video_url_dict["video_height"] = 0

    else:
        video_url_dict["video_url"] = ""
        video_url_dict["audio_url"] = ""
        video_url_dict["video_width"] = 0
        video_url_dict["video_height"] = 0

    return video_url_dict


class XiGuaSearch(object):
    """
    XiGuaSearch
    """

    def __init__(self, platform, mode, rule_dict, user_list, env="prod"):
        self.platform = platform
        self.mode = mode
        self.rule_dict = rule_dict
        self.user_list = user_list
        self.env = env
        self.download_cnt = 0
        self.mq = MQ(topic_name="topic_crawler_etl_" + self.env)
        self.expire_flag = False
        self.aliyun_log = AliyunLogger(platform=self.platform, mode=self.mode)

    async def search(self, keyword):
        """搜索"""
        keyword = urllib.parse.quote(keyword)
        base_url = "https://www.ixigua.com/search/{}/ab_name=search&fss=input".format(
            keyword
        )
        headers = {
            "authority": "www.ixigua.com",
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "accept-language": "zh,en;q=0.9,zh-CN;q=0.8",
            "cache-control": "max-age=0",
            "cookie": "ixigua-a-s=1; support_webp=true; support_avif=true; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; tt_scid=Ur23fgYD2pMJOvi1BpILyfaobg8wA7IhGwmQx260ULRa8Dvjaxc5ZA63BUIP-6Vi473f; ttwid=1%7CNtTtSp4Iej-v0nWtepdZH3d3Ts6uGNMFzTN20ps1cdo%7C1708236945%7Cc1f301c64aa3bf69cdaa41f28856e2bb7b7eed16583f8c92d50cffa2d9944fc6; msToken=rr418opQf04vm8n9s8FAGdr1AoCUsvAOGKSDPbBEfwVS1sznxxZCvcZTI93qVz5uAXlX9yRwcKlNQZ4wMro2DmlHw5yWHAVeKr_SzgO1KtVVnjUMTUNEux_cq1-EIkI=",
            "sec-ch-ua": '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": '"macOS"',
            "sec-fetch-dest": "document",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "none",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
        }
        basic_response = requests.get(url=base_url, headers=headers)
        html = etree.HTML(basic_response.text)
        result = html.xpath(
            '//a[@class="HorizontalFeedCard__coverWrapper disableZoomAnimation"]/@href'
        )
        print(result)
        async with aiohttp.ClientSession() as session:
            tasks = [self.get_video_info(session, page_id[1:-2]) for page_id in result]
            await asyncio.gather(*tasks)

    async def get_video_info(self, session, page_id):
        """
        :param session:
        :param page_id: 视频主页 id
        :return:
        """
        url = "https://www.ixigua.com/api/mixVideo/information?"
        headers = {
            "accept-encoding": "gzip, deflate",
            "accept-language": "zh-CN,zh-Hans;q=0.9",
            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
            "referer": "https://www.ixigua.com/7102614741050196520?logTag=0531c88ac04f38ab2c62",
        }
        params = {
            "mixId": str(page_id),
            "msToken": "IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfC"
            "NVVIOBNjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA",
            "X-Bogus": "DFSzswVupYTANCJOSBk0P53WxM-r",
            "_signature": "_02B4Z6wo0000119LvEwAAIDCuktNZ0y5wkdfS7jAALThuOR8D9yWNZ.EmWHKV0WSn6Px"
            "fPsH9-BldyxVje0f49ryXgmn7Tzk-swEHNb15TiGqa6YF.cX0jW8Eds1TtJOIZyfc9s5emH7gdWN94",
        }
        cookies = {
            "ixigua-a-s": "1",
            "msToken": "IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfCNVVIOB"
            "NjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA",
            "ttwid": "1%7C_yXQeHWwLZgCsgHClOwTCdYSOt_MjdOkgnPIkpi-Sr8%7C1661241238%7Cf57d0c5ef3f1d7"
            "6e049fccdca1ac54887c34d1f8731c8e51a49780ff0ceab9f8",
            "tt_scid": "QZ4l8KXDG0YAEaMCSbADdcybdKbUfG4BC6S4OBv9lpRS5VyqYLX2bIR8CTeZeGHR9ee3",
            "MONITOR_WEB_ID": "0a49204a-7af5-4e96-95f0-f4bafb7450ad",
            "__ac_nonce": "06304878000964fdad287",
            "__ac_signature": "_02B4Z6wo00f017Rcr3AAAIDCUVxeW1tOKEu0fKvAAI4cvoYzV-wBhq7B6D8k0no7lb"
            "FlvYoinmtK6UXjRIYPXnahUlFTvmWVtb77jsMkKAXzAEsLE56m36RlvL7ky.M3Xn52r9t1IEb7IR3ke8",
            "ttcid": "e56fabf6e85d4adf9e4d91902496a0e882",
            "_tea_utm_cache_1300": "undefined",
            "support_avif": "false",
            "support_webp": "false",
            "xiguavideopcwebid": "7134967546256016900",
            "xiguavideopcwebid.sig": "xxRww5R1VEMJN_dQepHorEu_eAc",
        }
        async with session.get(
            url, headers=headers, params=params, cookies=cookies
        ) as response:
            video_info = await response.json()
            video_info = (
                video_info["data"]
                .get("gidInformation", {})
                .get("packerData", {})
                .get("video", {})
            )
            # print(video_info)
            item = VideoItem()
            item.add_video_info("video_title", video_info.get("title", ""))
            item.add_video_info(
                "video_id", video_info.get("videoResource", {}).get("vid", "")
            )
            item.add_video_info("play_cnt", int(video_info.get("video_watch_count", 0)))
            item.add_video_info("like_cnt", int(video_info.get("video_like_count", 0)))
            item.add_video_info("duration", int(video_info.get("video_duration", 0)))
            item.add_video_info(
                "publish_time_stamp", int(video_info.get("video_publish_time", 0))
            )
            item.add_video_info(
                "publish_time_str",
                time.strftime(
                    "%Y-%m-%d %H:%M:%S",
                    time.localtime(int(video_info.get("video_publish_time", 0))),
                ),
            )
            item.add_video_info(
                "user_name", video_info.get("user_info", {}).get("name", "")
            )
            item.add_video_info(
                "user_id", str(video_info.get("user_info", {}).get("user_id", ""))
            )
            item.add_video_info(
                "avatar_url", str(video_info.get("user_info", {}).get("avatar_url", ""))
            )
            item.add_video_info("cover_url", video_info.get("poster_url", ""))
            item.add_video_info("audio_url", get_video_url(video_info)["audio_url"])
            item.add_video_info("video_url", get_video_url(video_info)["video_url"])
            item.add_video_info("session", "xigua-search-{}".format(int(time.time())))
            item.add_video_info("out_video_id",  video_info.get("videoResource", {}).get("vid", ""))
            item.add_video_info("platform", self.platform)
            item.add_video_info("strategy", self.mode)
            # item.add_video_info("")
            mq_obj = item.produce_item()
            # print(mq_obj)
            print(json.dumps(mq_obj, ensure_ascii=False, indent=4))


if __name__ == "__main__":
    S = XiGuaSearch(platform=1, mode=2, rule_dict=3, user_list=1)
    loop = asyncio.get_event_loop()
    loop.run_until_complete(S.search("春节"))
    # await