piaoquan_crawler/xigua/xigua_follow/xigua_follow.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/2/17
import json

from lxml import etree
import base64
import os
import random
import shutil
import string
import sys
import time
import requests
import urllib3

from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from seleniumwire import webdriver

sys.path.append(os.getcwd())
from common.db import MysqlHelper
from common.users import Users
from common.common import Common
from common.feishu import Feishu
from common.publish import Publish
proxies = {"http": None, "https": None}


class Follow:
    # 个人主页视频翻页参数
    offset = 0

    platform = "西瓜视频"
    tag = "西瓜视频爬虫,定向爬虫策略"

    # 下载规则
    @staticmethod
    def download_rule(duration, width, height):
        if int(duration) >= 60:
            if int(width) >= 720 or int(height) >= 720:
                return True
            else:
                return False
        else:
            return False

    # 过滤词库
    @classmethod
    def filter_words(cls, log_type, crawler):
        try:
            filter_words_sheet = Feishu.get_values_batch(log_type, crawler, 'KGB4Hc')
            filter_words_list = []
            for x in filter_words_sheet:
                for y in x:
                    if y is None:
                        pass
                    else:
                        filter_words_list.append(y)
            return filter_words_list
        except Exception as e:
            Common.logger(log_type, crawler).error(f'filter_words异常:{e}\n')

    @classmethod
    def get_out_user_info(cls, log_type, crawler, out_uid):
        try:
            headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.41',
                       'referer': f'https://www.ixigua.com/home/{out_uid}',
                       'Cookie': f'ixigua-a-s=1; support_webp=true; support_avif=false; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; __ac_signature={cls.random_signature()}; MONITOR_WEB_ID=67cb5099-a022-4ec3-bb8e-c4de6ba51dd0; s_v_web_id=verify_lef4i99x_32SosrdH_Qrtk_4LJn_8S7q_fhu16xe3s8ZV; tt_scid=QLJjPuHf6wxVqu6IIq6gHiJXQpVrCwrdhjH2zpm7-E3ZniE1RXBcP6M8b41FJOdo41e1; ttwid=1%7CHHtv2QqpSGuSu8r-zXF1QoWsvjmNi1SJrqOrZzg-UCY%7C1677047013%7C5866a444e5ae10a9df8c11551db75010fb77b657f214ccf84e503fae8d313d09; msToken=PerXJcDdIsZ6zXkGITsftXX4mDaVaW21GuqtzSVdctH46oXXT2GcELIs9f0XW2hunRzP6KVHLZaYElRvNYflLKUXih7lC27XKxs3HjdZiXPK9NQaoKbLfA==; ixigua-a-s=1',}
            url = f"https://www.ixigua.com/home/{out_uid}"

            response = requests.get(url=url, headers=headers, proxies=proxies).text
            html = etree.HTML(response)
            out_follow_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[1]/span')[0].text.encode('raw_unicode_escape').decode()
            out_fans_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[2]/span')[0].text.encode('raw_unicode_escape').decode()
            out_like_str = html.xpath('//div[@class="userDetailV3__header__detail2"]/*[3]/span')[0].text.encode('raw_unicode_escape').decode()
            out_avatar_url = f"""https:{html.xpath('//span[@class="component-avatar__inner"]//img/@src')[0]}"""
            if "万" in out_follow_str:
                out_follow = int(float(out_follow_str.split("万")[0])*10000)
            else:
                out_follow = int(out_follow_str.replace(",", ""))
            if "万" in out_fans_str:
                out_fans = int(float(out_fans_str.split("万")[0])*10000)
            else:
                out_fans = int(out_fans_str.replace(",", ""))
            if "万" in out_like_str:
                out_like = int(float(out_like_str.split("万")[0])*10000)
            else:
                out_like = int(out_like_str.replace(",", ""))
            out_user_dict = {
                "out_follow": out_follow,
                "out_fans": out_fans,
                "out_like": out_like,
                "out_avatar_url": out_avatar_url,
            }
            # for k, v in out_user_dict.items():
            #     print(f"{k}:{v}")
            return out_user_dict
        except Exception as e:
            Common.logger(log_type, crawler).error(f"get_out_user_info:{e}\n")


    # 获取用户信息（字典格式）. 注意：部分 user_id 字符类型是 int / str
    @classmethod
    def get_user_list(cls, log_type, crawler, sheetid, env, machine):
        try:
            user_sheet = Feishu.get_values_batch(log_type, crawler, sheetid)
            our_user_list = []
            for i in range(1, len(user_sheet)):
                out_uid = user_sheet[i][2]
                user_name = user_sheet[i][3]
                our_uid = user_sheet[i][6]
                our_user_link = user_sheet[i][7]
                if out_uid is None or user_name is None:
                    Common.logger(log_type, crawler).info("空行\n")
                else:
                    Common.logger(log_type, crawler).info(f"正在更新 {user_name} 用户信息\n")
                    if our_uid is None:
                        out_user_info = cls.get_out_user_info(log_type, crawler, out_uid)
                        out_user_dict = {
                            "out_uid": out_uid,
                            "user_name": user_name,
                            "out_avatar_url": out_user_info["out_avatar_url"],
                            "out_create_time": '',
                            "out_tag": '',
                            "out_play_cnt": 0,
                            "out_fans": out_user_info["out_fans"],
                            "out_follow": out_user_info["out_follow"],
                            "out_friend": 0,
                            "out_like": out_user_info["out_like"],
                            "platform": cls.platform,
                            "tag": cls.tag,
                        }
                        our_user_dict = Users.create_user(log_type=log_type, crawler=crawler, out_user_dict=out_user_dict, env=env, machine=machine)
                        our_uid = our_user_dict['our_uid']
                        our_user_link = our_user_dict['our_user_link']
                        Feishu.update_values(log_type, crawler, sheetid, f'G{i + 1}:H{i + 1}', [[our_uid, our_user_link]])
                        Common.logger(log_type, crawler).info(f'站内用户信息写入飞书成功！\n')
                        our_user_list.append(our_user_dict)
                    else:
                        our_user_dict = {
                            'out_uid': out_uid,
                            'user_name': user_name,
                            'our_uid': our_uid,
                            'our_user_link': our_user_link,
                        }
                        our_user_list.append(our_user_dict)
            return our_user_list
        except Exception as e:
            Common.logger(log_type, crawler).error(f'get_user_id_from_feishu异常:{e}\n')

    @classmethod
    def random_signature(cls):
        src_digits = string.digits  # string_数字
        src_uppercase = string.ascii_uppercase  # string_大写字母
        src_lowercase = string.ascii_lowercase  # string_小写字母
        digits_num = random.randint(1, 6)
        uppercase_num = random.randint(1, 26 - digits_num - 1)
        lowercase_num = 26 - (digits_num + uppercase_num)
        password = random.sample(src_digits, digits_num) + random.sample(src_uppercase, uppercase_num) + random.sample(
            src_lowercase, lowercase_num)
        random.shuffle(password)
        new_password = 'AAAAAAAAAA' + ''.join(password)[10:-4] + 'AAAB'
        new_password_start = new_password[0:18]
        new_password_end = new_password[-7:]
        if new_password[18] == '8':
            new_password = new_password_start + 'w' + new_password_end
        elif new_password[18] == '9':
            new_password = new_password_start + 'x' + new_password_end
        elif new_password[18] == '-':
            new_password = new_password_start + 'y' + new_password_end
        elif new_password[18] == '.':
            new_password = new_password_start + 'z' + new_password_end
        else:
            new_password = new_password_start + 'y' + new_password_end
        return new_password

    @classmethod
    def get_signature(cls, log_type, crawler, out_uid, machine):
        try:
            # 打印请求配置
            ca = DesiredCapabilities.CHROME
            ca["goog:loggingPrefs"] = {"performance": "ALL"}

            # 不打开浏览器运行
            chrome_options = webdriver.ChromeOptions()
            chrome_options.add_argument("--headless")
            chrome_options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36')
            chrome_options.add_argument("--no-sandbox")

            # driver初始化
            if machine == 'aliyun' or machine == 'aliyun_hk':
                driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options)
            elif machine == 'macpro':
                driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options,
                                          service=Service('/Users/lieyunye/Downloads/chromedriver_v86/chromedriver'))
            elif machine == 'macair':
                driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options,
                                          service=Service('/Users/piaoquan/Downloads/chromedriver'))
            else:
                driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options, service=Service('/Users/wangkun/Downloads/chromedriver/chromedriver_v110/chromedriver'))
            driver.implicitly_wait(10)
            driver.get(f'https://www.ixigua.com/home/{out_uid}/')
            time.sleep(3)
            data_src = driver.find_elements(By.XPATH, '//img[@class="tt-img BU-MagicImage tt-img-loaded"]')[1].get_attribute("data-src")
            signature = data_src.split("x-signature=")[-1]
            return signature
        except Exception as e:
            Common.logger(log_type, crawler).error(f'get_signature异常:{e}\n')

    # 获取视频详情
    @classmethod
    def get_video_url(cls, log_type, crawler, gid):
        try:
            url = 'https://www.ixigua.com/api/mixVideo/information?'
            headers = {
                "accept-encoding": "gzip, deflate",
                "accept-language": "zh-CN,zh-Hans;q=0.9",
                "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                              "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
                "referer": "https://www.ixigua.com/7102614741050196520?logTag=0531c88ac04f38ab2c62",
            }
            params = {
                'mixId': gid,
                'msToken': 'IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfC'
                           'NVVIOBNjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA',
                'X-Bogus': 'DFSzswVupYTANCJOSBk0P53WxM-r',
                '_signature': '_02B4Z6wo0000119LvEwAAIDCuktNZ0y5wkdfS7jAALThuOR8D9yWNZ.EmWHKV0WSn6Px'
                              'fPsH9-BldyxVje0f49ryXgmn7Tzk-swEHNb15TiGqa6YF.cX0jW8Eds1TtJOIZyfc9s5emH7gdWN94',
            }
            cookies = {
                'ixigua-a-s': '1',
                'msToken': 'IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfCNVVIOB'
                           'NjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA',
                'ttwid': '1%7C_yXQeHWwLZgCsgHClOwTCdYSOt_MjdOkgnPIkpi-Sr8%7C1661241238%7Cf57d0c5ef3f1d7'
                         '6e049fccdca1ac54887c34d1f8731c8e51a49780ff0ceab9f8',
                'tt_scid': 'QZ4l8KXDG0YAEaMCSbADdcybdKbUfG4BC6S4OBv9lpRS5VyqYLX2bIR8CTeZeGHR9ee3',
                'MONITOR_WEB_ID': '0a49204a-7af5-4e96-95f0-f4bafb7450ad',
                '__ac_nonce': '06304878000964fdad287',
                '__ac_signature': '_02B4Z6wo00f017Rcr3AAAIDCUVxeW1tOKEu0fKvAAI4cvoYzV-wBhq7B6D8k0no7lb'
                                  'FlvYoinmtK6UXjRIYPXnahUlFTvmWVtb77jsMkKAXzAEsLE56m36RlvL7ky.M3Xn52r9t1IEb7IR3ke8',
                'ttcid': 'e56fabf6e85d4adf9e4d91902496a0e882',
                '_tea_utm_cache_1300': 'undefined',
                'support_avif': 'false',
                'support_webp': 'false',
                'xiguavideopcwebid': '7134967546256016900',
                'xiguavideopcwebid.sig': 'xxRww5R1VEMJN_dQepHorEu_eAc',
            }
            urllib3.disable_warnings()
            response = requests.get(url=url, headers=headers, params=params, cookies=cookies, verify=False)
            if 'data' not in response.json() or response.json()['data'] == '':
                Common.logger(log_type, crawler).warning('get_video_info: response: {}', response)
            else:
                video_info = response.json()['data']['gidInformation']['packerData']['video']
                video_url_dict = {}
                # video_url
                if 'videoResource' not in video_info:
                    video_url_dict["video_url"] = ''
                    video_url_dict["audio_url"] = ''
                    video_url_dict["video_width"] = 0
                    video_url_dict["video_height"] = 0

                elif 'dash_120fps' in video_info['videoResource']:
                    if "video_list" in video_info['videoResource']['dash_120fps'] and 'video_4' in video_info['videoResource']['dash_120fps']['video_list']:
                        video_url = video_info['videoResource']['dash_120fps']['video_list']['video_4']['backup_url_1']
                        audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_4']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash_120fps']['video_list']['video_4']['vwidth']
                        video_height = video_info['videoResource']['dash_120fps']['video_list']['video_4']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_3' in video_info['videoResource']['dash_120fps']['video_list']:
                        video_url = video_info['videoResource']['dash_120fps']['video_list']['video_3']['backup_url_1']
                        audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_3']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash_120fps']['video_list']['video_3']['vwidth']
                        video_height = video_info['videoResource']['dash_120fps']['video_list']['video_3']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_2' in video_info['videoResource']['dash_120fps']['video_list']:
                        video_url = video_info['videoResource']['dash_120fps']['video_list']['video_2']['backup_url_1']
                        audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_2']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash_120fps']['video_list']['video_2']['vwidth']
                        video_height = video_info['videoResource']['dash_120fps']['video_list']['video_2']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_1' in video_info['videoResource']['dash_120fps']['video_list']:
                        video_url = video_info['videoResource']['dash_120fps']['video_list']['video_1']['backup_url_1']
                        audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_1']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash_120fps']['video_list']['video_1']['vwidth']
                        video_height = video_info['videoResource']['dash_120fps']['video_list']['video_1']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height

                    elif 'dynamic_video' in video_info['videoResource']['dash_120fps'] \
                            and 'dynamic_video_list' in video_info['videoResource']['dash_120fps']['dynamic_video'] \
                            and 'dynamic_audio_list' in video_info['videoResource']['dash_120fps']['dynamic_video'] \
                            and len(video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list']) != 0 \
                            and len(video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_audio_list']) != 0:

                        video_url = video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1]['backup_url_1']
                        audio_url = video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_audio_list'][-1]['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1]['vwidth']
                        video_height = video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1]['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    else:
                        video_url_dict["video_url"] = ''
                        video_url_dict["audio_url"] = ''
                        video_url_dict["video_width"] = 0
                        video_url_dict["video_height"] = 0

                elif 'dash' in video_info['videoResource']:
                    if "video_list" in video_info['videoResource']['dash'] and 'video_4' in video_info['videoResource']['dash']['video_list']:
                        video_url = video_info['videoResource']['dash']['video_list']['video_4']['backup_url_1']
                        audio_url = video_info['videoResource']['dash']['video_list']['video_4']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash']['video_list']['video_4']['vwidth']
                        video_height = video_info['videoResource']['dash']['video_list']['video_4']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['dash'] and 'video_3' in video_info['videoResource']['dash']['video_list']:
                        video_url = video_info['videoResource']['dash']['video_list']['video_3']['backup_url_1']
                        audio_url = video_info['videoResource']['dash']['video_list']['video_3']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash']['video_list']['video_3']['vwidth']
                        video_height = video_info['videoResource']['dash']['video_list']['video_3']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['dash'] and 'video_2' in video_info['videoResource']['dash']['video_list']:
                        video_url = video_info['videoResource']['dash']['video_list']['video_2']['backup_url_1']
                        audio_url = video_info['videoResource']['dash']['video_list']['video_2']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash']['video_list']['video_2']['vwidth']
                        video_height = video_info['videoResource']['dash']['video_list']['video_2']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['dash'] and 'video_1' in video_info['videoResource']['dash']['video_list']:
                        video_url = video_info['videoResource']['dash']['video_list']['video_1']['backup_url_1']
                        audio_url = video_info['videoResource']['dash']['video_list']['video_1']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash']['video_list']['video_1']['vwidth']
                        video_height = video_info['videoResource']['dash']['video_list']['video_1']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height

                    elif 'dynamic_video' in video_info['videoResource']['dash'] \
                            and 'dynamic_video_list' in video_info['videoResource']['dash']['dynamic_video'] \
                            and 'dynamic_audio_list' in video_info['videoResource']['dash']['dynamic_video'] \
                            and len(video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list']) != 0 \
                            and len(video_info['videoResource']['dash']['dynamic_video']['dynamic_audio_list']) != 0:

                        video_url = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1]['backup_url_1']
                        audio_url = video_info['videoResource']['dash']['dynamic_video']['dynamic_audio_list'][-1]['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1]['vwidth']
                        video_height = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1]['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    else:
                        video_url_dict["video_url"] = ''
                        video_url_dict["audio_url"] = ''
                        video_url_dict["video_width"] = 0
                        video_url_dict["video_height"] = 0

                elif 'normal' in video_info['videoResource']:
                    if "video_list" in video_info['videoResource']['normal'] and 'video_4' in \
                            video_info['videoResource']['normal']['video_list']:
                        video_url = video_info['videoResource']['normal']['video_list']['video_4']['backup_url_1']
                        audio_url = video_info['videoResource']['normal']['video_list']['video_4']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['normal']['video_list']['video_4']['vwidth']
                        video_height = video_info['videoResource']['normal']['video_list']['video_4']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['normal'] and 'video_3' in \
                            video_info['videoResource']['normal']['video_list']:
                        video_url = video_info['videoResource']['normal']['video_list']['video_3']['backup_url_1']
                        audio_url = video_info['videoResource']['normal']['video_list']['video_3']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['normal']['video_list']['video_3']['vwidth']
                        video_height = video_info['videoResource']['normal']['video_list']['video_3']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['normal'] and 'video_2' in \
                            video_info['videoResource']['normal']['video_list']:
                        video_url = video_info['videoResource']['normal']['video_list']['video_2']['backup_url_1']
                        audio_url = video_info['videoResource']['normal']['video_list']['video_2']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['normal']['video_list']['video_2']['vwidth']
                        video_height = video_info['videoResource']['normal']['video_list']['video_2']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    elif "video_list" in video_info['videoResource']['normal'] and 'video_1' in \
                            video_info['videoResource']['normal']['video_list']:
                        video_url = video_info['videoResource']['normal']['video_list']['video_1']['backup_url_1']
                        audio_url = video_info['videoResource']['normal']['video_list']['video_1']['backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['normal']['video_list']['video_1']['vwidth']
                        video_height = video_info['videoResource']['normal']['video_list']['video_1']['vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height

                    elif 'dynamic_video' in video_info['videoResource']['normal'] \
                            and 'dynamic_video_list' in video_info['videoResource']['normal']['dynamic_video'] \
                            and 'dynamic_audio_list' in video_info['videoResource']['normal']['dynamic_video'] \
                            and len(video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list']) != 0 \
                            and len(video_info['videoResource']['normal']['dynamic_video']['dynamic_audio_list']) != 0:

                        video_url = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
                            'backup_url_1']
                        audio_url = video_info['videoResource']['normal']['dynamic_video']['dynamic_audio_list'][-1][
                            'backup_url_1']
                        if len(video_url) % 3 == 1:
                            video_url += '=='
                        elif len(video_url) % 3 == 2:
                            video_url += '='
                        elif len(audio_url) % 3 == 1:
                            audio_url += '=='
                        elif len(audio_url) % 3 == 2:
                            audio_url += '='
                        video_url = base64.b64decode(video_url).decode('utf8')
                        audio_url = base64.b64decode(audio_url).decode('utf8')
                        video_width = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
                            'vwidth']
                        video_height = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
                            'vheight']
                        video_url_dict["video_url"] = video_url
                        video_url_dict["audio_url"] = audio_url
                        video_url_dict["video_width"] = video_width
                        video_url_dict["video_height"] = video_height
                    else:
                        video_url_dict["video_url"] = ''
                        video_url_dict["audio_url"] = ''
                        video_url_dict["video_width"] = 0
                        video_url_dict["video_height"] = 0

                else:
                    video_url_dict["video_url"] = ''
                    video_url_dict["audio_url"] = ''
                    video_url_dict["video_width"] = 0
                    video_url_dict["video_height"] = 0

                return video_url_dict

        except Exception as e:
            Common.logger(log_type, crawler).error(f'get_video_url:{e}\n')

    @classmethod
    def get_videolist(cls, log_type, crawler, strategy, our_uid, out_uid, oss_endpoint, env, machine):
        signature = cls.random_signature()
        while True:
            url = "https://www.ixigua.com/api/videov2/author/new_video_list?"
            params = {
                'to_user_id': str(out_uid),
                'offset': str(cls.offset),
                'limit': '30',
                'maxBehotTime': '0',
                'order': 'new',
                'isHome': '0',
                'msToken': 'G0eRzNkw189a8TLaXjc6nTHVMQwh9XcxVAqTbGKi7iPJdQcLwS3-XRrJ3MZ7QBfqErpxp3EX1WtvWOIcZ3NIgr41hgcd-v64so_RRj3YCRw1UsKW8mIssNLlIMspsg==',
                'X-Bogus': 'DFSzswVuEkUANjW9ShFTgR/F6qHt',
                '_signature': signature,
            }
            headers = {
                'authority': 'www.ixigua.com',
                'accept': 'application/json, text/plain, */*',
                'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
                'cache-control': 'no-cache',
                'cookie': f'MONITOR_WEB_ID=7168304743566296612; __ac_signature={signature}; ixigua-a-s=1; support_webp=true; support_avif=false; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; msToken=G0eRzNkw189a8TLaXjc6nTHVMQwh9XcxVAqTbGKi7iPJdQcLwS3-XRrJ3MZ7QBfqErpxp3EX1WtvWOIcZ3NIgr41hgcd-v64so_RRj3YCRw1UsKW8mIssNLlIMspsg==; tt_scid=o4agqz7u9SKPwfBoPt6S82Cw0q.9KDtqmNe0JHxMqmpxNHQWq1BmrQdgVU6jEoX7ed99; ttwid=1%7CHHtv2QqpSGuSu8r-zXF1QoWsvjmNi1SJrqOrZzg-UCY%7C1676618894%7Cee5ad95378275f282f230a7ffa9947ae7eff40d0829c5a2568672a6dc90a1c96; ixigua-a-s=1',
                'pragma': 'no-cache',
                'referer': f'https://www.ixigua.com/home/{out_uid}/video/?preActiveKey=hotsoon&list_entrance=userdetail',
                'sec-ch-ua': '"Chromium";v="110", "Not A(Brand";v="24", "Microsoft Edge";v="110"',
                'sec-ch-ua-mobile': '?0',
                'sec-ch-ua-platform': '"macOS"',
                'sec-fetch-dest': 'empty',
                'sec-fetch-mode': 'cors',
                'sec-fetch-site': 'same-origin',
                'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.41',
                'x-secsdk-csrf-token': '00010000000119e3f9454d1dcbb288704cda1960f241e2d19bd21f2fd283520c3615a990ac5a17448bfbb902a249'
            }
            urllib3.disable_warnings()
            response = requests.get(url=url, headers=headers, params=params, proxies=proxies, verify=False)
            cls.offset += 30
            if response.status_code != 200:
                Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.text}\n")
            elif 'data' not in response.text:
                Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.text}\n")
            elif 'videoList' not in response.json()["data"]:
                Common.logger(log_type, crawler).warning(f"get_videolist_response:{response.json()}\n")
            else:
                videoList = response.json()['data']['videoList']
                for i in range(len(videoList)):
                    # video_title
                    if 'title' not in videoList[i]:
                        video_title = 0
                    else:
                        video_title = videoList[i]['title'].strip().replace('手游', '') \
                            .replace('/', '').replace('\/', '').replace('\n', '')

                    # video_id
                    if 'video_id' not in videoList[i]:
                        video_id = 0
                    else:
                        video_id = videoList[i]['video_id']

                    # gid
                    if 'gid' not in videoList[i]:
                        gid = 0
                    else:
                        gid = videoList[i]['gid']

                    # play_cnt
                    if 'video_detail_info' not in videoList[i]:
                        play_cnt = 0
                    elif 'video_watch_count' not in videoList[i]['video_detail_info']:
                        play_cnt = 0
                    else:
                        play_cnt = videoList[i]['video_detail_info']['video_watch_count']

                    # comment_cnt
                    if 'comment_count' not in videoList[i]:
                        comment_cnt = 0
                    else:
                        comment_cnt = videoList[i]['comment_count']

                    # like_cnt
                    if 'digg_count' not in videoList[i]:
                        like_cnt = 0
                    else:
                        like_cnt = videoList[i]['digg_count']

                    # share_cnt
                    share_cnt = 0

                    # video_duration
                    if 'video_duration' not in videoList[i]:
                        video_duration = 0
                    else:
                        video_duration = videoList[i]['video_duration']

                    # send_time
                    if 'publish_time' not in videoList[i]:
                        publish_time = 0
                    else:
                        publish_time = videoList[i]['publish_time']

                    publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time))

                    # is_top
                    if 'is_top' not in videoList[i]:
                        is_top = 0
                    else:
                        is_top = videoList[i]['is_top']

                    # user_name
                    if 'user_info' not in videoList[i]:
                        user_name = 0
                    elif 'name' not in videoList[i]['user_info']:
                        user_name = 0
                    else:
                        user_name = videoList[i]['user_info']['name']

                    # user_id
                    if 'user_info' not in videoList[i]:
                        user_id = 0
                    elif 'user_id' not in videoList[i]['user_info']:
                        user_id = 0
                    else:
                        user_id = videoList[i]['user_info']['user_id']

                    # avatar_url
                    if 'user_info' not in videoList[i]:
                        avatar_url = 0
                    elif 'avatar_url' not in videoList[i]['user_info']:
                        avatar_url = 0
                    else:
                        avatar_url = videoList[i]['user_info']['avatar_url']

                    # cover_url
                    if 'video_detail_info' not in videoList[i]:
                        cover_url = 0
                    elif 'detail_video_large_image' not in videoList[i]['video_detail_info']:
                        cover_url = 0
                    elif 'url' in videoList[i]['video_detail_info']['detail_video_large_image']:
                        cover_url = videoList[i]['video_detail_info']['detail_video_large_image']['url']
                    else:
                        cover_url = videoList[i]['video_detail_info']['detail_video_large_image']['url_list'][0]['url']

                    if gid == 0 or video_id == 0 or cover_url == 0:
                        Common.logger(log_type, crawler).info('无效视频\n')
                    elif is_top is True and int(time.time()) - int(publish_time) > 3600 * 24 * 10:
                        Common.logger(log_type, crawler).info(f'置顶视频，且发布时间超过10天:{publish_time_str}\n')
                    elif int(time.time()) - int(publish_time) > 3600 * 24 * 10:
                        Common.logger(log_type, crawler).info(f'发布时间超过10天:{publish_time_str}\n')
                        cls.offset = 0
                        return
                    else:
                        video_url_dict = cls.get_video_url(log_type, crawler, gid)
                        video_url = video_url_dict["video_url"]
                        audio_url = video_url_dict["audio_url"]
                        video_width = video_url_dict["video_width"]
                        video_height = video_url_dict["video_height"]

                        video_dict = {'video_title': video_title,
                                      'video_id': video_id,
                                      'gid': gid,
                                      'play_cnt': play_cnt,
                                      'comment_cnt': comment_cnt,
                                      'like_cnt': like_cnt,
                                      'share_cnt': share_cnt,
                                      'video_width': video_width,
                                      'video_height': video_height,
                                      'duration': video_duration,
                                      'publish_time_stamp': publish_time,
                                      'publish_time_str': publish_time_str,
                                      'is_top': is_top,
                                      'user_name': user_name,
                                      'user_id': user_id,
                                      'avatar_url': avatar_url,
                                      'cover_url': cover_url,
                                      'audio_url': audio_url,
                                      'video_url': video_url,
                                      'session': signature}
                        for k, v in video_dict.items():
                            Common.logger(log_type, crawler).info(f"{k}:{v}")
                        # cls.download_publish(log_type=log_type,
                        #                      crawler=crawler,
                        #                      video_dict=video_dict,
                        #                      strategy=strategy,
                        #                      our_uid=our_uid,
                        #                      oss_endpoint=oss_endpoint,
                        #                      env=env,
                        #                      machine=machine)

    # 下载 / 上传
    @classmethod
    def download_publish(cls, log_type, crawler, strategy, video_dict, our_uid, oss_endpoint, env, machine):
        # try:
        if cls.download_rule(video_dict['duration'], video_dict['video_width'], video_dict['video_height']) is False:
            Common.logger(log_type, crawler).info('不满足抓取规则\n')
        elif any(word if word in video_dict['video_title'] else False for word in cls.filter_words(log_type, crawler)) is True:
            Common.logger(log_type, crawler).info('标题已中过滤词:{}\n', video_dict['video_title'])
        elif str(video_dict['video_id']) in [x for y in Feishu.get_values_batch(log_type, 'xigua', 'e075e9') for x in y]:
            Common.logger(log_type, crawler).info('视频已下载\n')
        elif str(video_dict['video_id']) in [x for y in Feishu.get_values_batch(log_type, 'xigua', '3Ul6wZ') for x in y]:
            Common.logger(log_type, crawler).info('视频已下载\n')
        elif str(video_dict['video_id']) in [x for y in Feishu.get_values_batch(log_type, 'xigua', 'QOWqMo') for x in y]:
            Common.logger(log_type, crawler).info('视频已下载\n')
        elif str(video_dict['video_id']) in [x for y in Feishu.get_values_batch(log_type, 'xigua', 'wjhpDs') for x in y]:
            Common.logger(log_type, crawler).info('视频已存在\n')
        else:
            # 下载封面
            Common.download_method(log_type=log_type, crawler=crawler, text='cover', title=video_dict['video_title'], url=video_dict['cover_url'])
            # 下载视频
            Common.download_method(log_type=log_type, crawler=crawler, text='xigua_video', title=video_dict['video_title'], url=video_dict['video_url'])
            # 下载音频
            Common.download_method(log_type=log_type, crawler=crawler, text='xigua_audio', title=video_dict['video_title'], url=video_dict['audio_url'])
            # 保存视频信息至txt
            Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)
            # 合成音视频
            Common.video_compose(log_type=log_type, crawler=crawler, video_dir=f"./{crawler}/videos/{video_dict['video_title']}")

            # 上传视频
            Common.logger(log_type, crawler).info("开始上传视频...")
            our_video_id = Publish.upload_and_publish(log_type=log_type,
                                                      crawler=crawler,
                                                      strategy=strategy,
                                                      our_uid=our_uid,
                                                      env=env,
                                                      oss_endpoint=oss_endpoint)
            if env == 'dev':
                our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
            else:
                our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
            Common.logger(log_type, crawler).info("视频上传完成")

            if our_video_id is None:
                # 删除视频文件夹
                shutil.rmtree(f"./{crawler}/videos/{video_dict['video_title']}/")
                return

            # 视频写入飞书
            Feishu.insert_columns(log_type, 'xigua', "e075e9", "ROWS", 1, 2)
            upload_time = int(time.time())
            values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
                       "定向榜",
                       video_dict['video_title'],
                       str(video_dict['video_id']),
                       our_video_link,
                       video_dict['gid'],
                       video_dict['play_cnt'],
                       video_dict['comment_cnt'],
                       video_dict['like_cnt'],
                       video_dict['share_cnt'],
                       video_dict['duration'],
                       str(video_dict['video_width']) + '*' + str(video_dict['video_height']),
                       video_dict['publish_time_str'],
                       video_dict['user_name'],
                       video_dict['user_id'],
                       video_dict['avatar_url'],
                       video_dict['cover_url'],
                       video_dict['video_url'],
                       video_dict['audio_url']]]
            time.sleep(1)
            Feishu.update_values(log_type, 'xigua', "e075e9", "F2:Z2", values)
            Common.logger(log_type, crawler).info(f"视频已保存至云文档\n")

            # 视频信息保存数据库
            sql = f""" insert into crawler_video(video_id,
                            user_id,
                            out_user_id,
                            platform,
                            strategy,
                            out_video_id,
                            video_title,
                            cover_url,
                            video_url,
                            duration,
                            publish_time,
                            play_cnt,
                            crawler_rule,
                            width,
                            height)
                            values({our_video_id},
                            "{our_uid}",
                            "{video_dict['user_id']}",
                            "{cls.platform}",
                            "定向爬虫策略",
                            "{video_dict['video_id']}",
                            "{video_dict['video_title']}",
                            "{video_dict['cover_url']}",
                            "{video_dict['video_url']}",
                            {int(video_dict['duration'])},
                            "{video_dict['publish_time_str']}",
                            {int(video_dict['play_cnt'])},
                            "4,5,6",
                            {int(video_dict['video_width'])},
                            {int(video_dict['video_height'])}) """
            MysqlHelper.update_values(log_type, crawler, sql, env, machine)
            Common.logger(log_type, crawler).info('视频信息插入数据库成功！\n')
        # except Exception as e:
        #     Common.logger(log_type, crawler).error(f'download_publish异常:{e}\n')


if __name__ == '__main__':
    # print(Follow.get_signature("follow", "xigua", "95420624045", "local"))
    # Follow.get_videolist(log_type="follow",
    #                      crawler="xigua",
    #                      strategy="定向爬虫策略",
    #                      our_uid="6267141",
    #                      out_uid="95420624045",
    #                      oss_endpoint="out",
    #                      env="dev",
    #                      machine="local")
    # print(Follow.random_signature())
    user_list = Follow.get_user_list(log_type="follow", crawler="xigua", sheetid="5tlTYB", env="dev", machine="local")
    print(len(user_list))
    for user in user_list:
        print(user)
    pass