piaoquan_crawler/xigua/xigua_search/xigua_search_new.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/2/17
import base64
import json
import os
import random
import shutil
import string
import sys
import time
from hashlib import md5
import requests
import urllib3
from requests.adapters import HTTPAdapter
from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.chrome.service import Service
from selenium import webdriver
from selenium.webdriver.common.by import By
sys.path.append(os.getcwd())
from common.scheduling_db import MysqlHelper
from common.getuser import getUser
from common.common import Common
from common.feishu import Feishu
from common.publish import Publish
from common.public import get_config_from_mysql
from common.userAgent import get_random_user_agent


class XiguaSearchNew:
    # 抓取视频数
    i = 0
    # 已下载视频数
    videos_cnt = 0
    platform = "西瓜视频"
    tag = "西瓜视频爬虫,搜索爬虫策略"

    @classmethod
    def get_rule_dict(cls, log_type, crawler):
        while True:
            rule_sheet = Feishu.get_values_batch(log_type, crawler, "shxOl7")
            if rule_sheet is None:
                Common.logger(log_type, crawler).info(f"get_rule:{rule_sheet}，2秒钟后重试")
                time.sleep(2)
                continue
            rule_dict = {
                "play_cnt": int(rule_sheet[1][2]),
                "duration_min": int(rule_sheet[2][2]),
                "duration_max": int(rule_sheet[3][2]),
                "publish_time": int(rule_sheet[4][2]),
                "like_cnt": int(rule_sheet[5][2]),
                "comment_cnt": int(rule_sheet[6][2])
            }
            return rule_dict

    # 下载规则
    @classmethod
    def download_rule(cls, log_type, crawler, video_dict, rule_dict):
        Common.logger(log_type, crawler).info(f'play_cnt: {video_dict["play_cnt"]} >= {rule_dict["play_cnt"]}')
        Common.logger(log_type, crawler).info(f'duration: {rule_dict["duration_max"]} >= {video_dict["duration"]} >= {rule_dict["duration_min"]}')
        Common.logger(log_type, crawler).info(f'publish_time: {int(time.time())} - {video_dict["publish_time_stamp"]} = {int(time.time())-video_dict["publish_time_stamp"]} <= {rule_dict["publish_time"] * 3600 * 24}')
        Common.logger(log_type, crawler).info(f'like_cnt: {video_dict["like_cnt"]} >= {rule_dict["like_cnt"]}')
        Common.logger(log_type, crawler).info(f'comment_cnt: {video_dict["comment_cnt"]} >= {rule_dict["comment_cnt"]}')
        if video_dict["play_cnt"] >= rule_dict["play_cnt"] \
            and rule_dict["duration_max"] >= video_dict["duration"] >= rule_dict["duration_min"] \
            and int(time.time()) - video_dict["publish_time_stamp"] <= rule_dict["publish_time"]*3600*24 \
            and video_dict["like_cnt"] >= rule_dict["like_cnt"] \
            and video_dict["comment_cnt"] >= rule_dict["comment_cnt"]:
            return True
        else:
            return False

    # 过滤词库
    @classmethod
    def filter_words(cls, log_type, crawler, env):
        filter_words_list = get_config_from_mysql(log_type, crawler, env, "filter")
        return filter_words_list

    # 获取用户信息（字典格式）. 注意：部分 user_id 字符类型是 int / str
    @classmethod
    def get_user_list(cls, log_type, crawler, sheetid, env):
        try:
            while True:
                user_sheet = Feishu.get_values_batch(log_type, crawler, sheetid)
                if user_sheet is None:
                    Common.logger(log_type, crawler).warning(f"user_sheet:{user_sheet} 10秒钟后重试")
                    continue
                our_user_list = []
                for i in range(1, len(user_sheet)):
                    our_uid = user_sheet[i][6]
                    search_word = user_sheet[i][4]
                    tag1 = user_sheet[i][8]
                    tag2 = user_sheet[i][9]
                    tag3 = user_sheet[i][10]
                    tag4 = user_sheet[i][11]
                    tag5 = user_sheet[i][12]
                    tag6 = user_sheet[i][13]
                    tag7 = user_sheet[i][14]
                    Common.logger(log_type, crawler).info(f"正在更新 {search_word} 关键词信息\n")
                    if our_uid is None:
                        default_user = getUser.get_default_user()
                        # 用来创建our_id的信息
                        user_dict = {
                            'recommendStatus': -6,
                            'appRecommendStatus': -6,
                            'nickName': default_user['nickName'],
                            'avatarUrl': default_user['avatarUrl'],
                            'tagName': f'{tag1},{tag2},{tag3},{tag4},{tag5},{tag6},{tag7}',
                        }
                        Common.logger(log_type, crawler).info(f'新创建的站内UID:{our_uid}')
                        our_uid = getUser.create_uid(log_type, crawler, user_dict, env)
                        if env == 'prod':
                            our_user_link = f'https://admin.piaoquantv.com/ums/user/{our_uid}/post'
                        else:
                            our_user_link = f'https://testadmin.piaoquantv.com/ums/user/{our_uid}/post'
                        Feishu.update_values(log_type, crawler, sheetid, f'G{i + 1}:H{i + 1}',
                                             [[our_uid, our_user_link]])
                        Common.logger(log_type, crawler).info(f'站内用户信息写入飞书成功！\n')
                    our_user_dict = {
                        'out_uid': '',
                        'search_word': search_word,
                        'our_uid': our_uid,
                        'our_user_link': f'https://admin.piaoquantv.com/ums/user/{our_uid}/post',
                    }
                    our_user_list.append(our_user_dict)

                return our_user_list
        except Exception as e:
            Common.logger(log_type, crawler).error(f'get_user_id_from_feishu异常:{e}\n')

    @classmethod
    def videos_cnt_rule(cls, log_type, crawler):
        while True:
            videos_cnt_sheet = Feishu.get_values_batch(log_type, crawler, "shxOl7")
            if videos_cnt_sheet is None:
                time.sleep(2)
                continue
            return int(videos_cnt_sheet[7][2])

    @classmethod
    def random_signature(cls):
        src_digits = string.digits  # string_数字
        src_uppercase = string.ascii_uppercase  # string_大写字母
        src_lowercase = string.ascii_lowercase  # string_小写字母
        digits_num = random.randint(1, 6)
        uppercase_num = random.randint(1, 26 - digits_num - 1)
        lowercase_num = 26 - (digits_num + uppercase_num)
        password = random.sample(src_digits, digits_num) + random.sample(src_uppercase, uppercase_num) + random.sample(
            src_lowercase, lowercase_num)
        random.shuffle(password)
        new_password = 'AAAAAAAAAA' + ''.join(password)[10:-4] + 'AAAB'
        new_password_start = new_password[0:18]
        new_password_end = new_password[-7:]
        if new_password[18] == '8':
            new_password = new_password_start + 'w' + new_password_end
        elif new_password[18] == '9':
            new_password = new_password_start + 'x' + new_password_end
        elif new_password[18] == '-':
            new_password = new_password_start + 'y' + new_password_end
        elif new_password[18] == '.':
            new_password = new_password_start + 'z' + new_password_end
        else:
            new_password = new_password_start + 'y' + new_password_end
        return new_password

    @classmethod
    def get_video_url(cls, video_info):
        video_url_dict = {}
        # video_url
        if 'videoResource' not in video_info:
            video_url_dict["video_url"] = ''
            video_url_dict["audio_url"] = ''
            video_url_dict["video_width"] = 0
            video_url_dict["video_height"] = 0

        elif 'dash_120fps' in video_info['videoResource']:
            if "video_list" in video_info['videoResource']['dash_120fps'] and 'video_4' in \
                    video_info['videoResource']['dash_120fps']['video_list']:
                video_url = video_info['videoResource']['dash_120fps']['video_list']['video_4']['backup_url_1']
                audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_4']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash_120fps']['video_list']['video_4']['vwidth']
                video_height = video_info['videoResource']['dash_120fps']['video_list']['video_4']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_3' in \
                    video_info['videoResource']['dash_120fps']['video_list']:
                video_url = video_info['videoResource']['dash_120fps']['video_list']['video_3']['backup_url_1']
                audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_3']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash_120fps']['video_list']['video_3']['vwidth']
                video_height = video_info['videoResource']['dash_120fps']['video_list']['video_3']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_2' in \
                    video_info['videoResource']['dash_120fps']['video_list']:
                video_url = video_info['videoResource']['dash_120fps']['video_list']['video_2']['backup_url_1']
                audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_2']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash_120fps']['video_list']['video_2']['vwidth']
                video_height = video_info['videoResource']['dash_120fps']['video_list']['video_2']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['dash_120fps'] and 'video_1' in \
                    video_info['videoResource']['dash_120fps']['video_list']:
                video_url = video_info['videoResource']['dash_120fps']['video_list']['video_1']['backup_url_1']
                audio_url = video_info['videoResource']['dash_120fps']['video_list']['video_1']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash_120fps']['video_list']['video_1']['vwidth']
                video_height = video_info['videoResource']['dash_120fps']['video_list']['video_1']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height

            elif 'dynamic_video' in video_info['videoResource']['dash_120fps'] \
                    and 'dynamic_video_list' in video_info['videoResource']['dash_120fps']['dynamic_video'] \
                    and 'dynamic_audio_list' in video_info['videoResource']['dash_120fps']['dynamic_video'] \
                    and len(
                video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list']) != 0 \
                    and len(
                video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_audio_list']) != 0:

                video_url = \
                    video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1][
                        'backup_url_1']
                audio_url = \
                    video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_audio_list'][-1][
                        'backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = \
                    video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1][
                        'vwidth']
                video_height = \
                    video_info['videoResource']['dash_120fps']['dynamic_video']['dynamic_video_list'][-1][
                        'vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            else:
                video_url_dict["video_url"] = ''
                video_url_dict["audio_url"] = ''
                video_url_dict["video_width"] = 0
                video_url_dict["video_height"] = 0

        elif 'dash' in video_info['videoResource']:
            if "video_list" in video_info['videoResource']['dash'] and 'video_4' in \
                    video_info['videoResource']['dash']['video_list']:
                video_url = video_info['videoResource']['dash']['video_list']['video_4']['backup_url_1']
                audio_url = video_info['videoResource']['dash']['video_list']['video_4']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash']['video_list']['video_4']['vwidth']
                video_height = video_info['videoResource']['dash']['video_list']['video_4']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['dash'] and 'video_3' in \
                    video_info['videoResource']['dash']['video_list']:
                video_url = video_info['videoResource']['dash']['video_list']['video_3']['backup_url_1']
                audio_url = video_info['videoResource']['dash']['video_list']['video_3']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash']['video_list']['video_3']['vwidth']
                video_height = video_info['videoResource']['dash']['video_list']['video_3']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['dash'] and 'video_2' in \
                    video_info['videoResource']['dash']['video_list']:
                video_url = video_info['videoResource']['dash']['video_list']['video_2']['backup_url_1']
                audio_url = video_info['videoResource']['dash']['video_list']['video_2']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash']['video_list']['video_2']['vwidth']
                video_height = video_info['videoResource']['dash']['video_list']['video_2']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['dash'] and 'video_1' in \
                    video_info['videoResource']['dash']['video_list']:
                video_url = video_info['videoResource']['dash']['video_list']['video_1']['backup_url_1']
                audio_url = video_info['videoResource']['dash']['video_list']['video_1']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash']['video_list']['video_1']['vwidth']
                video_height = video_info['videoResource']['dash']['video_list']['video_1']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height

            elif 'dynamic_video' in video_info['videoResource']['dash'] \
                    and 'dynamic_video_list' in video_info['videoResource']['dash']['dynamic_video'] \
                    and 'dynamic_audio_list' in video_info['videoResource']['dash']['dynamic_video'] \
                    and len(video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list']) != 0 \
                    and len(video_info['videoResource']['dash']['dynamic_video']['dynamic_audio_list']) != 0:

                video_url = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
                    'backup_url_1']
                audio_url = video_info['videoResource']['dash']['dynamic_video']['dynamic_audio_list'][-1][
                    'backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
                    'vwidth']
                video_height = video_info['videoResource']['dash']['dynamic_video']['dynamic_video_list'][-1][
                    'vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            else:
                video_url_dict["video_url"] = ''
                video_url_dict["audio_url"] = ''
                video_url_dict["video_width"] = 0
                video_url_dict["video_height"] = 0

        elif 'normal' in video_info['videoResource']:
            if "video_list" in video_info['videoResource']['normal'] and 'video_4' in \
                    video_info['videoResource']['normal']['video_list']:
                video_url = video_info['videoResource']['normal']['video_list']['video_4']['backup_url_1']
                audio_url = video_info['videoResource']['normal']['video_list']['video_4']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['normal']['video_list']['video_4']['vwidth']
                video_height = video_info['videoResource']['normal']['video_list']['video_4']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['normal'] and 'video_3' in \
                    video_info['videoResource']['normal']['video_list']:
                video_url = video_info['videoResource']['normal']['video_list']['video_3']['backup_url_1']
                audio_url = video_info['videoResource']['normal']['video_list']['video_3']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['normal']['video_list']['video_3']['vwidth']
                video_height = video_info['videoResource']['normal']['video_list']['video_3']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['normal'] and 'video_2' in \
                    video_info['videoResource']['normal']['video_list']:
                video_url = video_info['videoResource']['normal']['video_list']['video_2']['backup_url_1']
                audio_url = video_info['videoResource']['normal']['video_list']['video_2']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['normal']['video_list']['video_2']['vwidth']
                video_height = video_info['videoResource']['normal']['video_list']['video_2']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            elif "video_list" in video_info['videoResource']['normal'] and 'video_1' in \
                    video_info['videoResource']['normal']['video_list']:
                video_url = video_info['videoResource']['normal']['video_list']['video_1']['backup_url_1']
                audio_url = video_info['videoResource']['normal']['video_list']['video_1']['backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['normal']['video_list']['video_1']['vwidth']
                video_height = video_info['videoResource']['normal']['video_list']['video_1']['vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height

            elif 'dynamic_video' in video_info['videoResource']['normal'] \
                    and 'dynamic_video_list' in video_info['videoResource']['normal']['dynamic_video'] \
                    and 'dynamic_audio_list' in video_info['videoResource']['normal']['dynamic_video'] \
                    and len(video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list']) != 0 \
                    and len(video_info['videoResource']['normal']['dynamic_video']['dynamic_audio_list']) != 0:

                video_url = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
                    'backup_url_1']
                audio_url = video_info['videoResource']['normal']['dynamic_video']['dynamic_audio_list'][-1][
                    'backup_url_1']
                if len(video_url) % 3 == 1:
                    video_url += '=='
                elif len(video_url) % 3 == 2:
                    video_url += '='
                elif len(audio_url) % 3 == 1:
                    audio_url += '=='
                elif len(audio_url) % 3 == 2:
                    audio_url += '='
                video_url = base64.b64decode(video_url).decode('utf8')
                audio_url = base64.b64decode(audio_url).decode('utf8')
                video_width = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
                    'vwidth']
                video_height = video_info['videoResource']['normal']['dynamic_video']['dynamic_video_list'][-1][
                    'vheight']
                video_url_dict["video_url"] = video_url
                video_url_dict["audio_url"] = audio_url
                video_url_dict["video_width"] = video_width
                video_url_dict["video_height"] = video_height
            else:
                video_url_dict["video_url"] = ''
                video_url_dict["audio_url"] = ''
                video_url_dict["video_width"] = 0
                video_url_dict["video_height"] = 0

        else:
            video_url_dict["video_url"] = ''
            video_url_dict["audio_url"] = ''
            video_url_dict["video_width"] = 0
            video_url_dict["video_height"] = 0

        return video_url_dict

    @classmethod
    def get_comment_cnt(cls, item_id):
        url = "https://www.ixigua.com/tlb/comment/article/v5/tab_comments/?"
        params = {
            "tab_index": "0",
            "count": "10",
            "offset": "10",
            "group_id": str(item_id),
            "item_id": str(item_id),
            "aid": "1768",
            "msToken": "50-JJObWB07HfHs-BMJWT1eIDX3G-6lPSF_i-QwxBIXE9VVa-iN0jbEXR5pG2DKjXBmP299n6ZTuXzY-GAy968CCvouSAYIS4GzvGQT3pNlKNejr5G4-1g==",
            "X-Bogus": "DFSzswVOyGtANVeWtCLMqR/F6q9U",
            "_signature": cls.random_signature(),
        }
        headers = {
            'authority': 'www.ixigua.com',
            'accept': 'application/json, text/plain, */*',
            'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'cache-control': 'no-cache',
            'cookie': 'MONITOR_WEB_ID=67cb5099-a022-4ec3-bb8e-c4de6ba51dd0; passport_csrf_token=72b2574f3c99f8ba670e42df430218fd; passport_csrf_token_default=72b2574f3c99f8ba670e42df430218fd; sid_guard=c7472b508ea631823ba765a60cf8757f%7C1680867422%7C3024002%7CFri%2C+12-May-2023+11%3A37%3A04+GMT; uid_tt=c13f47d51767f616befe32fb3e9f485a; uid_tt_ss=c13f47d51767f616befe32fb3e9f485a; sid_tt=c7472b508ea631823ba765a60cf8757f; sessionid=c7472b508ea631823ba765a60cf8757f; sessionid_ss=c7472b508ea631823ba765a60cf8757f; sid_ucp_v1=1.0.0-KGUzNWYxNmRkZGJiZjgxY2MzZWNkMTEzMTkwYjY1Yjg5OTY5NzVlNmMKFQiu3d-eqQIQ3oDAoQYYGCAMOAhACxoCaGwiIGM3NDcyYjUwOGVhNjMxODIzYmE3NjVhNjBjZjg3NTdm; ssid_ucp_v1=1.0.0-KGUzNWYxNmRkZGJiZjgxY2MzZWNkMTEzMTkwYjY1Yjg5OTY5NzVlNmMKFQiu3d-eqQIQ3oDAoQYYGCAMOAhACxoCaGwiIGM3NDcyYjUwOGVhNjMxODIzYmE3NjVhNjBjZjg3NTdm; odin_tt=b893608d4dde2e1e8df8cd5d97a0e2fbeafc4ca762ac72ebef6e6c97e2ed19859bb01d46b4190ddd6dd17d7f9678e1de; SEARCH_CARD_MODE=7168304743566296612_0; support_webp=true; support_avif=false; csrf_session_id=a5355d954d3c63ed1ba35faada452b4d; tt_scid=7Pux7s634-z8DYvCM20y7KigwH5u7Rh6D9C-RROpnT.aGMEcz6Vsxp.oai47wJqa4f86; ttwid=1%7CHHtv2QqpSGuSu8r-zXF1QoWsvjmNi1SJrqOrZzg-UCY%7C1683858689%7Ca5223fe1500578e01e138a0d71d6444692018296c4c24f5885af174a65873c95; ixigua-a-s=3; msToken=50-JJObWB07HfHs-BMJWT1eIDX3G-6lPSF_i-QwxBIXE9VVa-iN0jbEXR5pG2DKjXBmP299n6ZTuXzY-GAy968CCvouSAYIS4GzvGQT3pNlKNejr5G4-1g==; __ac_nonce=0645dcbf0005064517440; __ac_signature=_02B4Z6wo00f01FEGmAwAAIDBKchzCGqn-MBRJpyAAHAjieFC5GEg6gGiwz.I4PRrJl7f0GcixFrExKmgt6QI1i1S-dQyofPEj2ugWTCnmKUdJQv-wYuDofeKNe8VtMtZq2aKewyUGeKU-5Ud21; ixigua-a-s=3',
            'pragma': 'no-cache',
            'referer': f'https://www.ixigua.com/{item_id}?logTag=3c5aa86a8600b9ab8540',
            'sec-ch-ua': '"Microsoft Edge";v="113", "Chromium";v="113", "Not-A.Brand";v="24"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"macOS"',
            'sec-fetch-dest': 'empty',
            'sec-fetch-mode': 'cors',
            'sec-fetch-site': 'same-origin',
            'tt-anti-token': 'cBITBHvmYjEygzv-f9c78c1297722cf1f559c74b084e4525ce4900bdcf9e8588f20cc7c2e3234422',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35',
            'x-secsdk-csrf-token': '000100000001f8e733cf37f0cd255a51aea9a81ff7bc0c09490cfe41ad827c3c5c18ec809279175e4d9f5553d8a5'
        }
        urllib3.disable_warnings()
        s = requests.session()
        # max_retries=3 重试3次
        s.mount('http://', HTTPAdapter(max_retries=3))
        s.mount('https://', HTTPAdapter(max_retries=3))
        response = s.get(url=url, headers=headers, params=params, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
        response.close()
        if response.status_code != 200 or 'total_number' not in response.json() or response.json() == {}:
            return 0
        return response.json().get("total_number", 0)

    # 获取视频详情
    @classmethod
    def get_video_info(cls, log_type, crawler, item_id):
        url = 'https://www.ixigua.com/api/mixVideo/information?'
        headers = {
            "accept-encoding": "gzip, deflate",
            "accept-language": "zh-CN,zh-Hans;q=0.9",
            "user-agent": get_random_user_agent('pc'),
            "referer": "https://www.ixigua.com/7102614741050196520?logTag=0531c88ac04f38ab2c62",
        }
        params = {
            'mixId': str(item_id),
            'msToken': 'IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfC'
                       'NVVIOBNjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA',
            'X-Bogus': 'DFSzswVupYTANCJOSBk0P53WxM-r',
            '_signature': '_02B4Z6wo0000119LvEwAAIDCuktNZ0y5wkdfS7jAALThuOR8D9yWNZ.EmWHKV0WSn6Px'
                          'fPsH9-BldyxVje0f49ryXgmn7Tzk-swEHNb15TiGqa6YF.cX0jW8Eds1TtJOIZyfc9s5emH7gdWN94',
        }
        cookies = {
            'ixigua-a-s': '1',
            'msToken': 'IlG0wd0Pylyw9ghcYiB2YseUmTwrsrqqhXrbIcsSaTcLTJyVlbYJzk20zw3UO-CfrfCNVVIOB'
                       'NjIl7vfBoxnVUwO9ZyzAI3umSKsT5-pef_RRfQCJwmA',
            'ttwid': '1%7C_yXQeHWwLZgCsgHClOwTCdYSOt_MjdOkgnPIkpi-Sr8%7C1661241238%7Cf57d0c5ef3f1d7'
                     '6e049fccdca1ac54887c34d1f8731c8e51a49780ff0ceab9f8',
            'tt_scid': 'QZ4l8KXDG0YAEaMCSbADdcybdKbUfG4BC6S4OBv9lpRS5VyqYLX2bIR8CTeZeGHR9ee3',
            'MONITOR_WEB_ID': '0a49204a-7af5-4e96-95f0-f4bafb7450ad',
            '__ac_nonce': '06304878000964fdad287',
            '__ac_signature': '_02B4Z6wo00f017Rcr3AAAIDCUVxeW1tOKEu0fKvAAI4cvoYzV-wBhq7B6D8k0no7lb'
                              'FlvYoinmtK6UXjRIYPXnahUlFTvmWVtb77jsMkKAXzAEsLE56m36RlvL7ky.M3Xn52r9t1IEb7IR3ke8',
            'ttcid': 'e56fabf6e85d4adf9e4d91902496a0e882',
            '_tea_utm_cache_1300': 'undefined',
            'support_avif': 'false',
            'support_webp': 'false',
            'xiguavideopcwebid': '7134967546256016900',
            'xiguavideopcwebid.sig': 'xxRww5R1VEMJN_dQepHorEu_eAc',
        }
        urllib3.disable_warnings()
        s = requests.session()
        # max_retries=3 重试3次
        s.mount('http://', HTTPAdapter(max_retries=3))
        s.mount('https://', HTTPAdapter(max_retries=3))
        response = s.get(url=url, headers=headers, params=params, cookies=cookies, verify=False, proxies=Common.tunnel_proxies(), timeout=5)
        response.close()
        if response.status_code != 200 or 'data' not in response.json() or response.json()['data'] == {}:
            Common.logger(log_type, crawler).warning(f"get_video_info:{response.status_code}, {response.text}\n")
            return None
        else:
            video_info = response.json()['data'].get("gidInformation", {}).get("packerData", {}).get("video", {})
            if video_info == {}:
                return None
            video_dict = {
                "video_title": video_info.get("title", ""),
                "video_id": video_info.get("videoResource", {}).get("vid", ""),
                "gid": str(item_id),
                "play_cnt": int(video_info.get("video_watch_count", 0)),
                "like_cnt": int(video_info.get("video_like_count", 0)),
                "comment_cnt": int(cls.get_comment_cnt(item_id)),
                "share_cnt": 0,
                "favorite_cnt": 0,
                "duration": int(video_info.get("video_duration", 0)),
                "video_width": int(cls.get_video_url(video_info)["video_width"]),
                "video_height": int(cls.get_video_url(video_info)["video_height"]),
                "publish_time_stamp": int(video_info.get("video_publish_time", 0)),
                "publish_time_str": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(video_info.get("video_publish_time", 0)))),
                "user_name": video_info.get("user_info", {}).get("name", ""),
                "user_id": str(video_info.get("user_info", {}).get("user_id", "")),
                "avatar_url": str(video_info.get("user_info", {}).get("avatar_url", "")),
                "cover_url": video_info.get("poster_url", ""),
                "audio_url": cls.get_video_url(video_info)["audio_url"],
                "video_url": cls.get_video_url(video_info)["video_url"],
                "session": f"xigua-search-{int(time.time())}"
            }
            return video_dict

    @classmethod
    def get_videoList(cls, log_type, crawler, search_word, our_uid, env):
        # 打印请求配置
        ca = DesiredCapabilities.CHROME
        ca["goog:loggingPrefs"] = {"performance": "ALL"}
        # # 不打开浏览器运行
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument(f'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36')
        chrome_options.add_argument("--headless")
        chrome_options.add_argument("--no-sandbox")
        if env == "dev":
            chromedriver = "/Users/wangkun/Downloads/chromedriver/chromedriver_v112/chromedriver"
        else:
            chromedriver = "/usr/bin/chromedriver"
        # driver初始化
        driver = webdriver.Chrome(desired_capabilities=ca, options=chrome_options, service=Service(chromedriver))
        driver.implicitly_wait(10)
        Common.logger(log_type, crawler).info(f"打开搜索页:{search_word}")
        driver.get(f"https://www.ixigua.com/search/{search_word}/")
        time.sleep(1)

        index = 0
        while True:
            video_elements = driver.find_elements(By.XPATH, '//*[@class="HorizontalFeedCard searchPageV2__card"]')
            video_element_temp = video_elements[index:]
            if len(video_element_temp) == 0:
                Common.logger(log_type, crawler).info('到底啦~~~~~~~~~~~~~\n')
                cls.i = 0
                cls.videos_cnt = 0
                driver.quit()
                return
            for i, video_element in enumerate(video_element_temp):
                try:
                    if cls.videos_cnt >= cls.videos_cnt_rule(log_type, crawler):
                        Common.logger(log_type, crawler).info(f"搜索词: {search_word}，已下载视频数: {cls.videos_cnt}\n")
                        cls.i = 0
                        cls.videos_cnt = 0
                        driver.quit()
                        return
                    # Common.logger(log_type, crawler).info(f"i:{i}, video_element:{video_element}")
                    if video_element is None:
                        Common.logger(log_type, crawler).info('到底啦~\n')
                        cls.i = 0
                        cls.videos_cnt = 0
                        driver.quit()
                        return
                    cls.i += 1
                    Common.logger(log_type, crawler).info(f'拖动"视频"列表第{cls.i}个至屏幕中间')
                    # Common.logger(log_type, crawler).info(f"video_elements:{len(video_elements)}")
                    # Common.logger(log_type, crawler).info(f"index+i:{index+i}")
                    driver.execute_script("arguments[0].scrollIntoView({block:'center',inline:'center'})", video_element)
                    time.sleep(1)
                    item_id = video_element.find_elements(By.XPATH, '//*[@class="HorizontalFeedCard__coverWrapper disableZoomAnimation"]')[index+i].get_attribute('href')
                    item_id = item_id.split("com/")[-1].split("?&")[0]
                    video_dict = cls.get_video_info(log_type, crawler, item_id)
                    if video_dict is None:
                        Common.logger(log_type, crawler).info("无效视频")
                    else:
                        for k, v in video_dict.items():
                            Common.logger(log_type, crawler).info(f"{k}:{v}")
                        rule_dict = cls.get_rule_dict(log_type, crawler)
                        if cls.download_rule(log_type=log_type, crawler=crawler, video_dict=video_dict, rule_dict=rule_dict) is False:
                            Common.logger(log_type, crawler).info("不满足抓取规则\n")
                        elif any(str(word) if str(word) in video_dict["video_title"] else False for word in cls.filter_words(log_type, crawler, env)) is True:
                            Common.logger(log_type, crawler).info("已中过滤词\n")
                        elif cls.repeat_video(log_type, crawler, video_dict["video_id"], env) != 0:
                            Common.logger(log_type, crawler).info("视频已下载\n")
                        else:
                            cls.download_publish(log_type=log_type,
                                                 crawler=crawler,
                                                 search_word=search_word,
                                                 video_dict=video_dict,
                                                 rule_dict=rule_dict,
                                                 our_uid=our_uid,
                                                 env=env)
                except Exception as e:
                    Common.logger(log_type, crawler).warning(f"抓取单条视频异常:{e}\n")

            Common.logger(log_type, crawler).info('已抓取完一组视频，休眠3秒\n')
            time.sleep(3)
            index = index + len(video_element_temp)

    @classmethod
    def repeat_video(cls, log_type, crawler, video_id, env):
        sql = f""" select * from crawler_video where platform="{cls.platform}" and out_video_id="{video_id}"; """
        repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env, action="")
        return len(repeat_video)

    # 下载 / 上传
    @classmethod
    def download_publish(cls, log_type, crawler, search_word, video_dict, rule_dict, our_uid, env):

        Common.download_method(log_type=log_type, crawler=crawler, text='xigua_video',
                               title=video_dict['video_title'], url=video_dict['video_url'])
        # 下载音频
        Common.download_method(log_type=log_type, crawler=crawler, text='xigua_audio',
                               title=video_dict['video_title'], url=video_dict['audio_url'])
        # 合成音视频
        Common.video_compose(log_type=log_type, crawler=crawler,
                             video_dir=f"./{crawler}/videos/{video_dict['video_title']}")
        md_title = md5(video_dict['video_title'].encode('utf8')).hexdigest()
        try:
            if os.path.getsize(f"./{crawler}/videos/{md_title}/video.mp4") == 0:
                # 删除视频文件夹
                shutil.rmtree(f"./{crawler}/videos/{md_title}")
                Common.logger(log_type, crawler).info("视频size=0，删除成功\n")
                return
        except FileNotFoundError:
            # 删除视频文件夹
            shutil.rmtree(f"./{crawler}/videos/{md_title}")
            Common.logger(log_type, crawler).info("视频文件不存在，删除文件夹成功\n")
            return
        # 下载封面
        Common.download_method(log_type=log_type, crawler=crawler, text='cover',
                               title=video_dict['video_title'], url=video_dict['cover_url'])
        # 保存视频信息至txt
        Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_dict)

        # 上传视频
        Common.logger(log_type, crawler).info("开始上传视频...")
        if env == "dev":
            oss_endpoint = "out"
        else:
            oss_endpoint = "inner"
        our_video_id = Publish.upload_and_publish(log_type=log_type,
                                                  crawler=crawler,
                                                  strategy="搜索爬虫策略",
                                                  our_uid=our_uid,
                                                  env=env,
                                                  oss_endpoint=oss_endpoint)
        if env == 'dev':
            our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
        else:
            our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
        Common.logger(log_type, crawler).info("视频上传完成")

        if our_video_id is None:
            try:
                # 删除视频文件夹
                shutil.rmtree(f"./{crawler}/videos/{md_title}")
                return
            except FileNotFoundError:
                return

        # 视频信息保存数据库
        insert_sql = f""" insert into crawler_video(video_id,
                                user_id,
                                out_user_id,
                                platform,
                                strategy,
                                out_video_id,
                                video_title,
                                cover_url,
                                video_url,
                                duration,
                                publish_time,
                                play_cnt,
                                crawler_rule,
                                width,
                                height)
                                values({our_video_id},
                                {our_uid},
                                "{video_dict['user_id']}",
                                "{cls.platform}",
                                "搜索爬虫策略",
                                "{video_dict['video_id']}",
                                "{video_dict['video_title']}",
                                "{video_dict['cover_url']}",
                                "{video_dict['video_url']}",
                                {int(video_dict['duration'])},
                                "{video_dict['publish_time_str']}",
                                {int(video_dict['play_cnt'])},
                                '{json.dumps(rule_dict)}',
                                {int(video_dict['video_width'])},
                                {int(video_dict['video_height'])}) """
        Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
        MysqlHelper.update_values(log_type, crawler, insert_sql, env, action="")
        Common.logger(log_type, crawler).info("视频信息写入数据库完成")

        # 视频信息写入飞书
        Feishu.insert_columns(log_type, crawler, "BUNvGC", "ROWS", 1, 2)
        values = [[
            search_word,
            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))),
            "关键词搜索",
            video_dict['video_title'],
            str(video_dict['video_id']),
            our_video_link,
            video_dict['gid'],
            video_dict['play_cnt'],
            video_dict['comment_cnt'],
            video_dict['like_cnt'],
            video_dict['share_cnt'],
            video_dict['duration'],
            str(video_dict['video_width']) + '*' + str(video_dict['video_height']),
            video_dict['publish_time_str'],
            video_dict['user_name'],
            video_dict['user_id'],
            video_dict['avatar_url'],
            video_dict['cover_url'],
            video_dict['video_url'],
            video_dict['audio_url']]]
        time.sleep(0.5)
        Feishu.update_values(log_type, crawler, "BUNvGC", "E2:Z2", values)
        Common.logger(log_type, crawler).info('视频信息写入飞书完成\n')
        cls.videos_cnt += 1

    @classmethod
    def get_search_videos(cls, log_type, crawler, env):
        user_list = cls.get_user_list(log_type=log_type, crawler=crawler, sheetid="SSPNPW", env=env)
        for user in user_list:
            try:
                cls.i = 0
                cls.videos_cnt = 0
                search_word = user["search_word"]
                our_uid = user["our_uid"]
                Common.logger(log_type, crawler).info(f"开始抓取 {search_word} 视频\n")
                cls.get_videoList(log_type=log_type,
                                  crawler=crawler,
                                  search_word=search_word,
                                  our_uid=our_uid,
                                  env=env)
            except Exception as e:
                Common.logger(log_type, crawler).error(f"get_search_videos:{e}\n")


if __name__ == '__main__':
    # XiguaSearch.get_search_videos('search', 'xigua', 'dev')
    # XiguaSearch.get_videoList("search", "xigua", "长寿食物", "dev")
    # XiguaSearch.get_video_info("search", "xigua", "7027495456829768196")
    # print(XiguaSearch.get_comment_cnt("7027495456829768196"))
    # print(XiguaSearch.videos_cnt_rule("search", "xigua"))
    # XiguaSearch.filter_words('search', 'xigua', 'dev')
    # print(XiguaSearchNew.get_rule_dict('search', 'xigua'))
    # os.system("ps aux | grep Chrome | grep -v grep | awk '{print $2}' | xargs kill -9")
    pass