piaoquan_crawler/xiaoniangao/xiaoniangao_hour/xiaoniangao_hour_scheduling.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2023/3/15
import datetime
import json
import os
import random
import shutil
import sys
import time
from hashlib import md5
import requests
import urllib3
from common.mq import MQ
sys.path.append(os.getcwd())
from common.common import Common
from common.feishu import Feishu
from common.publish import Publish
from common.scheduling_db import MysqlHelper
from common.public import get_config_from_mysql, download_rule
proxies = {"http": None, "https": None}


class XiaoniangaoHourScheduling:
    platform = "小年糕"
    words = "abcdefghijklmnopqrstuvwxyz0123456789"
    uid_token_dict = {
        "uid": f"""{"".join(random.sample(words, 8))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 12))}""",
        "token": "".join(random.sample(words, 32))
    }

    @classmethod
    def repeat_video(cls, log_type, crawler, video_id, env):
        sql = f""" select * from crawler_video where platform="小年糕" and out_video_id="{video_id}"; """
        repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
        return len(repeat_video)

    @classmethod
    def repeat_hour(cls, log_type, crawler, video_id, env):
        sql = f""" select * from crawler_xiaoniangao_hour where platform="小年糕" and out_video_id="{video_id}"; """
        repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env)
        return len(repeat_video)

    # 获取列表
    @classmethod
    def get_videoList(cls, log_type, crawler, rule_dict, env):
        uid_token_dict = cls.uid_token_dict
        url = "https://kapi-xng-app.xiaoniangao.cn/v2/trends/recommend"
        payload = "{\"topic_name\":\"recommend\",\"ext\":{\"current_item\":0,\"items\":[]},\"tag_id\":0,\"refresh\":false,\"share_width\":300,\"share_height\":240,\"staggered_style\":0,\"qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!750x500r\\/interlace\\/1\\/format\\/jpg\",\"topic_id\":2,\"h_qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!80x80r\\/interlace\\/1\\/format\\/jpg\",\"log\":{\"net\":\"wifi\",\"product\":\"xng\",\"uid\":\"2F310D09-5E32-5985-8644-3BCB6920E76F\",\"brand\":\"iPad\",\"page\":\"\",\"session_id\":\"71C77648-3224-4083-894C-B6282131F286\",\"resolution\":\"750*1334\",\"pf\":\"4\",\"app_version\":\"1.22.5\",\"device\":\"iPad Pro (12.9-inch) (3rd generation)\",\"os_version\":\"15.7\",\"idfa\":\"\",\"channel\":\"ios_app_store\"},\"token\":\"\"}"
        headers = {
            'Host': 'kapi-xng-app.xiaoniangao.cn',
            'content-type': 'application/json; charset=utf-8',
            'accept': 'application/json',
            'authorization': 'PsrUTBCQ5G7UVZdgx+JxymPHcKU=',
            'verb': 'POST',
            'content-md5': '08fa0e6bf725fd6ef83c16d2ceb8a544',
            'x-b3-traceid': '45a6c5b4c471eecc',
            'accept-language': 'zh-Hans-CN;q=1.0',
            'date': 'Mon, 19 Jun 2023 09:47:40 GMT',
            'x-token-id': '',
            'x-signaturemethod': 'hmac-sha1',
            'user-agent': 'xngapp/1.22.5 (cn.xiaoniangao.xngapp; build:157; iOS 15.7.0) Alamofire/5.2.2'
        }
        urllib3.disable_warnings()
        r = requests.post(url=url, headers=headers, data=payload, proxies=proxies, verify=False)
        if 'data' not in r.text or r.status_code != 200:
            Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n")
            Common.logging(log_type, crawler, env, f"get_videoList:{r.text}\n")
            return
        elif "data" not in r.json():
            Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()}\n")
            Common.logging(log_type, crawler, env, f"get_videoList:{r.json()}\n")
            return
        elif "list" not in r.json()["data"]:
            Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']}\n")
            Common.logging(log_type, crawler, env, f"get_videoList:{r.json()['data']}\n")
            return
        elif len(r.json()['data']['list']) == 0:
            Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']['list']}\n")
            Common.logging(log_type, crawler, env, f"get_videoList:{r.json()['data']['list']}\n")
            return
        else:
            # 视频列表数据
            feeds = r.json()["data"]["list"]
            for i in range(len(feeds)):
                try:
                    # 标题，表情随机加在片头、片尾，或替代句子中间的标点符号
                    xiaoniangao_title = feeds[i].get("title", "").strip().replace("\n", "") \
                            .replace("/", "").replace("\r", "").replace("#", "") \
                            .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                            .replace(":", "").replace("*", "").replace("？", "") \
                            .replace("?", "").replace('"', "").replace("<", "") \
                            .replace(">", "").replace("|", "").replace(" ", "")\
                            .replace('"', '').replace("'", '')
                    # 随机取一个表情/符号
                    emoji = random.choice(get_config_from_mysql(log_type, crawler, env, "emoji"))
                    # 生成最终标题，标题list[表情+title, title+表情]随机取一个
                    video_title = random.choice([f"{emoji}{xiaoniangao_title}", f"{xiaoniangao_title}{emoji}"])
                    # 发布时间
                    publish_time_stamp = int(int(feeds[i].get("t", 0))/1000)
                    publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))
                    # 用户名 / 头像
                    user_name = feeds[i].get("user", {}).get("nick", "").strip().replace("\n", "") \
                            .replace("/", "").replace(" ", "") \
                            .replace(" ", "").replace("&NBSP", "").replace("\r", "")

                    video_dict = {
                        "video_title": video_title,
                        "video_id": feeds[i].get("vid", ""),
                        "duration": int(feeds[i].get("du", 0)/1000),
                        "play_cnt": feeds[i].get("play_pv", 0),
                        "like_cnt": feeds[i].get("favor", {}).get("total", 0),
                        "comment_cnt": feeds[i].get("comment_count", 0),
                        "share_cnt": feeds[i].get("share", 0),
                        "user_name": user_name,
                        "publish_time_stamp": publish_time_stamp,
                        "publish_time_str": publish_time_str,
                        "video_width": int(feeds[i].get("vw", 0)),
                        "video_height": int(feeds[i].get("vh", 0)),
                        "avatar_url": feeds[i].get("user", {}).get("hurl", ""),
                        "profile_id": feeds[i]["id"],
                        "profile_mid": feeds[i]["user"]["mid"],
                        "cover_url": feeds[i].get("url", ""),
                        "video_url": feeds[i].get("v_url", ""),
                        "session": f"xiaoniangao-hour-{int(time.time())}"
                    }
                    for k, v in video_dict.items():
                        Common.logger(log_type, crawler).info(f"{k}:{v}")
                    Common.logging(log_type, crawler, env, f"{video_dict}")
                    # 过滤无效视频
                    if video_title == "" or video_dict["video_id"] == "" or video_dict["video_url"] == "":
                        Common.logger(log_type, crawler).warning("无效视频\n")
                        Common.logging(log_type, crawler, env, "无效视频\n")
                    # 抓取基础规则过滤
                    elif download_rule(log_type=log_type, crawler=crawler, video_dict=video_dict, rule_dict=rule_dict) is False:
                        Common.logger(log_type, crawler).info("不满足抓取规则\n")
                        Common.logging(log_type, crawler, env, "不满足抓取规则\n")
                    elif any(str(word) if str(word) in video_dict["video_title"] else False
                             for word in get_config_from_mysql(log_type=log_type,
                                                               source=crawler,
                                                               env=env,
                                                               text="filter",
                                                               action="")) is True:
                        Common.logger(log_type, crawler).info('已中过滤词\n')
                        Common.logging(log_type, crawler, env, '已中过滤词\n')
                    elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env) != 0:
                        Common.logger(log_type, crawler).info('视频已下载\n')
                        Common.logging(log_type, crawler, env, '视频已下载\n')
                    else:
                        # 写入飞书小时级feeds数据库表
                        insert_sql = f""" insert into crawler_xiaoniangao_hour(profile_id,
                        profile_mid,
                        platform,
                        out_video_id,
                        video_title,
                        user_name,
                        cover_url,
                        video_url,
                        duration,
                        publish_time,
                        play_cnt,
                        crawler_time_stamp,
                        crawler_time)
                        values({video_dict["profile_id"]},
                        {video_dict["profile_mid"]},
                        "{cls.platform}",
                        "{video_dict["video_id"]}",
                        "{video_title}",
                        "{user_name}",
                        "{video_dict["cover_url"]}",
                        "{video_dict["video_url"]}",
                        {video_dict["duration"]},
                        "{publish_time_str}",
                        {video_dict["play_cnt"]},
                        {int(time.time())},
                        "{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time())))}"
                        )"""
                        Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
                        Common.logging(log_type, crawler, env, f"insert_sql:{insert_sql}")
                        MysqlHelper.update_values(log_type, crawler, insert_sql, env)
                        Common.logger(log_type, crawler).info('视频信息写入小时级数据库成功！\n')
                        Common.logging(log_type, crawler, env, '视频信息写入小时级数据库成功！\n')
                except Exception as e:
                    Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n")
                    Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n")

    @classmethod
    def get_video_info(cls, log_type, crawler, p_id, p_mid, v_title, v_id):
        uid_token_dict = cls.uid_token_dict
        url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id"
        headers = {
            "x-b3-traceid": '1c403a4aa72e3c',
            "X-Token-Id": 'ab619e96d801f1567388629260aa68ec-1202200806',
            "uid": uid_token_dict['uid'],
            "content-type": "application/json",
            "Accept-Encoding": "gzip,compress,br,deflate",
            "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                          ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                          'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
            "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/624/page-frame.html'
        }
        data = {
            "play_src": "1",
            "profile_id": int(p_id),
            "profile_mid": int(p_mid),
            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/"
                  "!400x400r/crop/400x400/interlace/1/format/jpg",
            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail"
                    "/!80x80r/crop/80x80/interlace/1/format/jpg",
            "share_width": 625,
            "share_height": 500,
            "no_comments": True,
            "no_follow": True,
            "vid": v_id,
            "hot_l1_comment": True,
            "token": uid_token_dict['token'],
            "uid": uid_token_dict['uid'],
            "proj": "ma",
            "wx_ver": "8.0.20",
            "code_ver": "3.62.0",
            "log_common_params": {
                "e": [{
                    "data": {
                        "page": "dynamicSharePage"
                    }
                }],
                "ext": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.3",
                    "net": "wifi",
                    "scene": "1089"
                },
                "pj": "1",
                "pf": "2",
                "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29"
            }
        }
        urllib3.disable_warnings()
        r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False)
        if r.status_code != 200 or 'data' not in r.text:
            Common.logger(log_type, crawler).warning(f"get_videoInfo:{r.text}\n")
        else:
            hour_play_cnt = r.json()["data"]["play_pv"]
            hour_cover_url = r.json()["data"]["url"]
            hour_video_url = r.json()["data"]["v_url"]
            hour_video_duration = r.json()["data"]["du"]
            hour_video_comment_cnt = r.json()["data"]["comment_count"]
            hour_video_like_cnt = r.json()["data"]["favor"]["total"]
            hour_video_share_cnt = r.json()["data"]["share"]
            hour_video_width = r.json()["data"]["w"]
            hour_video_height = r.json()["data"]["h"]
            hour_video_send_time = r.json()["data"]["t"]
            publish_time_stamp = int(int(hour_video_send_time) / 1000)
            publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp))
            hour_user_name = r.json()["data"]["user"]["nick"]
            hour_head_url = r.json()["data"]["user"]["hurl"]
            video_info_dict = {
                "video_id": v_id,
                "video_title": v_title,
                "duration": hour_video_duration,
                "play_cnt": hour_play_cnt,
                "like_cnt": hour_video_like_cnt,
                "comment_cnt": hour_video_comment_cnt,
                "share_cnt": hour_video_share_cnt,
                "user_name": hour_user_name,
                "publish_time_stamp": publish_time_stamp,
                "publish_time_str": publish_time_str,
                "video_width": hour_video_width,
                "video_height": hour_video_height,
                "avatar_url": hour_head_url,
                "profile_id": p_id,
                "profile_mid": p_mid,
                "cover_url": hour_cover_url,
                "video_url": hour_video_url,
                "session": f"xiaoniangao-hour-{int(time.time())}"
            }
            return video_info_dict

    # 更新小时榜数据
    @classmethod
    def update_videoList(cls, log_type, crawler, rule_dict, our_uid, env):
        """
        更新小时榜数据
        """
        befor_yesterday = (datetime.date.today() + datetime.timedelta(days=-3)).strftime("%Y-%m-%d %H:%M:%S")
        update_time_stamp = int(time.mktime(time.strptime(befor_yesterday, "%Y-%m-%d %H:%M:%S")))
        select_sql = f""" select * from crawler_xiaoniangao_hour where crawler_time_stamp>={update_time_stamp} GROUP BY out_video_id DESC """
        update_video_list = MysqlHelper.get_values(log_type, crawler, select_sql, env)
        if len(update_video_list) == 0:
            Common.logger(log_type, crawler).info("暂无需要更新的小时榜数据\n")
            Common.logging(log_type, crawler, env, "暂无需要更新的小时榜数据\n")
            return
        for update_video_info in update_video_list:
            try:
                profile_id = update_video_info["profile_id"]
                profile_mid = update_video_info["profile_mid"]
                video_title = update_video_info["video_title"]
                video_id = update_video_info["out_video_id"]
                if datetime.datetime.now().hour == 10 and datetime.datetime.now().minute <= 10:
                    video_info_dict = cls.get_video_info(log_type=log_type,
                                                         crawler=crawler,
                                                         p_id=profile_id,
                                                         p_mid=profile_mid,
                                                         v_title=video_title,
                                                         v_id=video_id)
                    ten_play_cnt = video_info_dict['play_cnt']
                    Common.logger(log_type, crawler).info(f"ten_play_cnt:{ten_play_cnt}")
                    Common.logging(log_type, crawler, env, f"ten_play_cnt:{ten_play_cnt}")
                    update_sql = f""" update crawler_xiaoniangao_hour set ten_play_cnt={ten_play_cnt} WHERE out_video_id="{video_id}"; """
                    # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}")
                    MysqlHelper.update_values(log_type, crawler, update_sql, env)
                    cls.download_publish(log_type=log_type,
                                         crawler=crawler,
                                         video_info_dict=video_info_dict,
                                         rule_dict=rule_dict,
                                         update_video_info=update_video_info,
                                         our_uid=our_uid,
                                         env=env)
                elif datetime.datetime.now().hour == 15 and datetime.datetime.now().minute <= 10:
                    video_info_dict = cls.get_video_info(log_type=log_type,
                                                         crawler=crawler,
                                                         p_id=profile_id,
                                                         p_mid=profile_mid,
                                                         v_title=video_title,
                                                         v_id=video_id)
                    fifteen_play_cnt = video_info_dict['play_cnt']
                    Common.logger(log_type, crawler).info(f"fifteen_play_cnt:{fifteen_play_cnt}")
                    Common.logging(log_type, crawler, env, f"fifteen_play_cnt:{fifteen_play_cnt}")
                    update_sql = f""" update crawler_xiaoniangao_hour set fifteen_play_cnt={fifteen_play_cnt} WHERE out_video_id="{video_id}"; """
                    # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}")
                    MysqlHelper.update_values(log_type, crawler, update_sql, env)
                    cls.download_publish(log_type=log_type,
                                         crawler=crawler,
                                         video_info_dict=video_info_dict,
                                         rule_dict=rule_dict,
                                         update_video_info=update_video_info,
                                         our_uid=our_uid,
                                         env=env)
                elif datetime.datetime.now().hour == 20 and datetime.datetime.now().minute <= 10:
                    video_info_dict = cls.get_video_info(log_type=log_type,
                                                         crawler=crawler,
                                                         p_id=profile_id,
                                                         p_mid=profile_mid,
                                                         v_title=video_title,
                                                         v_id=video_id)
                    twenty_play_cnt = video_info_dict['play_cnt']
                    Common.logger(log_type, crawler).info(f"twenty_play_cnt:{twenty_play_cnt}")
                    Common.logging(log_type, crawler, env, f"twenty_play_cnt:{twenty_play_cnt}")
                    update_sql = f""" update crawler_xiaoniangao_hour set twenty_play_cnt={twenty_play_cnt} WHERE out_video_id="{video_id}"; """
                    # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}")
                    MysqlHelper.update_values(log_type, crawler, update_sql, env)
                    cls.download_publish(log_type=log_type,
                                         crawler=crawler,
                                         video_info_dict=video_info_dict,
                                         rule_dict=rule_dict,
                                         update_video_info=update_video_info,
                                         our_uid=our_uid,
                                         env=env)
                else:
                    pass
            except Exception as e:
                Common.logger(log_type, crawler).error(f'更新{update_video_info["video_title"]}时异常:{e}\n')
                Common.logging(log_type, crawler, env, f'更新{update_video_info["video_title"]}时异常:{e}\n')

    @classmethod
    def send_to_mq(cls, log_type, crawler, video_info_dict, rule_dict, env):
        video_info_dict["out_user_id"] = video_info_dict["profile_id"]
        video_info_dict["platform"] = crawler
        video_info_dict["strategy"] = log_type
        video_info_dict["out_video_id"] = video_info_dict["video_id"]
        video_info_dict["width"] = video_info_dict["video_width"]
        video_info_dict["height"] = video_info_dict["video_height"]
        video_info_dict["crawler_rule"] = json.dumps(rule_dict)
        video_info_dict["user_id"] = video_info_dict["uid"]
        video_info_dict["publish_time"] = video_info_dict["publish_time_str"]
        video_info_dict["strategy_type"] = "hour"

        mq = MQ(topic_name="topic_crawler_etl_" + env)
        mq.send_msg(video_info_dict)

    @classmethod
    def download(cls, log_type, crawler, video_info_dict, rule_dict, our_uid, env):
        # 下载视频
        Common.download_method(log_type=log_type, crawler=crawler, text="video", title=video_info_dict["video_title"],
                               url=video_info_dict["video_url"])
        md_title = md5(video_info_dict['video_title'].encode('utf8')).hexdigest()
        try:
            if os.path.getsize(f"./{crawler}/videos/{md_title}/video.mp4") == 0:
                # 删除视频文件夹
                shutil.rmtree(f"./{crawler}/videos/{md_title}")
                Common.logger(log_type, crawler).info("视频size=0，删除成功\n")
                Common.logging(log_type, crawler, env, "视频size=0，删除成功\n")
                return
        except FileNotFoundError:
            # 删除视频文件夹
            shutil.rmtree(f"./{crawler}/videos/{md_title}")
            Common.logger(log_type, crawler).info("视频文件不存在，删除文件夹成功\n")
            Common.logging(log_type, crawler, env, "视频文件不存在，删除文件夹成功\n")
            return
        # 下载封面
        Common.download_method(log_type=log_type, crawler=crawler, text="cover", title=video_info_dict["video_title"],
                               url=video_info_dict["cover_url"])
        # 保存视频信息至 "./videos/{download_video_title}/info.txt"
        Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_info_dict)

        # 上传视频
        Common.logger(log_type, crawler).info("开始上传视频...")
        Common.logging(log_type, crawler, env, "开始上传视频...")
        if env == "dev":
            oss_endpoint = "out"
            our_video_id = Publish.upload_and_publish(log_type=log_type,
                                                      crawler=crawler,
                                                      strategy="上升榜抓取策略",
                                                      our_uid=our_uid,
                                                      env=env,
                                                      oss_endpoint=oss_endpoint)
            our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info"
        else:
            oss_endpoint = "inner"
            our_video_id = Publish.upload_and_publish(log_type=log_type,
                                                      crawler=crawler,
                                                      strategy="上升榜抓取策略",
                                                      our_uid=our_uid,
                                                      env=env,
                                                      oss_endpoint=oss_endpoint)

            our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info"

        if our_video_id is None:
            try:
                # 删除视频文件夹
                shutil.rmtree(f"./{crawler}/videos/{md_title}")
                return
            except FileNotFoundError:
                return

        insert_sql = f""" insert into crawler_video(video_id,
                                                    out_user_id,
                                                    platform,
                                                    strategy,
                                                    out_video_id,
                                                    video_title,
                                                    cover_url,
                                                    video_url,
                                                    duration,
                                                    publish_time,
                                                    play_cnt,
                                                    crawler_rule,
                                                    width,
                                                    height)
                                                    values({our_video_id},
                                                    "{video_info_dict['profile_id']}",
                                                    "{cls.platform}",
                                                    "上升榜抓取策略",
                                                    "{video_info_dict['video_id']}",
                                                    "{video_info_dict['video_title']}",
                                                    "{video_info_dict['cover_url']}",
                                                    "{video_info_dict['video_url']}",
                                                    {int(video_info_dict['duration'])},
                                                    "{video_info_dict['publish_time_str']}",
                                                    {int(video_info_dict['play_cnt'])},
                                                    '{json.dumps(rule_dict)}',
                                                    {int(video_info_dict['video_width'])},
                                                    {int(video_info_dict['video_height'])}) """
        Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}")
        Common.logging(log_type, crawler, env, f"insert_sql:{insert_sql}")
        MysqlHelper.update_values(log_type, crawler, insert_sql, env)
        Common.logger(log_type, crawler).info('视频信息插入数据库成功！')
        Common.logging(log_type, crawler, env, '视频信息插入数据库成功！')

        # 视频写入飞书
        Feishu.insert_columns(log_type, crawler, "yatRv2", "ROWS", 1, 2)
        # 视频ID工作表，首行写入数据
        upload_time = int(time.time())
        values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)),
                   "上升榜抓取策略",
                   str(video_info_dict['video_id']),
                   str(video_info_dict['video_title']),
                   our_video_link,
                   video_info_dict['play_cnt'],
                   video_info_dict['comment_cnt'],
                   video_info_dict['like_cnt'],
                   video_info_dict['share_cnt'],
                   video_info_dict['duration'],
                   f"{video_info_dict['video_width']}*{video_info_dict['video_height']}",
                   str(video_info_dict['publish_time_str'].replace("-", "/")),
                   str(video_info_dict['user_name']),
                   str(video_info_dict['profile_id']),
                   str(video_info_dict['profile_mid']),
                   str(video_info_dict['avatar_url']),
                   str(video_info_dict['cover_url']),
                   str(video_info_dict['video_url'])]]
        time.sleep(1)
        Feishu.update_values(log_type, crawler, "yatRv2", "F2:Z2", values)
        Common.logger(log_type, crawler).info('视频信息写入飞书成功\n')
        Common.logging(log_type, crawler, env, '视频信息写入飞书成功\n')

    # 下载/上传
    @classmethod
    def download_publish(cls, log_type, crawler, video_info_dict, rule_dict, update_video_info, our_uid, env):
        if cls.repeat_video(log_type, crawler, video_info_dict["video_id"], env) != 0:
            Common.logger(log_type, crawler).info('视频已下载\n')
            Common.logging(log_type, crawler, env, '视频已下载\n')
        # 播放量大于 50000，直接下载
        elif int(video_info_dict["play_cnt"]) >= 30000:
            Common.logger(log_type, crawler).info(f"播放量:{video_info_dict['play_cnt']} >= 30000，满足下载规则，开始下载视频")
            Common.logging(log_type, crawler, env, f"播放量:{video_info_dict['play_cnt']} >= 30000，满足下载规则，开始下载视频")
            # cls.download(log_type=log_type,
            #              crawler=crawler,
            #              video_info_dict=video_info_dict,
            #              rule_dict=rule_dict,
            #              our_uid=our_uid,
            #              env=env)
            cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env)

        # 上升榜判断逻辑，任意时间段上升量>=5000，连续两个时间段上升量>=2000
        elif int(update_video_info['ten_play_cnt']) >= 3000 or int(
                update_video_info['fifteen_play_cnt']) >= 3000 or int(update_video_info['twenty_play_cnt']) >= 3000:
            Common.logger(log_type, crawler).info(f"10:00 or 15:00 or 20:00 数据上升量:{int(update_video_info['ten_play_cnt'])} or {int(update_video_info['fifteen_play_cnt'])} or {int(update_video_info['twenty_play_cnt'])} >= 3000")
            Common.logging(log_type, crawler, env, f"10:00 or 15:00 or 20:00 数据上升量:{int(update_video_info['ten_play_cnt'])} or {int(update_video_info['fifteen_play_cnt'])} or {int(update_video_info['twenty_play_cnt'])} >= 3000")
            Common.logger(log_type, crawler).info("满足下载规则，开始下载视频")
            Common.logging(log_type, crawler, env, "满足下载规则，开始下载视频")
            # cls.download(log_type=log_type,
            #              crawler=crawler,
            #              video_info_dict=video_info_dict,
            #              rule_dict=rule_dict,
            #              our_uid=our_uid,
            #              env=env)
            cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env)

        elif int(update_video_info['ten_play_cnt']) >= 1000 and int(update_video_info['fifteen_play_cnt']) >= 1000:
            Common.logger(log_type, crawler).info(f"10:00 and 15:00 数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['fifteen_play_cnt'])} >= 1000")
            Common.logging(log_type, crawler, env, f"10:00 and 15:00 数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['fifteen_play_cnt'])} >= 1000")
            Common.logger(log_type, crawler).info("满足下载规则，开始下载视频")
            Common.logging(log_type, crawler, env, "满足下载规则，开始下载视频")
            # cls.download(log_type=log_type,
            #              crawler=crawler,
            #              video_info_dict=video_info_dict,
            #              rule_dict=rule_dict,
            #              our_uid=our_uid,
            #              env=env)
            cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env)

        elif int(update_video_info['fifteen_play_cnt']) >= 1000 and int(update_video_info['twenty_play_cnt']) >= 1000:
            Common.logger(log_type, crawler).info(
                f"15:00 and 20:00 数据上升量:{int(update_video_info['fifteen_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 1000")
            Common.logging(log_type, crawler, env, f"15:00 and 20:00 数据上升量:{int(update_video_info['fifteen_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 1000")
            Common.logger(log_type, crawler).info("满足下载规则，开始下载视频")
            Common.logging(log_type, crawler, env, "满足下载规则，开始下载视频")
            # cls.download(log_type=log_type,
            #              crawler=crawler,
            #              video_info_dict=video_info_dict,
            #              rule_dict=rule_dict,
            #              our_uid=our_uid,
            #              env=env)
            cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env)

        elif int(update_video_info['ten_play_cnt']) >= 1000 and int(update_video_info['twenty_play_cnt']) >= 1000:
            Common.logger(log_type, crawler).info(
                f"今日10:00 / 20:00数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 1000")
            Common.logging(log_type, crawler, env, f"今日10:00 / 20:00数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 1000")
            Common.logger(log_type, crawler).info("满足下载规则，开始下载视频")
            Common.logging(log_type, crawler, env, "满足下载规则，开始下载视频")
            # cls.download(log_type=log_type,
            #              crawler=crawler,
            #              video_info_dict=video_info_dict,
            #              rule_dict=rule_dict,
            #              our_uid=our_uid,
            #              env=env)
            cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env)

        else:
            Common.logger(log_type, crawler).info("上升量不满足下载规则")
            Common.logging(log_type, crawler, env, "上升量不满足下载规则")


if __name__ == "__main__":
    print(get_config_from_mysql(log_type='hour', source='xiaoniangao', env='dev', text='filter'))

    pass