crawler_kanyikan/main/download_up.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/4/18

"""
下载并上传：上升榜视频
规则：
    1.满足基本规则
    2.每隔一小时，检查视频播放量，>=1000，则下载及上传
    3.超过 2 小时，则删除该视频信息
"""

import json
import os
import sys
import time
import requests
import urllib3
sys.path.append(os.getcwd())
from main.common import Common
from main.get_feeds import get_feeds
from main.publish import Publish
from main.feishu_lib import Feishu

proxies = {"http": None, "https": None}


class DownloadUp:

    @staticmethod
    def up_rule(up_width, up_height, up_duration, up_play_cnt):
        """
        1.分辨率，宽或者高 >= 720 or ==0
        2.600s >= 时长 >= 60s
        3.视频播放量 >= 0
        """
        if int(up_width) >= 720 or int(up_height) >= 720 or str(up_width) == "0" or str(up_height) == "0":
            if 600 >= int(up_duration) >= 60:
                if int(up_play_cnt) >= 0:
                    return True
                else:
                    return False
            else:
                return False
        else:
            return False

    @classmethod
    def download_up_video(cls, env):
        """
        1.从 kanyikan_feeds.txt 中获取 videoid
        2.根据 videoid，从 videoinfo 接口，获取当前视频最新的信息
        3.根据下载规则判断，符合规则进行下载：
            1 更新视频 ID 到 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
            2 视频信息写入文件 "./videos/{d_title}/info.txt"
        4.上传完成：
            1 删除该视频在 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM 中的信息
        """
        try:
            for i in range(1, len(Feishu.get_values_batch("SdCHOM"))+1):
                time.sleep(1)
                video_info_session = Common.get_session()
                download_time = Feishu.get_values_batch("SdCHOM")[i][0]  # 第一次获取该视频的时间
                download_time = int(time.mktime(time.strptime(download_time, "%Y/%m/%d %H:%M:%S")))
                download_video_id = Feishu.get_values_batch("SdCHOM")[i][2]  # 外网视频 ID
                download_video_title = Feishu.get_values_batch("SdCHOM")[i][3]  # 视频标题
                download_video_play_cnt = Feishu.get_values_batch("SdCHOM")[i][4]  # 播放量

                url = "https://search.weixin.qq.com/cgi-bin/recwxa/recwxagetonevideoinfo?"
                param = {
                    "session": video_info_session,
                    "vid": download_video_id,
                    "wxaVersion": "3.9.2",
                    "channelid": "208201",
                    "scene": "32",
                    "subscene": "1089",
                    "model": "iPhone 11<iPhone12,1>14.7.1",
                    "clientVersion": "8.0.18",
                    "sharesearchid": "447665862521758270",
                    "sharesource": "-1"
                }
                urllib3.disable_warnings()
                r = requests.get(url=url, params=param, proxies=proxies, verify=False)
                response = json.loads(r.content.decode("utf8"))
                if "data" not in response:
                    Common.logger().error("获取视频info时错误，删除该视频:{}", download_video_title)
                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
                else:
                    data = response["data"]
                    v_duration = data["duration"]
                    v_play_cnt_up = data["played_cnt"]
                    v_comment_cnt = data["comment_cnt"]
                    v_liked_cnt = data["liked_cnt"]
                    v_shared_cnt = data["shared_cnt"]
                    v_width = data["width"]
                    v_height = data["height"]
                    v_resolution = str(v_width) + "*" + str(v_height)
                    v_send_date = data["upload_time"]
                    v_username = data["user_info"]["nickname"].strip().replace("\n", "")
                    v_user_id = data["openid"]
                    v_user_cover = data["user_info"]["headimg_url"]
                    v_video_cover = data["cover_url"]
                    if "items" not in data["play_info"]:
                        if len(data["play_info"]) > 2:
                            download_url_up = data["play_info"][2]["play_url"]
                        else:
                            download_url_up = data["play_info"][0]["play_url"]
                    else:
                        if len(data["play_info"]["items"]) > 2:
                            download_url_up = data["play_info"]["items"][2]["play_url"]
                        else:
                            download_url_up = data["play_info"]["items"][0]["play_url"]

                    Common.logger().info("正在判断第{}行，视频:{}", i, download_video_title)

                    # 判断无效视频
                    if download_video_id == "" \
                            or download_video_id is None\
                            and download_video_title == ""\
                            or download_video_title is None\
                            and v_duration == "" \
                            and v_play_cnt_up == ""\
                            and v_send_date == ""\
                            and v_user_cover == "" \
                            and v_video_cover == ""\
                            and download_url_up == "":
                        Common.logger().info("无效视频，删除该视频信息:{}", download_video_title)
                        # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
                        cls.download_up_video("prod")

                    # 上升榜时长不足 1 小时
                    elif int(time.time()) - int(download_time) < 3600:
                        Common.logger().info("距上次获取该视频时间：{}分钟；{}",
                                             int((int(int(time.time()) - int(download_time))) / 60),
                                             download_video_title)

                    # 上升榜时长超过 2 小时
                    elif int(time.time()) - int(download_time) > 7200:
                        Common.logger().info("距上次获取该视频时间：""{}分钟。超过2小时，删除该视频；标题：{}",
                                             int((int(time.time()) - int(download_time)) / 60),
                                             download_video_title)
                        # 删除行或列，可选 ROWS、COLUMNS
                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
                        cls.download_up_video("prod")

                    # 上升榜下载规则
                    elif cls.up_rule(v_width, v_height, v_duration, v_play_cnt_up) is False:
                        Common.logger().info("不满足上升榜下载规则，删除视频:{}", download_video_title)
                        # 删除行或列，可选 ROWS、COLUMNS
                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
                        cls.download_up_video("prod")

                    # 从已下载视频表中去重:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
                    elif download_video_id in [j for m in Feishu.get_values_batch("20ce0c") for j in m]:
                        Common.logger().info("视频已下载，删除该视频信息:{}", download_video_title)
                        # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
                        cls.download_up_video("prod")

                    # 上升榜时长在 1-2 小时内，播放量增长>=1000
                    elif (7200 >= int(time.time()) - int(download_time) >= 3600)\
                            and (int(v_play_cnt_up) - int(download_video_play_cnt) >= 1000):
                        Common.logger().info(
                            "视频:{}，在上升榜时间内的播放量{}>=1000，开始下载视频",
                            download_video_title, int(v_play_cnt_up) - int(download_video_play_cnt))

                        # 下载封面
                        Common.download_method("cover", download_video_title, v_video_cover)
                        # 下载视频
                        Common.download_method("video", download_video_title, download_url_up)
                        # 保存视频信息到 "./files/{视频标题}/videoinfo.txt"
                        with open(r"./videos/" + download_video_title
                                  + "/" + "info.txt", "a", encoding="utf8") as f_a2:
                            f_a2.write(str(download_video_id) + "\n" +
                                       str(download_video_title) + "\n" +
                                       str(v_duration) + "\n" +
                                       str(v_play_cnt_up) + "\n" +
                                       str(v_comment_cnt) + "\n" +
                                       str(v_liked_cnt) + "\n" +
                                       str(v_shared_cnt) + "\n" +
                                       str(v_resolution) + "\n" +
                                       str(v_send_date) + "\n" +
                                       str(v_username) + "\n" +
                                       str(v_user_cover) + "\n" +
                                       str(download_url_up) + "\n" +
                                       str(v_video_cover) + "\n" +
                                       str(video_info_session))
                        Common.logger().info("==========视频信息已保存至info.txt==========")

                        # 上传该视频
                        Common.logger().info("开始上传视频:{}", download_video_title)
                        Publish.upload_and_publish(env, "up")

                        # 保存视频 ID 到云文档:
                        # https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
                        Common.logger().info("保存视频ID至云文档:{}", download_video_title)
                        # 看一看+ ，视频ID工作表，插入首行
                        Feishu.insert_columns("20ce0c", "ROWS", 1, 2)
                        # 看一看+ ，视频ID工作表，首行写入数据
                        upload_time = int(time.time())
                        values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
                                   "上升榜",
                                   str(download_video_id),
                                   str(download_video_title),
                                   v_play_cnt_up,
                                   v_comment_cnt,
                                   v_liked_cnt,
                                   v_shared_cnt,
                                   v_duration,
                                   v_resolution,
                                   time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(v_send_date)),
                                   str(v_username),
                                   str(v_user_id),
                                   str(v_user_cover),
                                   str(v_video_cover),
                                   str(download_url_up)]]
                        time.sleep(1)
                        Feishu.update_values("20ce0c", "A2:Q2", values)

                        # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
                        Common.logger().info("从云文档删除该视频信息:{}", download_video_title)
                        # 删除行或列，可选 ROWS、COLUMNS
                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
                        cls.download_up_video("prod")

                    # 上升榜时长在 1-2 小时内，播放量增长<1000
                    elif (7200 >= int(time.time()) - int(download_time) >= 3600)\
                            and (int(v_play_cnt_up) - int(download_video_play_cnt) < 1000):
                        # 删除之前保存的该视频信息，并把现在的信息保存进去
                        Common.logger().info("该视频1小时内的播放量：{}<1000；更新该视频信息:{}",
                                             int(v_play_cnt_up) - int(download_video_play_cnt),
                                             download_video_title)
                        # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
                        Common.logger().info("从云文档删除该视频信息:{}", download_video_title)
                        # 删除行或列，可选 ROWS、COLUMNS
                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)

                        # 看一看+工作表，插入首行
                        Feishu.insert_columns("SdCHOM", "ROWS", 1, 2)

                        # 获取当前时间
                        download_up_time = int(time.time())
                        values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(download_up_time)),
                                   "上升榜",
                                   str(download_video_id),
                                   str(download_video_title),
                                   v_play_cnt_up,
                                   v_comment_cnt,
                                   v_liked_cnt,
                                   v_shared_cnt,
                                   v_duration,
                                   v_resolution,
                                   time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(v_send_date)),
                                   str(v_username),
                                   str(v_user_id),
                                   str(v_user_cover),
                                   str(v_video_cover),
                                   str(download_url_up)]]
                        time.sleep(1)
                        Feishu.update_values("SdCHOM", "A2:Q2", values)

        except Exception as e:
            # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
            Common.logger().error("获取视频info异常:{}", e)


if __name__ == "__main__":
    downloadup = DownloadUp()
    get_feeds()
    downloadup.download_up_video("dev")