crawler_xiaoniangao/main/hour_list.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/5/16
import datetime
import time

import requests
import urllib3

from main.common import Common
from main.feishu_lib import Feishu

proxies = {"http": None, "https": None}


class HourList:
    # 今天的日期:年-月-日
    today = datetime.datetime.now().strftime("%Y-%m-%d")

    # 已下载小时榜视频列表
    download_hour_video_list = []

    # 下载规则
    @staticmethod
    def download_rule(d_duration, d_width, d_height, d_play_cnt, d_like_cnt, d_share_cnt):
        """
        下载视频的基本规则
        :param d_duration: 时长
        :param d_width: 宽
        :param d_height: 高
        :param d_play_cnt: 播放量
        :param d_like_cnt: 点赞量
        :param d_share_cnt: 分享量
        :return: 满足规则，返回 True；反之，返回 False
        """
        if 600 >= int(float(d_duration)) >= 60:
            if int(d_width) >= 0 or int(d_height) >= 0:
                if int(d_play_cnt) >= 0:
                    if int(d_like_cnt) >= 0:
                        if int(d_share_cnt) >= 0:
                            return True
                        else:
                            return False
                    else:
                        return False
                else:
                    return False
            return False
        return False

    # 获取列表
    @classmethod
    def get_hour_list_feeds(cls):
        """
        1.从列表获取视频，7 天内，播放量>=5000
        2.时长 1-10min
        3.每天10:00、15:00、20:00 把符合规则的视频，写入云文档
        https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=k2rKkv
        """
        url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends"
        headers = {
            "x-b3-traceid": "695dcad76ccca",
            "X-Token-Id": "af9c47bb6c942236ff35ee10d355f3b0-1164637358",
            "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0",
            "content-type": "application/json",
            "Accept-Encoding": "gzip,compress,br,deflate",
            "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                          ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                          'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
            "Referer": "https://servicewechat.com/wxd7911e4c177690e4/617/page-frame.html"
        }
        data = {
            "log_params": {
                "page": "discover_rec",
                "common": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.2",
                    "net": "wifi",
                    "scene": 1089
                }
            },
            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg",
            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg",
            "share_width": 625,
            "share_height": 500,
            "ext": {
                "fmid": 0,
                "items": {}
            },
            "app": "xng",
            "rec_scene": "discover_rec",
            "log_common_params": {
                "e": [{
                    "data": {
                        "page": "discoverIndexPage",
                        "topic": "recommend"
                    },
                    "ab": {}
                }],
                "ext": {
                    "brand": "iPhone",
                    "device": "iPhone 11",
                    "os": "iOS 14.7.1",
                    "weixinver": "8.0.20",
                    "srcver": "2.24.3",
                    "net": "wifi",
                    "scene": "1089"
                },
                "pj": "1",
                "pf": "2",
                "session_id": "1a20f033-3511-4e7b-9b2a-95e5c542461f"
            },
            "refresh": False,
            "token": "451273638af2c8bb90266bcfaf601a68",
            "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0",
            "proj": "ma",
            "wx_ver": "8.0.20",
            "code_ver": "3.62.0"
        }
        try:
            urllib3.disable_warnings()
            r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False)
            if "data" not in r.json():
                Common.logger().warning("获取视频feeds错误:{}", r.text)
            elif "list" not in r.json()["data"]:
                Common.logger().warning("获取视频feeds无数据，休眠10s:{}", r.json()["data"])
            else:
                # 视频列表数据
                feeds = r.json()["data"]["list"]
                for i in range(len(feeds)):
                    # 标题
                    if "title" in feeds[i]:
                        video_title = feeds[i]["title"].strip().replace("\n", "") \
                            .replace("/", "").replace("\r", "").replace("#", "") \
                            .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                            .replace(":", "").replace("*", "").replace("？", "") \
                            .replace("?", "").replace('"', "").replace("<", "") \
                            .replace(">", "").replace("|", "").replace(" ", "")
                        Common.logger().info("标题:{}", video_title)
                    else:
                        video_title = ""
                        Common.logger().info("当前视频无标题:{}", video_title)

                    # 视频 ID
                    if "vid" in feeds[i]:
                        video_id = feeds[i]["vid"]
                        Common.logger().info("视频ID:{}", video_id)
                    else:
                        video_id = ""
                        Common.logger().info("当前视频无ID:{}", video_id)

                    # 播放量
                    if "play_pv" in feeds[i]:
                        video_play_cnt = feeds[i]["play_pv"]
                        Common.logger().info("视频播放量:{}", video_play_cnt)
                    else:
                        video_play_cnt = ""
                        Common.logger().info("当前视频无播放量:{}", video_play_cnt)

                    # 点赞量
                    if "favor" in feeds[i]:
                        video_like_cnt = feeds[i]["favor"]["total"]
                        Common.logger().info("视频点赞量:{}", video_like_cnt)
                    else:
                        video_like_cnt = ""
                        Common.logger().info("当前视频无点赞量:{}", video_like_cnt)

                    # 分享量
                    if "share" in feeds[i]:
                        video_share_cnt = feeds[i]["share"]
                        Common.logger().info("视频分享量:{}", video_share_cnt)
                    else:
                        video_share_cnt = ""
                        Common.logger().info("当前视频无分享量:{}", video_share_cnt)

                    # 评论量
                    if "comment_count" in feeds[i]:
                        video_comment_cnt = feeds[i]["comment_count"]
                        Common.logger().info("视频评论数:{}", video_comment_cnt)
                    else:
                        video_comment_cnt = ""
                        Common.logger().info("当前视频无评论:{}", video_comment_cnt)

                    # 时长
                    if "du" in feeds[i]:
                        video_duration = int(feeds[i]["du"] / 1000)
                        Common.logger().info("视频时长:{}秒", video_duration)
                    else:
                        video_duration = ""
                        Common.logger().info("当前视频无时长:{}", video_duration)

                    # 宽和高
                    if "w" or "h" in feeds[i]:
                        video_width = feeds[i]["w"]
                        video_height = feeds[i]["h"]
                        Common.logger().info("视频宽高:{}*{}", video_width, video_height)
                    else:
                        video_width = ""
                        video_height = ""
                        Common.logger().info("当前视频无宽高:{}{}", video_width, video_height)

                    # 发布时间
                    if "t" in feeds[i]:
                        video_send_time = feeds[i]["t"]
                        Common.logger().info(
                            "视频发布时间:{}", time.strftime(
                                "%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)))
                    else:
                        video_send_time = ""
                        Common.logger().info("当前视频无发布时间:{}", video_send_time)

                    # 用户名 / 头像
                    if "user" in feeds[i]:
                        user_name = feeds[i]["user"]["nick"].strip().replace("\n", "") \
                            .replace("/", "").replace("快手", "").replace(" ", "") \
                            .replace(" ", "").replace("&NBSP", "").replace("\r", "")
                        head_url = feeds[i]["user"]["hurl"]
                        Common.logger().info("用户名:{}", user_name)
                        Common.logger().info("用户头像:{}", head_url)
                    else:
                        user_name = ""
                        head_url = ""
                        Common.logger().info("当前视频无用户名:{}", user_name)
                        Common.logger().info("当前视频无用户头像:{}", head_url)

                    # 用户 ID
                    profile_id = feeds[i]["id"]

                    # 用户 mid
                    profile_mid = feeds[i]["user"]["mid"]

                    # 视频封面
                    if "url" in feeds[i]:
                        cover_url = feeds[i]["url"]
                        Common.logger().info("视频封面:{}", cover_url)
                    else:
                        cover_url = ""
                        Common.logger().info("当前视频无视频封面:{}", cover_url)

                    # 视频播放地址
                    if "v_url" in feeds[i]:
                        video_url = feeds[i]["v_url"]
                        Common.logger().info("播放地址:{}", video_url)
                    else:
                        video_url = ""
                        Common.logger().info("当前视频无播放地址:{}", video_url)

                    # 过滤无效视频
                    if video_title == "" or video_id == "" or video_duration == "" \
                            or video_send_time == "" or user_name == "" or head_url == "" \
                            or cover_url == "" or video_url == "":
                        Common.logger().warning("无效视频")
                    # 判断发布时间是否 > 7天
                    # elif int(time.time()) - int(video_send_time)/1000 > 2592000:
                    #     Common.logger().info("发布时间大于30天", video_title)
                    elif int(time.time()) - int(video_send_time)/1000 > 604800:
                        Common.logger().info("发布时间大于7天", video_title)
                    # 判断播放量是否 > 5000
                    # elif int(video_play_cnt) < 500:
                    #     Common.logger().info("该视频30天内播放量<500:{}", video_title)
                    elif int(video_play_cnt) < 5000:
                        Common.logger().info("该视频7天内播放量<5000:{}", video_title)
                    # 从云文档去重:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=onyBDH
                    elif video_id in [j for i in Feishu.get_values_batch("k2rKkv") for j in i]:
                        Common.logger().info("该视频已保存过：{}", video_title)
                    else:
                        Common.logger().info("该视频未下载，添加至feeds中:{}".format(video_title))
                        # feeds工作表，插入空行
                        time.sleep(1)
                        Feishu.insert_columns("k2rKkv", "ROWS", 2, 3)

                        # 获取当前时间
                        get_feeds_time = int(time.time())
                        # 看一看云文档，工作表中写入数据
                        values = [[profile_id, profile_mid, video_id, video_title, user_name, video_url, time.strftime(
                                "%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)),
                                   str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(get_feeds_time))),
                                   video_play_cnt]]
                        # 等待 1s，防止操作云文档太频繁，导致报错
                        time.sleep(1)
                        Feishu.update_hour_list_values("k2rKkv", "A3:I3", values)

        except Exception as e:
            Common.logger().error("获取小时榜视频列表异常:{}", e)

    # 检查是否有今日的上升榜日期
    @classmethod
    def check_hour_list_data(cls):
        # 判断J1单元格的日期是否为今天
        if Feishu.get_range_value("k2rKkv", "J1:J1")[0] != cls.today:
            # 插入3列 J1:L1，并写入日期和时间数据
            values = [[datetime.datetime.now().strftime("%Y-%m-%d")], ["10:00", "15:00", "20:00"]]
            Feishu.insert_columns("k2rKkv", "COLUMNS", 9, 12)
            Feishu.update_hour_list_values("k2rKkv", "J1:L2", values)
            Feishu.merge_cells("k2rKkv", "J1:L1")
            Common.logger().info("插入今天日期成功")
        else:
            Common.logger().info("今日上升榜日期已存在")

    # 清除空行
    @classmethod
    def del_null_rows(cls):
        for i in range(3, len(Feishu.get_values_batch("k2rKkv")) + 1):
            time.sleep(0.5)
            Common.logger().info("正在检查第:{}行", i)
            # 删除空行
            if Feishu.get_range_value("k2rKkv", "A" + str(i) + ":" + "A" + str(i))[0] is None:
                Common.logger().info("当前第{}行为空行，删除", i)
                Feishu.dimension_range("k2rKkv", "ROWS", i, i)
        Common.logger().info("删除空行完成")

    # 更新小时榜数据
    @classmethod
    def update_hour_list_data(cls):
        """
        更新小时榜数据
        """
        try:
            if len(Feishu.get_values_batch("k2rKkv")) == 2:
                Common.logger().info("当前工作表无数据")
            else:
                for i in range(3, len(Feishu.get_values_batch("k2rKkv"))+1):
                    time.sleep(0.5)
                    Common.logger().info("更新第:{}条视频信息", i-2)

                    # 略过空行
                    if Feishu.get_range_value("k2rKkv", "D" + str(i) + ":" + "D" + str(i))[0] is None\
                            or Feishu.get_range_value("k2rKkv", "C" + str(i) + ":" + "C" + str(i))[0] is None\
                            or Feishu.get_range_value("k2rKkv", "A" + str(i) + ":" + "A" + str(i))[0] is None:
                        Common.logger().info("空行，略过")
                    else:
                        # 视频标题
                        v_title = Feishu.get_range_value("k2rKkv", "D" + str(i) + ":" + "D" + str(i))[0]
                        Common.logger().info("视频详情，video_title:{}，{}", v_title, type(v_title))

                        # 视频 ID
                        v_id = Feishu.get_range_value("k2rKkv", "C" + str(i) + ":" + "C" + str(i))[0]
                        Common.logger().info("视频详情，video_id:{}，{}", v_id, type(v_id))

                        # profile_id，用户 ID
                        p_id = Feishu.get_range_value("k2rKkv", "A" + str(i) + ":" + "A" + str(i))[0]
                        Common.logger().info("视频详情，profile_id:{}，{}", p_id, type(p_id))

                        # profile_mid
                        p_mid = Feishu.get_range_value("k2rKkv", "B" + str(i) + ":" + "B" + str(i))[0]
                        Common.logger().info("视频详情，profile_mid:{}，{}", p_mid, type(p_mid))

                        # 抓取时的播放量
                        v_play_cnt = Feishu.get_range_value("k2rKkv", "I" + str(i) + ":" + "I" + str(i))[0]
                        Common.logger().info("视频详情，video_play_cnt:{}，{}", v_play_cnt, type(v_play_cnt))

                        # 抓取时间
                        v_upload_time = Feishu.get_range_value("k2rKkv", "H" + str(i) + ":" + "H" + str(i))[0]
                        Common.logger().info("视频详情，video_send_time:{}，{}", v_upload_time, type(v_upload_time))

                        # 上传时间
                        upload_hour = v_upload_time.split(" ")[-1].split(":")[0]

                        url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id"
                        headers = {
                            "x-b3-traceid": "158f6ce5d71337",
                            "X-Token-Id": "af9c47bb6c942236ff35ee10d355f3b0-1164637358",
                            "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0",
                            "content-type": "application/json",
                            "Accept-Encoding": "gzip,compress,br,deflate",
                            "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)'
                                          ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 '
                                          'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN',
                            "Referer": "https://servicewechat.com/wxd7911e4c177690e4/617/page-frame.html"
                        }
                        data = {
                            "play_src": "1",
                            "profile_id": int(p_id),
                            "profile_mid": int(p_mid),
                            "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/"
                                  "!400x400r/crop/400x400/interlace/1/format/jpg",
                            "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail"
                                    "/!80x80r/crop/80x80/interlace/1/format/jpg",
                            "share_width": 625,
                            "share_height": 500,
                            "no_comments": True,
                            "no_follow": True,
                            "vid": v_id,
                            "hot_l1_comment": True,
                            "token": "451273638af2c8bb90266bcfaf601a68",
                            "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0",
                            "proj": "ma",
                            "wx_ver": "8.0.20",
                            "code_ver": "3.62.0",
                            "log_common_params": {
                                "e": [{
                                    "data": {
                                        "page": "dynamicSharePage"
                                    }
                                }],
                                "ext": {
                                    "brand": "iPhone",
                                    "device": "iPhone 11",
                                    "os": "iOS 14.7.1",
                                    "weixinver": "8.0.20",
                                    "srcver": "2.24.3",
                                    "net": "wifi",
                                    "scene": "1089"
                                },
                                "pj": "1",
                                "pf": "2",
                                "session_id": "1a20f033-3511-4e7b-9b2a-95e5c542461f"
                            }
                        }
                        try:
                            urllib3.disable_warnings()
                            r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False)
                            hour_play_cnt = r.json()["data"]["play_pv"]
                            Common.logger().info("视频详情，当前播放量:{}", hour_play_cnt)
                            # 固定时间获取符合规则的视频，写入云文档:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=k2rKkv
                            update_hour = datetime.datetime.now()
                            if update_hour.hour == 10 and int(upload_hour) <= 10:
                                Common.logger().info("满足条件: update_hour == 10 and int(upload_hour) < 10")
                                # 当天 10:00 视频播放量
                                ten_hour_play_cnt = hour_play_cnt
                                Common.logger().info("当天 10:00 视频播放量:{}", ten_hour_play_cnt)
                                # 10:00 的上升榜写入数据
                                values = int(ten_hour_play_cnt) - int(v_play_cnt)
                                time.sleep(1)
                                Feishu.update_hour_list_values("k2rKkv", "J"+str(i) + ":" + "J"+str(i), [[values]])
                                Common.logger().info("10:00数据更新成功:{}", values)
                            elif update_hour.hour == 15 and int(upload_hour) <= 10:
                                Common.logger().info("满足条件: update_hour == 15 and int(upload_hour) <= 10")
                                # 当天 15:00 视频播放量
                                fifteen_hour_play_cnt = hour_play_cnt
                                Common.logger().info("当天 15:00 视频播放量:{}", fifteen_hour_play_cnt)
                                # 当天 10:00 上升的数据
                                if Feishu.get_range_value("k2rKkv", "J"+str(i) + ":" + "J"+str(i))[0] is None:
                                    ten_up_cnt = 0
                                else:
                                    ten_up_cnt = Feishu.get_range_value("k2rKkv", "J"+str(i) + ":" + "J"+str(i))[0]
                                # 15:00 的上升榜写入数据
                                values = int(fifteen_hour_play_cnt) - (int(v_play_cnt) + int(ten_up_cnt))
                                time.sleep(1)
                                Feishu.update_hour_list_values("k2rKkv", "K" + str(i) + ":" + "K" + str(i), [[values]])
                                Common.logger().info("15:00数据更新成功:{}", values)
                            elif update_hour.hour == 15 and 10 < int(upload_hour) <= 15:
                                Common.logger().info("满足条件: update_hour == 15 and 10 < int(upload_hour) < 15")
                                # 当天 15:00 视频播放量
                                fifteen_hour_play_cnt = hour_play_cnt
                                # 15:00 的上升榜写入数据
                                values = int(fifteen_hour_play_cnt) - int(v_play_cnt)
                                time.sleep(1)
                                Feishu.update_hour_list_values("k2rKkv", "K" + str(i) + ":" + "K" + str(i), [[values]])
                                Common.logger().info("15:00数据更新成功:{}", values)
                            elif update_hour.hour == 20 and int(upload_hour) <= 10:
                                Common.logger().info("满足条件: update_hour == 20 and int(upload_hour) <= 10")
                                # 当天 20:00 视频播放量
                                twenty_hour_play_cnt = hour_play_cnt
                                # 当天 10:00 上升的数据
                                if Feishu.get_range_value("k2rKkv", "J" + str(i) + ":" + "J" + str(i))[0] is None:
                                    ten_up_cnt = 0
                                else:
                                    ten_up_cnt = Feishu.get_range_value("k2rKkv", "J" + str(i) + ":" + "J" + str(i))[0]
                                # 当天 15:00 上升的数据
                                if Feishu.get_range_value("k2rKkv", "K" + str(i) + ":" + "K" + str(i))[0] is None:
                                    fifteen_up_cnt = 0
                                else:
                                    fifteen_up_cnt = Feishu.get_range_value("k2rKkv", "K" + str(i) + ":" + "K" + str(i))[0]
                                # 20:00 的上升榜写入数据
                                values = int(twenty_hour_play_cnt) - (
                                        int(v_play_cnt) + int(ten_up_cnt) + int(fifteen_up_cnt))
                                time.sleep(1)
                                Feishu.update_hour_list_values("k2rKkv", "L" + str(i) + ":" + "L" + str(i), [[values]])
                                Common.logger().info("20:00数据更新成功:{}", values)
                            elif update_hour.hour == 20 and 10 < int(upload_hour) <= 15:
                                Common.logger().info("满足条件: update_hour == 20 and 10 < int(upload_hour) < 15")
                                # 当天 20:00 视频播放量
                                twenty_hour_play_cnt = hour_play_cnt
                                # 当天 15:00 上升的数据
                                if Feishu.get_range_value("k2rKkv", "K" + str(i) + ":" + "K" + str(i))[0] is None:
                                    fifteen_up_cnt = 0
                                else:
                                    fifteen_up_cnt = Feishu.get_range_value("k2rKkv", "K" + str(i) + ":" + "K" + str(i))[0]
                                # 20:00 的上升榜写入数据
                                values = int(twenty_hour_play_cnt) - (int(v_play_cnt) + int(fifteen_up_cnt))
                                time.sleep(1)
                                Feishu.update_hour_list_values("k2rKkv", "L" + str(i) + ":" + "L" + str(i), [[values]])
                                Common.logger().info("20:00数据更新成功:{}", values)
                            elif update_hour.hour == 20 and 15 < int(upload_hour) <= 20:
                                Common.logger().info("满足条件: update_hour == 20 and 15 < int(upload_hour) < 20")
                                # 当天 20:00 视频播放量
                                twenty_hour_play_cnt = hour_play_cnt
                                # 20:00 的上升榜写入数据
                                values = int(twenty_hour_play_cnt) - int(v_play_cnt)
                                time.sleep(1)
                                Feishu.update_hour_list_values("k2rKkv", "L" + str(i) + ":" + "L" + str(i), [[values]])
                                Common.logger().info("20:00数据更新成功:{}", values)
                        except Exception as e:
                            Common.logger().error("视频详情:{}，异常:{}", v_title, e)
        except Exception as e:
            Common.logger().error("获取小时榜数据异常:{}", e)


if __name__ == "__main__":
    hour_list = HourList()
    # hour_list.get_hour_list_feeds()
    hour_list.del_null_rows()
    hour_list.update_hour_list_data()