# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2022/5/16 import datetime import time import requests import urllib3 from main.common import Common from main.feishu_lib import Feishu proxies = {"http": None, "https": None} class HourList: # 今天的日期:年-月-日 today = datetime.datetime.now().strftime("%Y-%m-%d") # 已下载小时榜视频列表 download_hour_video_list = [] # 下载规则 @staticmethod def download_rule(d_duration, d_width, d_height, d_play_cnt, d_like_cnt, d_share_cnt): """ 下载视频的基本规则 :param d_duration: 时长 :param d_width: 宽 :param d_height: 高 :param d_play_cnt: 播放量 :param d_like_cnt: 点赞量 :param d_share_cnt: 分享量 :return: 满足规则，返回 True；反之，返回 False """ if 600 >= int(float(d_duration)) >= 60: if int(d_width) >= 0 or int(d_height) >= 0: if int(d_play_cnt) >= 0: if int(d_like_cnt) >= 0: if int(d_share_cnt) >= 0: return True else: return False else: return False else: return False return False return False # 获取列表 @classmethod def get_hour_list_feeds(cls): """ 1.从列表获取视频，7 天内，播放量>=5000 2.时长 1-10min 3.每天10:00、15:00、20:00 把符合规则的视频，写入云文档 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=k2rKkv """ url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends" headers = { "x-b3-traceid": "15d0e28c9a74d5", "X-Token-Id": "af9c47bb6c942236ff35ee10d355f3b0-1164637358", "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0", "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": "https://servicewechat.com/wxd7911e4c177690e4/617/page-frame.html" } data = { "log_params": { "page": "discover_rec", "common": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "none" } }, "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "ext": { "fmid": 0, "items": {} }, "app": "xng", "rec_scene": "discover_rec", "log_common_params": { "e": [{ "data": { "page": "discoverIndexPage", "topic": "recommend" }, "ab": {} }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "c0a5d52b-8f09-48ee-99ab-0feb4f881fe8" }, "refresh": False, "token": "451273638af2c8bb90266bcfaf601a68", "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0", "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0" } try: urllib3.disable_warnings() r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False) if "data" not in r.json(): Common.logger().warning("获取视频feeds错误:{}", r.text) elif "list" not in r.json()["data"]: Common.logger().warning("获取视频feeds无数据，休眠10s:{}", r.json()["data"]) else: # 视频列表数据 feeds = r.json()["data"]["list"] for i in range(len(feeds)): # 标题 if "title" in feeds[i]: video_title = feeds[i]["title"].strip().replace("\n", "") \ .replace("/", "").replace("\r", "").replace("#", "") \ .replace(".", "。").replace("\\", "").replace("&NBSP", "") \ .replace(":", "").replace("*", "").replace("？", "") \ .replace("?", "").replace('"', "").replace("<", "") \ .replace(">", "").replace("|", "").replace(" ", "") Common.logger().info("标题:{}", video_title) else: video_title = "" Common.logger().info("当前视频无标题:{}", video_title) # 视频 ID if "vid" in feeds[i]: video_id = feeds[i]["vid"] Common.logger().info("视频ID:{}", video_id) else: video_id = "" Common.logger().info("当前视频无ID:{}", video_id) # 播放量 if "play_pv" in feeds[i]: video_play_cnt = feeds[i]["play_pv"] Common.logger().info("视频播放量:{}", video_play_cnt) else: video_play_cnt = "" Common.logger().info("当前视频无播放量:{}", video_play_cnt) # 点赞量 if "favor" in feeds[i]: video_like_cnt = feeds[i]["favor"]["total"] Common.logger().info("视频点赞量:{}", video_like_cnt) else: video_like_cnt = "" Common.logger().info("当前视频无点赞量:{}", video_like_cnt) # 分享量 if "share" in feeds[i]: video_share_cnt = feeds[i]["share"] Common.logger().info("视频分享量:{}", video_share_cnt) else: video_share_cnt = "" Common.logger().info("当前视频无分享量:{}", video_share_cnt) # 评论量 if "comment_count" in feeds[i]: video_comment_cnt = feeds[i]["comment_count"] Common.logger().info("视频评论数:{}", video_comment_cnt) else: video_comment_cnt = "" Common.logger().info("当前视频无评论:{}", video_comment_cnt) # 时长 if "du" in feeds[i]: video_duration = int(feeds[i]["du"] / 1000) Common.logger().info("视频时长:{}秒", video_duration) else: video_duration = "" Common.logger().info("当前视频无时长:{}", video_duration) # 宽和高 if "w" or "h" in feeds[i]: video_width = feeds[i]["w"] video_height = feeds[i]["h"] Common.logger().info("视频宽高:{}*{}", video_width, video_height) else: video_width = "" video_height = "" Common.logger().info("当前视频无宽高:{}{}", video_width, video_height) # 发布时间 if "t" in feeds[i]: video_send_time = feeds[i]["t"] Common.logger().info( "视频发布时间:{}", time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000))) else: video_send_time = "" Common.logger().info("当前视频无发布时间:{}", video_send_time) # 用户名 / 头像 if "user" in feeds[i]: user_name = feeds[i]["user"]["nick"].strip().replace("\n", "") \ .replace("/", "").replace("快手", "").replace(" ", "") \ .replace(" ", "").replace("&NBSP", "").replace("\r", "") head_url = feeds[i]["user"]["hurl"] Common.logger().info("用户名:{}", user_name) Common.logger().info("用户头像:{}", head_url) else: user_name = "" head_url = "" Common.logger().info("当前视频无用户名:{}", user_name) Common.logger().info("当前视频无用户头像:{}", head_url) # 用户 ID profile_id = feeds[i]["id"] # 用户 mid profile_mid = feeds[i]["user"]["mid"] # 视频封面 if "url" in feeds[i]: cover_url = feeds[i]["url"] Common.logger().info("视频封面:{}", cover_url) else: cover_url = "" Common.logger().info("当前视频无视频封面:{}", cover_url) # 视频播放地址 if "v_url" in feeds[i]: video_url = feeds[i]["v_url"] Common.logger().info("播放地址:{}", video_url) else: video_url = "" Common.logger().info("当前视频无播放地址:{}", video_url) # 过滤无效视频 if video_title == "" or video_id == "" or video_duration == "" \ or video_send_time == "" or user_name == "" or head_url == "" \ or cover_url == "" or video_url == "": Common.logger().warning("无效视频") # 判断发布时间是否 > 7天 # elif int(time.time()) - int(video_send_time)/1000 > 2592000: # Common.logger().info("发布时间大于30天", video_title) elif int(time.time()) - int(video_send_time)/1000 > 604800: Common.logger().info("发布时间大于7天", video_title) # 判断播放量是否 > 5000 # elif int(video_play_cnt) < 500: # Common.logger().info("该视频30天内播放量<500:{}", video_title) elif int(video_play_cnt) < 5000: Common.logger().info("该视频7天内播放量<5000:{}", video_title) # 从云文档去重:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=onyBDH elif video_id in [j for i in Feishu.get_values_batch("k2rKkv") for j in i]: Common.logger().info("该视频已保存过：{}", video_title) else: Common.logger().info("该视频未下载，添加至feeds中:{}".format(video_title)) # feeds工作表，插入空行 Feishu.insert_columns("k2rKkv", "ROWS", 2, 3) # 等待 1s，防止操作云文档太频繁，导致报错 time.sleep(1) # 获取当前时间 get_feeds_time = int(time.time()) # 看一看云文档，工作表中写入数据 values = [[profile_id, profile_mid, video_id, video_title, user_name, video_url, time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)), str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(get_feeds_time))), video_play_cnt]] Feishu.update_hour_list_values("k2rKkv", "A3:I3", values) except Exception as e: Common.logger().error("获取小时榜视频列表异常:{}", e) # 检查是否有今日的上升榜日期 @classmethod def check_hour_list_data(cls): # 判断J1单元格的日期是否为今天 if Feishu.get_range_value("k2rKkv", "J1:J1")[0] != cls.today: # 插入3列 J1:L1，并写入日期和时间数据 values = [[datetime.datetime.now().strftime("%Y-%m-%d")], ["10:00", "15:00", "20:00"]] Feishu.insert_columns("k2rKkv", "COLUMNS", 9, 12) Feishu.update_hour_list_values("k2rKkv", "J1:L2", values) Feishu.merge_cells("k2rKkv", "J1:L1") Common.logger().info("插入今天日期成功") else: Common.logger().info("今日上升榜日期已存在") # 清除空行 @classmethod def del_null_rows(cls): for i in range(3, len(Feishu.get_values_batch("k2rKkv")) + 1): time.sleep(0.5) Common.logger().info("正在检查第:{}行", i) # 删除空行 if Feishu.get_range_value("k2rKkv", "A" + str(i) + ":" + "A" + str(i))[0] is None: Common.logger().info("当前第{}行为空行，删除", i) Feishu.dimension_range("k2rKkv", "ROWS", i, i) Common.logger().info("删除空行完成") # 更新小时榜数据 @classmethod def update_hour_list_data(cls): """ 更新小时榜数据 """ try: if len(Feishu.get_values_batch("k2rKkv")) == 2: Common.logger().info("当前工作表无数据") else: for i in range(3, len(Feishu.get_values_batch("k2rKkv"))+1): time.sleep(0.5) Common.logger().info("更新第:{}条视频信息", i-2) # 视频标题 v_title = Feishu.get_range_value("k2rKkv", "D" + str(i) + ":" + "D" + str(i))[0] Common.logger().info("视频详情，video_title:{}，{}", v_title, type(v_title)) # time.sleep(0.5) # 视频 ID v_id = Feishu.get_range_value("k2rKkv", "C" + str(i) + ":" + "C" + str(i))[0] Common.logger().info("视频详情，video_id:{}，{}", v_id, type(v_id)) # time.sleep(0.5) # profile_id，用户 ID p_id = Feishu.get_range_value("k2rKkv", "A" + str(i) + ":" + "A" + str(i))[0] Common.logger().info("视频详情，profile_id:{}，{}", p_id, type(p_id)) # time.sleep(0.5) # profile_mid p_mid = Feishu.get_range_value("k2rKkv", "B" + str(i) + ":" + "B" + str(i))[0] Common.logger().info("视频详情，profile_mid:{}，{}", p_mid, type(p_mid)) # time.sleep(0.5) # 抓取时的播放量 v_play_cnt = Feishu.get_range_value("k2rKkv", "I" + str(i) + ":" + "I" + str(i))[0] Common.logger().info("视频详情，video_play_cnt:{}，{}", v_play_cnt, type(v_play_cnt)) # time.sleep(0.5) # 抓取时间 v_upload_time = Feishu.get_range_value("k2rKkv", "H" + str(i) + ":" + "H" + str(i))[0] Common.logger().info("视频详情，video_send_time:{}，{}", v_upload_time, type(v_upload_time)) # time.sleep(0.5) # 上传时间 upload_hour = v_upload_time.split(" ")[-1].split(":")[0] url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id" headers = { "x-b3-traceid": "b8cd153630f11", "X-Token-Id": "af9c47bb6c942236ff35ee10d355f3b0-1164637358", "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0", "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": "https://servicewechat.com/wxd7911e4c177690e4/617/page-frame.html" } data = { "play_src": "1", "profile_id": int(p_id), "profile_mid": int(p_mid), "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/" "!400x400r/crop/400x400/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail" "/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "no_comments": True, "no_follow": True, "vid": v_id, "hot_l1_comment": True, "token": "451273638af2c8bb90266bcfaf601a68", "uid": "3d460a1b-ab85-426b-bd80-62029acaa2c0", "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0", "log_common_params": { "e": [{ "data": { "page": "dynamicSharePage" } }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "c0a5d52b-8f09-48ee-99ab-0feb4f881fe8" } } try: urllib3.disable_warnings() r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False) hour_play_cnt = r.json()["data"]["play_pv"] Common.logger().info("视频详情，当前播放量:{}", hour_play_cnt) # 固定时间获取符合规则的视频，写入云文档:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=k2rKkv update_hour = datetime.datetime.now() if update_hour.hour == 10 and int(upload_hour) <= 10: Common.logger().info("满足条件: update_hour == 10 and int(upload_hour) < 10") # 当天 10:00 视频播放量 ten_hour_play_cnt = hour_play_cnt Common.logger().info("当天 10:00 视频播放量:{}", ten_hour_play_cnt) # 10:00 的上升榜写入数据 values = int(ten_hour_play_cnt) - int(v_play_cnt) time.sleep(1) Feishu.update_hour_list_values("k2rKkv", "J"+str(i) + ":" + "J"+str(i), [[values]]) Common.logger().info("10:00数据更新成功:{}", values) elif update_hour.hour == 15 and int(upload_hour) <= 10: Common.logger().info("满足条件: update_hour == 15 and int(upload_hour) <= 10") # 当天 15:00 视频播放量 fifteen_hour_play_cnt = hour_play_cnt Common.logger().info("当天 15:00 视频播放量:{}", fifteen_hour_play_cnt) # 当天 10:00 上升的数据 if Feishu.get_range_value("k2rKkv", "J"+str(i) + ":" + "J"+str(i))[0] is None: ten_up_cnt = 0 else: ten_up_cnt = Feishu.get_range_value("k2rKkv", "J"+str(i) + ":" + "J"+str(i))[0] # 15:00 的上升榜写入数据 values = int(fifteen_hour_play_cnt) - (int(v_play_cnt) + int(ten_up_cnt)) time.sleep(1) Feishu.update_hour_list_values("k2rKkv", "K" + str(i) + ":" + "K" + str(i), [[values]]) Common.logger().info("15:00数据更新成功:{}", values) elif update_hour.hour == 15 and 10 < int(upload_hour) <= 15: Common.logger().info("满足条件: update_hour == 15 and 10 < int(upload_hour) < 15") # 当天 15:00 视频播放量 fifteen_hour_play_cnt = hour_play_cnt # 15:00 的上升榜写入数据 values = int(fifteen_hour_play_cnt) - int(v_play_cnt) time.sleep(1) Feishu.update_hour_list_values("k2rKkv", "K" + str(i) + ":" + "K" + str(i), [[values]]) Common.logger().info("15:00数据更新成功:{}", values) elif update_hour.hour == 20 and int(upload_hour) <= 10: Common.logger().info("满足条件: update_hour == 20 and int(upload_hour) <= 10") # 当天 20:00 视频播放量 twenty_hour_play_cnt = hour_play_cnt # 当天 10:00 上升的数据 if Feishu.get_range_value("k2rKkv", "J" + str(i) + ":" + "J" + str(i))[0] is None: ten_up_cnt = 0 else: ten_up_cnt = Feishu.get_range_value("k2rKkv", "J" + str(i) + ":" + "J" + str(i))[0] # 当天 15:00 上升的数据 if Feishu.get_range_value("k2rKkv", "K" + str(i) + ":" + "K" + str(i))[0] is None: fifteen_up_cnt = 0 else: fifteen_up_cnt = Feishu.get_range_value("k2rKkv", "K" + str(i) + ":" + "K" + str(i))[0] # 20:00 的上升榜写入数据 values = int(twenty_hour_play_cnt) - ( int(v_play_cnt) + int(ten_up_cnt) + int(fifteen_up_cnt)) time.sleep(1) Feishu.update_hour_list_values("k2rKkv", "L" + str(i) + ":" + "L" + str(i), [[values]]) Common.logger().info("20:00数据更新成功:{}", values) elif update_hour.hour == 20 and 10 < int(upload_hour) <= 15: Common.logger().info("满足条件: update_hour == 20 and 10 < int(upload_hour) < 15") # 当天 20:00 视频播放量 twenty_hour_play_cnt = hour_play_cnt # 当天 15:00 上升的数据 if Feishu.get_range_value("k2rKkv", "K" + str(i) + ":" + "K" + str(i))[0] is None: fifteen_up_cnt = 0 else: fifteen_up_cnt = Feishu.get_range_value("k2rKkv", "K" + str(i) + ":" + "K" + str(i))[0] # 20:00 的上升榜写入数据 values = int(twenty_hour_play_cnt) - (int(v_play_cnt) + int(fifteen_up_cnt)) time.sleep(1) Feishu.update_hour_list_values("k2rKkv", "L" + str(i) + ":" + "L" + str(i), [[values]]) Common.logger().info("20:00数据更新成功:{}", values) elif update_hour.hour == 20 and 15 < int(upload_hour) <= 20: Common.logger().info("满足条件: update_hour == 20 and 15 < int(upload_hour) < 20") # 当天 20:00 视频播放量 twenty_hour_play_cnt = hour_play_cnt # 20:00 的上升榜写入数据 values = int(twenty_hour_play_cnt) - int(v_play_cnt) time.sleep(1) Feishu.update_hour_list_values("k2rKkv", "L" + str(i) + ":" + "L" + str(i), [[values]]) Common.logger().info("20:00数据更新成功:{}", values) except Exception as e: Common.logger().error("视频详情:{}，异常:{}", v_title, e) except Exception as e: Common.logger().error("获取小时榜数据异常:{}", e) if __name__ == "__main__": hour_list = HourList() # hour_list.get_hour_list_feeds() hour_list.del_null_rows() hour_list.update_hour_list_data()