# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2022/5/16 import datetime import time import requests import urllib3 from main.common import Common from main.feishu_lib import Feishu from main.publish import Publish proxies = {"http": None, "https": None} class HourList: hour_x_b3_traceid = Feishu.get_range_value("person-log", "xiaoniangao", "dzcWHw", "B3:B3")[0] hour_x_token_id = Feishu.get_range_value("person-log", "xiaoniangao", "dzcWHw", "B4:B4")[0] hour_referer = Feishu.get_range_value("person-log", "xiaoniangao", "dzcWHw", "B5:B5")[0][0]["link"] hour_uid = Feishu.get_range_value("person-log", "xiaoniangao", "dzcWHw", "B6:B6")[0] hour_token = Feishu.get_range_value("person-log", "xiaoniangao", "dzcWHw", "B7:B7")[0] # 过滤敏感词 @classmethod def sensitive_words(cls): # 敏感词库列表 word_list = [] # 从云文档读取所有敏感词，添加到词库列表 time.sleep(1) lists = Feishu.get_values_batch("person-logs", "xiaoniangao", "DRAnZh") for i in lists: for j in i: # 过滤空的单元格内容 if j is None: pass else: word_list.append(j) return word_list # 视频ID过滤字母 @classmethod def sensitive_videoid_words(cls): # 字母列表 words_list = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] return words_list # 基础门槛规则 @staticmethod def download_rule(d_duration, d_width, d_height, d_play_cnt, d_like_cnt, d_share_cnt, d_send_time): """ 下载视频的基本规则 :param d_duration: 时长 :param d_width: 宽 :param d_height: 高 :param d_play_cnt: 播放量 :param d_like_cnt: 点赞量 :param d_share_cnt: 分享量 :param d_send_time: 发布时间 :return: 满足规则，返回 True；反之，返回 False """ # 视频时长 if 600 >= int(float(d_duration)) >= 60: # 宽或高 if int(d_width) >= 0 or int(d_height) >= 0: # 播放量 if int(d_play_cnt) >= 5000: # 点赞量 if int(d_like_cnt) >= 0: # 分享量 if int(d_share_cnt) >= 0: # 发布时间 <= 7 天 if int(time.time()) - int(d_send_time)/1000 <= 604800: return True else: return False else: return False else: return False else: return False return False return False # 检查是否有今日的上升榜日期 @classmethod def check_hour_list_data(cls, date): # 判断J1单元格的日期是否为今天 time.sleep(1) if Feishu.get_range_value("logs", "xiaoniangao", "ba0da4", "L1:N1")[0] != date: # 插入3列 L1:N1，并写入日期和时间数据 values = [[date], ["10:00", "15:00", "20:00"]] time.sleep(1) Feishu.insert_columns("logs", "xiaoniangao", "ba0da4", "COLUMNS", 11, 14) time.sleep(1) Feishu.update_values("logs", "xiaoniangao", "ba0da4", "L1:N2", values) time.sleep(1) Feishu.merge_cells("logs", "xiaoniangao", "ba0da4", "L1:N1") Common.logger().info("插入今天日期成功") else: Common.logger().info("今日上升榜日期已存在") # 清除空行 @classmethod def del_null_rows(cls, crawler, sheetid, startindex): """ :params sheetid:工作表 ID :params startindex:从第几行开始清除 """ time.sleep(1) for i in range(int(startindex), len(Feishu.get_values_batch("logs", crawler, sheetid)) + 1): time.sleep(1) Common.logger().info("正在检查第:{}行", i) # 删除空行 if Feishu.get_range_value("logs", crawler, sheetid, "A" + str(i) + ":" + "A" + str(i))[0] is None\ and Feishu.get_range_value("logs", crawler, sheetid, "B" + str(i) + ":" + "B" + str(i))[0] is None\ and Feishu.get_range_value("logs", crawler, sheetid, "C" + str(i) + ":" + "C" + str(i))[0] is None\ and Feishu.get_range_value("logs", crawler, sheetid, "D" + str(i) + ":" + "D" + str(i))[0] is None: Common.logger().info("当前第{}行为空行，删除", i) time.sleep(1) Feishu.dimension_range("logs", crawler, sheetid, "ROWS", i, i) Common.logger().info("删除空行完成") # 获取列表 @classmethod def get_hour_list_feeds(cls): """ 1.从列表获取视频，7 天内，播放量>=5000 2.时长 1-10min 3.每天10:00、15:00、20:00 把符合规则的视频，写入云文档 https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=ba0da4 """ url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends" headers = { "x-b3-traceid": cls.hour_x_b3_traceid, "X-Token-Id": cls.hour_x_token_id, "uid": cls.hour_uid, "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": cls.hour_referer } data = { "log_params": { "page": "discover_rec", "common": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.2", "net": "wifi", "scene": 1089 } }, "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "ext": { "fmid": 0, "items": {} }, "app": "xng", "rec_scene": "discover_rec", "log_common_params": { "e": [{ "data": { "page": "discoverIndexPage", "topic": "recommend" }, "ab": {} }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" }, "refresh": False, "token": cls.hour_token, "uid": cls.hour_uid, "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0" } try: urllib3.disable_warnings() r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False) if "data" not in r.json(): Common.logger().warning("获取视频feeds错误:{}", r.text) elif "list" not in r.json()["data"]: Common.logger().warning("获取视频feeds无数据，休眠10s:{}", r.json()["data"]) else: # 视频列表数据 feeds = r.json()["data"]["list"] for i in range(len(feeds)): # 标题 if "title" in feeds[i]: video_title = feeds[i]["title"].strip().replace("\n", "") \ .replace("/", "").replace("\r", "").replace("#", "") \ .replace(".", "。").replace("\\", "").replace("&NBSP", "") \ .replace(":", "").replace("*", "").replace("？", "") \ .replace("?", "").replace('"', "").replace("<", "") \ .replace(">", "").replace("|", "").replace(" ", "") Common.logger().info("标题:{}", video_title) else: video_title = "" Common.logger().info("当前视频无标题:{}", video_title) # 视频 ID if "vid" in feeds[i]: video_id = feeds[i]["vid"] Common.logger().info("视频ID:{}", video_id) else: video_id = "" Common.logger().info("当前视频无ID:{}", video_id) # 播放量 if "play_pv" in feeds[i]: video_play_cnt = feeds[i]["play_pv"] Common.logger().info("视频播放量:{}", video_play_cnt) else: video_play_cnt = "" Common.logger().info("当前视频无播放量:{}", video_play_cnt) # 点赞量 if "favor" in feeds[i]: video_like_cnt = feeds[i]["favor"]["total"] Common.logger().info("视频点赞量:{}", video_like_cnt) else: video_like_cnt = "" Common.logger().info("当前视频无点赞量:{}", video_like_cnt) # 分享量 if "share" in feeds[i]: video_share_cnt = feeds[i]["share"] Common.logger().info("视频分享量:{}", video_share_cnt) else: video_share_cnt = "" Common.logger().info("当前视频无分享量:{}", video_share_cnt) # 评论量 if "comment_count" in feeds[i]: video_comment_cnt = feeds[i]["comment_count"] Common.logger().info("视频评论数:{}", video_comment_cnt) else: video_comment_cnt = "" Common.logger().info("当前视频无评论:{}", video_comment_cnt) # 时长 if "du" in feeds[i]: video_duration = int(feeds[i]["du"] / 1000) Common.logger().info("视频时长:{}秒", video_duration) else: video_duration = "" Common.logger().info("当前视频无时长:{}", video_duration) # 宽和高 if "w" or "h" in feeds[i]: video_width = feeds[i]["w"] video_height = feeds[i]["h"] Common.logger().info("视频宽高:{}*{}", video_width, video_height) else: video_width = "" video_height = "" Common.logger().info("当前视频无宽高:{}{}", video_width, video_height) # 发布时间 if "t" in feeds[i]: video_send_time = feeds[i]["t"] Common.logger().info( "视频发布时间:{}", time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000))) else: video_send_time = "" Common.logger().info("当前视频无发布时间:{}", video_send_time) # 用户名 / 头像 if "user" in feeds[i]: user_name = feeds[i]["user"]["nick"].strip().replace("\n", "") \ .replace("/", "").replace("快手", "").replace(" ", "") \ .replace(" ", "").replace("&NBSP", "").replace("\r", "") head_url = feeds[i]["user"]["hurl"] Common.logger().info("用户名:{}", user_name) Common.logger().info("用户头像:{}", head_url) else: user_name = "" head_url = "" Common.logger().info("当前视频无用户名:{}", user_name) Common.logger().info("当前视频无用户头像:{}", head_url) # 用户 ID profile_id = feeds[i]["id"] # 用户 mid profile_mid = feeds[i]["user"]["mid"] # 视频封面 if "url" in feeds[i]: cover_url = feeds[i]["url"] Common.logger().info("视频封面:{}", cover_url) else: cover_url = "" Common.logger().info("当前视频无视频封面:{}", cover_url) # 视频播放地址 if "v_url" in feeds[i]: video_url = feeds[i]["v_url"] Common.logger().info("播放地址:{}", video_url) else: video_url = "" Common.logger().info("当前视频无播放地址:{}", video_url) # 过滤无效视频 if video_title == "" or video_id == "" or video_duration == "" \ or video_send_time == "" or user_name == "" or head_url == "" \ or cover_url == "" or video_url == "": Common.logger().warning("无效视频") elif cls.download_rule( video_duration, video_width, video_height, video_play_cnt, video_like_cnt, video_share_cnt, video_send_time) is False: Common.logger().info("不满足基础门槛规则") # 过滤敏感词 elif any(word if word in video_title else False for word in cls.sensitive_words()) is True: Common.logger().info("视频已中敏感词:{}".format(video_title)) time.sleep(1) # 从云文档中去重:https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=yatRv2 elif video_id in [j for i in Feishu.get_values_batch("logs", "xiaoniangao", "yatRv2") for j in i]: Common.logger().info("该视频已下载:{}", video_title) time.sleep(1) # 从云文档去重:https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=ba0da4 elif video_id in [j for i in Feishu.get_values_batch("logs", "xiaoniangao", "ba0da4") for j in i]: Common.logger().info("该视频已保存过:{}", video_title) time.sleep(1) else: Common.logger().info("该视频未下载，添加至feeds中:{}".format(video_title)) # feeds工作表，插入空行 time.sleep(1) Feishu.insert_columns("logs", "xiaoniangao", "ba0da4", "ROWS", 2, 3) # 获取当前时间 get_feeds_time = int(time.time()) # 看一看云文档，工作表中写入数据 values = [[profile_id, profile_mid, video_id, video_title, user_name, video_duration, cover_url, video_url, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time) / 1000)), str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(get_feeds_time))), video_play_cnt]] # 等待 1s，防止操作云文档太频繁，导致报错 time.sleep(1) Feishu.update_values("logs", "xiaoniangao", "ba0da4", "A3:K3", values) except Exception as e: Common.logger().error("获取小时榜视频列表异常:{}", e) # 更新小时榜数据 @classmethod def update_hour_list_data(cls, today, yesterday, before_yesterday): """ 更新小时榜数据 """ try: time.sleep(1) if len(Feishu.get_values_batch("logs", "xiaoniangao", "ba0da4")) == 2: Common.logger().info("当前工作表无数据") else: time.sleep(1) for i in range(3, len(Feishu.get_values_batch("logs", "xiaoniangao", "ba0da4"))+1): time.sleep(1) Common.logger().info("更新第:{}行视频信息", i) # 略过空行 if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "D" + str(i) + ":" + "D" + str(i))[0] is None\ and Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "C"+str(i)+":"+"C"+str(i))[0] is None\ and Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "A"+str(i)+":"+"A"+str(i))[0] is None: Common.logger().info("空行，略过") else: # 视频标题 time.sleep(1) v_title = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "D" + str(i) + ":" + "D" + str(i))[0] Common.logger().info("视频详情，video_title:{}，{}", v_title, type(v_title)) # 视频 ID time.sleep(1) v_id = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "C" + str(i) + ":" + "C" + str(i))[0] Common.logger().info("视频详情，video_id:{}，{}", v_id, type(v_id)) # profile_id，用户 ID time.sleep(1) p_id = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "A" + str(i) + ":" + "A" + str(i))[0] Common.logger().info("视频详情，profile_id:{}，{}", p_id, type(p_id)) # profile_mid time.sleep(1) p_mid = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "B" + str(i) + ":" + "B" + str(i))[0] Common.logger().info("视频详情，profile_mid:{}，{}", p_mid, type(p_mid)) # 抓取时的播放量 time.sleep(1) v_play_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "K" + str(i) + ":" + "K" + str(i))[0] Common.logger().info("视频详情，video_play_cnt:{}，{}", v_play_cnt, type(v_play_cnt)) # 抓取时间 time.sleep(1) v_upload_time = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "J" + str(i) + ":" + "J" + str(i))[0] Common.logger().info("视频详情，video_send_time:{}，{}", v_upload_time, type(v_upload_time)) # 抓取时间的时间戳格式（秒为单位） v_time = int(time.mktime(time.strptime(v_upload_time, "%Y-%m-%d %H:%M:%S"))) # 抓取时间:日期 upload_data = v_upload_time.split(" ")[0] # 抓取时间:小时 upload_hour = v_upload_time.split(" ")[-1].split(":")[0] url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id" headers = { "x-b3-traceid": cls.hour_x_b3_traceid, "X-Token-Id": cls.hour_x_token_id, "uid": cls.hour_uid, "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": cls.hour_referer } data = { "play_src": "1", "profile_id": int(p_id), "profile_mid": int(p_mid), "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/" "!400x400r/crop/400x400/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail" "/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "no_comments": True, "no_follow": True, "vid": v_id, "hot_l1_comment": True, "token": cls.hour_token, "uid": cls.hour_uid, "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0", "log_common_params": { "e": [{ "data": { "page": "dynamicSharePage" } }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" } } try: urllib3.disable_warnings() r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False) hour_play_cnt = r.json()["data"]["play_pv"] Common.logger().info("视频详情，当前播放量:{}", hour_play_cnt) # 固定时间获取符合规则的视频，写入云文档:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=ba0da4 update_hour = datetime.datetime.now() if int(time.time()) - v_time >= 172800: Common.logger().info("抓取时间超过 2 天") return elif upload_data == today and update_hour.hour == 10 and int(upload_hour) <= 10: Common.logger().info("满足条件: 抓取日期为今天 and 当前时间:10点 and 抓取时间<=10点") # 当天 10:00 视频播放量 ten_hour_play_cnt = hour_play_cnt Common.logger().info("当天 10:00 视频播放量:{}", ten_hour_play_cnt) # 10:00 的上升榜写入数据 values = int(ten_hour_play_cnt) - int(v_play_cnt) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i), [[values]]) Common.logger().info("10:00数据更新成功:{}", values) elif upload_data == today and update_hour.hour == 15 and int(upload_hour) <= 10: Common.logger().info("满足条件: 抓取日期为今天 and 当前时间:15点 and 抓取时间<=10点") # 当天 15:00 视频播放量 fifteen_hour_play_cnt = hour_play_cnt Common.logger().info("当天 15:00 视频播放量:{}", fifteen_hour_play_cnt) # 当天 10:00 上升的数据 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L"+str(i) + ":" + "L"+str(i))[0] is None: ten_up_cnt = 0 else: ten_up_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L"+str(i) + ":" + "L"+str(i))[0] # 15:00 的上升榜写入数据 values = int(fifteen_hour_play_cnt) - (int(v_play_cnt) + int(ten_up_cnt)) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i), [[values]]) Common.logger().info("15:00数据更新成功:{}", values) elif upload_data == today and update_hour.hour == 15 and 10 < int(upload_hour) <= 15: Common.logger().info("满足条件: 抓取日期为今天 and 当前时间:15点 and 10<抓取时间<=15点") # 当天 15:00 视频播放量 fifteen_hour_play_cnt = hour_play_cnt Common.logger().info("当天 15:00 视频播放量:{}", fifteen_hour_play_cnt) # 15:00 的上升榜写入数据 values = int(fifteen_hour_play_cnt) - int(v_play_cnt) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i), [[values]]) Common.logger().info("15:00数据更新成功:{}", values) elif upload_data == today and update_hour.hour == 20 and int(upload_hour) <= 10: Common.logger().info("满足条件: 抓取日期为今天 and 当前时间:20点 and 抓取时间<=10点") # 当天 20:00 视频播放量 twenty_hour_play_cnt = hour_play_cnt Common.logger().info("当天 20:00 视频播放量:{}", twenty_hour_play_cnt) # 当天 10:00 上升的数据 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] is None: ten_up_cnt = 0 else: ten_up_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] # 当天 15:00 上升的数据 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] is None: fifteen_up_cnt = 0 else: fifteen_up_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] # 20:00 的上升榜写入数据 values = int(twenty_hour_play_cnt) - ( int(v_play_cnt) + int(ten_up_cnt) + int(fifteen_up_cnt)) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "N" + str(i) + ":" + "N" + str(i), [[values]]) Common.logger().info("20:00数据更新成功:{}", values) elif upload_data == today and update_hour.hour == 20 and 10 < int(upload_hour) <= 15: Common.logger().info("满足条件: 抓取日期为今天 and 当前时间:20点 and 10<抓取时间<=15点") # 当天 20:00 视频播放量 twenty_hour_play_cnt = hour_play_cnt Common.logger().info("当天 20:00 视频播放量:{}", twenty_hour_play_cnt) # 当天 15:00 上升的数据 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] is None: fifteen_up_cnt = 0 else: fifteen_up_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] # 20:00 的上升榜写入数据 values = int(twenty_hour_play_cnt) - (int(v_play_cnt) + int(fifteen_up_cnt)) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "N" + str(i) + ":" + "N" + str(i), [[values]]) Common.logger().info("20:00数据更新成功:{}", values) elif upload_data == today and update_hour.hour == 20 and 15 < int(upload_hour) <= 20: Common.logger().info("满足条件: 抓取日期为今天 and 当前时间:20点 and 15<抓取时间<=20点") # 当天 20:00 视频播放量 twenty_hour_play_cnt = hour_play_cnt Common.logger().info("当天 20:00 视频播放量:{}", twenty_hour_play_cnt) # 20:00 的上升榜写入数据 values = int(twenty_hour_play_cnt) - int(v_play_cnt) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "N" + str(i) + ":" + "N" + str(i), [[values]]) Common.logger().info("20:00数据更新成功:{}", values) elif (upload_data == yesterday or upload_data == before_yesterday)\ and update_hour.hour == 10: Common.logger().info("满足条件: 抓取时间小于今天 and 当前时间:10点") # 当天 10:00 视频播放量 ten_hour_play_cnt = hour_play_cnt Common.logger().info("当天 10:00 视频播放量:{}", ten_hour_play_cnt) # 10:00 的上升榜写入数据 values = int(ten_hour_play_cnt) - int(v_play_cnt) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i), [[values]]) Common.logger().info("10:00数据更新成功:{}", values) elif (upload_data == yesterday or upload_data == before_yesterday)\ and update_hour.hour == 15: Common.logger().info("满足条件: 抓取时间小于今天 and 当前时间:15点") # 当天 15:00 视频播放量 fifteen_hour_play_cnt = hour_play_cnt Common.logger().info("当天 15:00 视频播放量:{}", fifteen_hour_play_cnt) # 当天 10:00 上升的数据 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] is None: ten_up_cnt = 0 else: ten_up_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] # 15:00 的上升榜写入数据 values = int(fifteen_hour_play_cnt) - (int(v_play_cnt) + int(ten_up_cnt)) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i), [[values]]) Common.logger().info("15:00数据更新成功:{}", values) elif (upload_data == yesterday or upload_data == before_yesterday)\ and update_hour.hour == 20: Common.logger().info("满足条件: 抓取时间小于今天 and 当前时间:20点") # 当天 20:00 视频播放量 twenty_hour_play_cnt = hour_play_cnt Common.logger().info("当天 20:00 视频播放量:{}", twenty_hour_play_cnt) # 当天 10:00 上升的数据 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] is None: ten_up_cnt = 0 else: ten_up_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] # 当天 15:00 上升的数据 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] is None: fifteen_up_cnt = 0 else: fifteen_up_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] # 20:00 的上升榜写入数据 values = int(twenty_hour_play_cnt) - ( int(v_play_cnt) + int(ten_up_cnt) + int(fifteen_up_cnt)) time.sleep(1) Feishu.update_values( "logs", "xiaoniangao", "ba0da4", "N" + str(i) + ":" + "N" + str(i), [[values]]) Common.logger().info("20:00数据更新成功:{}", values) except Exception as e: Common.logger().error("视频详情:{}，异常:{}", v_title, e) except Exception as e: Common.logger().error("获取小时榜数据异常:{}", e) # 下载/上传 @classmethod def download_and_publish(cls): """ 1.从云文档中去重: https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=yatRv2 2.从云文档中下载符合规则的视频:https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=ba0da4 2.1 当日 10:00 or 15:00 or 20:00 视频播放量上升 > 5000 2.2 当日 10:00 and 15:00 视频播放量上升 > 2000 2.3 当日 15:00 and 20:00 视频播放量上升 > 2000 2.4 昨日 20:00 and 今日 10:00 视频播放量上升 > 2000 3.上传 """ try: time.sleep(1) if len(Feishu.get_values_batch("logs", "xiaoniangao", "ba0da4")) == 2: Common.logger().info("当前工作表无数据") else: time.sleep(1) for i in range(3, len(Feishu.get_values_batch("logs", "xiaoniangao", "ba0da4"))+1): time.sleep(1) Common.logger().info("分析第:{}行视频信息是否符合下载规则", i) # 略过空行 if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "D" + str(i) + ":" + "D" + str(i))[0] is None\ and Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "C"+str(i)+":"+"C"+str(i))[0] is None\ and Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "A"+str(i)+":"+"A"+str(i))[0] is None: Common.logger().info("空行，略过") else: # 今日 10:00 数据上升量 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] is None: ten_cnt = 0 else: ten_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "L" + str(i) + ":" + "L" + str(i))[0] # 今日 15:00 数据上升量 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] is None: fifteen_cnt = 0 else: fifteen_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "M" + str(i) + ":" + "M" + str(i))[0] # 今日 20:00 数据上升量 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "N" + str(i) + ":" + "N" + str(i))[0] is None: twenty_cnt = 0 else: twenty_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "N" + str(i) + ":" + "N" + str(i))[0] # 昨日 20:00 数据上升量 time.sleep(1) if Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "Q" + str(i) + ":" + "Q" + str(i))[0] is None: yesterday_twenty_cnt = 0 else: yesterday_twenty_cnt = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "Q" + str(i) + ":" + "Q" + str(i))[0] # 视频标题 time.sleep(1) v_title = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "D" + str(i) + ":" + "D" + str(i))[0] Common.logger().info("视频详情，video_title:{}，{}", v_title, type(v_title)) # 视频 ID time.sleep(1) v_id = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "C" + str(i) + ":" + "C" + str(i))[0] Common.logger().info("视频详情，video_id:{}，{}", v_id, type(v_id)) # profile_id，用户 ID time.sleep(1) p_id = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "A" + str(i) + ":" + "A" + str(i))[0] # 视频时长 time.sleep(1) v_duration = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "F" + str(i) + ":" + "F" + str(i))[0] # profile_mid time.sleep(1) p_mid = Feishu.get_range_value( "logs", "xiaoniangao", "ba0da4", "B" + str(i) + ":" + "B" + str(i))[0] Common.logger().info("10:00 / 15:00 / 20:00 上升量: {} / {} / {}", ten_cnt, fifteen_cnt, twenty_cnt) # 判断视频 ID 长度，不大于 13 位 if len(str(v_id)) > 13: Common.logger().info("视频ID长度大于13位:{}", v_id) # 判断视频时长：1-10min elif int(v_duration) > 600 or int(v_duration) < 60: Common.logger().info("视频时长不在 1-10分钟 之间") # 过滤带字母的视频ID elif any(word if word in v_id else False for word in cls.sensitive_videoid_words()) is True: Common.logger().info("视频ID带字母:{}".format(v_id)) # 从云文档中去重:https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=yatRv2 elif v_id in [j for i in Feishu.get_values_batch("logs", "xiaoniangao", "yatRv2") for j in i]: Common.logger().info("该视频已下载:{}", v_title) time.sleep(1) # 上升榜判断逻辑，任意时间段上升量>=5000，连续两个时间段上升量>=2000 elif int(ten_cnt) >= 5000 or int(fifteen_cnt) >= 5000 or int(twenty_cnt) >= 5000: Common.logger().info("10:00 or 15:00 or 20:00 数据上升量:{} or {} or {} >= 5000", ten_cnt, fifteen_cnt, twenty_cnt) Common.logger().info("满足下载规则，开始下载视频") try: url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id" headers = { "x-b3-traceid": cls.hour_x_b3_traceid, "X-Token-Id": cls.hour_x_token_id, "uid": cls.hour_uid, "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": cls.hour_referer } data = { "play_src": "1", "profile_id": int(p_id), "profile_mid": int(p_mid), "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/" "!400x400r/crop/400x400/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail" "/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "no_comments": True, "no_follow": True, "vid": v_id, "hot_l1_comment": True, "token": cls.hour_token, "uid": cls.hour_uid, "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0", "log_common_params": { "e": [{ "data": { "page": "dynamicSharePage" } }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" } } urllib3.disable_warnings() r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False) hour_play_cnt = r.json()["data"]["play_pv"] hour_cover_url = r.json()["data"]["url"] hour_video_url = r.json()["data"]["v_url"] hour_video_duration = r.json()["data"]["du"] hour_video_comment_cnt = r.json()["data"]["comment_count"] hour_video_like_cnt = r.json()["data"]["favor"]["total"] hour_video_share_cnt = r.json()["data"]["share"] hour_video_width = r.json()["data"]["w"] hour_video_height = r.json()["data"]["h"] hour_video_resolution = str(hour_video_width) + "*" + str(hour_video_height) hour_video_send_time = r.json()["data"]["t"] hour_user_name = r.json()["data"]["user"]["nick"] hour_head_url = r.json()["data"]["user"]["hurl"] Common.logger().info("视频详情，当前播放量:{}", hour_play_cnt) # 下载封面 Common.download_method("logs", "cover", v_title, hour_cover_url) # 下载视频 Common.download_method("logs", "video", v_title, hour_video_url) # 保存视频信息至 "./videos/{download_video_title}/info.txt" with open("./videos/" + v_title + "/" + "info.txt", "a", encoding="UTF-8") as f_a: f_a.write(str(v_id) + "\n" + str(v_title) + "\n" + str(int(int(hour_video_duration) / 1000)) + "\n" + str(hour_play_cnt) + "\n" + str(hour_video_comment_cnt) + "\n" + str(hour_video_like_cnt) + "\n" + str(hour_video_share_cnt) + "\n" + str(hour_video_resolution) + "\n" + str(hour_video_send_time) + "\n" + str(hour_user_name) + "\n" + str(hour_head_url) + "\n" + str(hour_video_url) + "\n" + str(hour_cover_url) + "\n" + str("90747742180aeb22c0fe3a3c6a38f3d9")) Common.logger().info("==========视频信息已保存至info.txt==========") Common.logger().info("开始上传视频") Publish.upload_and_publish("prod", "up") Common.logger().info("视频上传完成:{}", v_title) # 上传完成时间 upload_time = int(time.time()) # 保存视频信息到云文档 Common.logger().info("添加视频到云文档:{}", v_title) # 插入空行 time.sleep(1) Feishu.insert_columns("logs", "xiaoniangao", "yatRv2", "ROWS", 1, 2) # 视频信息写入云文档 values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(upload_time))), "小时级上升榜", v_id, v_title, hour_play_cnt, hour_video_comment_cnt, hour_video_like_cnt, hour_video_share_cnt, int(hour_video_duration)/1000, hour_video_resolution, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(hour_video_send_time) / 1000)), hour_user_name, p_id, p_mid, hour_head_url, hour_cover_url, hour_video_url]] time.sleep(1) Feishu.update_values("logs", "xiaoniangao", "yatRv2", "A2:Q2", values) except Exception as e: Common.logger().error("下载视频异常:{}", e) elif int(ten_cnt) >= 2000 and int(fifteen_cnt) >= 2000: Common.logger().info("10:00 and 15:00 数据上升量:{} and {} >= 2000", ten_cnt, fifteen_cnt) Common.logger().info("满足下载规则，开始下载视频") try: url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id" headers = { "x-b3-traceid": cls.hour_x_b3_traceid, "X-Token-Id": cls.hour_x_token_id, "uid": cls.hour_uid, "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": cls.hour_referer } data = { "play_src": "1", "profile_id": int(p_id), "profile_mid": int(p_mid), "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/" "!400x400r/crop/400x400/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail" "/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "no_comments": True, "no_follow": True, "vid": v_id, "hot_l1_comment": True, "token": cls.hour_token, "uid": cls.hour_uid, "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0", "log_common_params": { "e": [{ "data": { "page": "dynamicSharePage" } }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" } } urllib3.disable_warnings() r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False) hour_play_cnt = r.json()["data"]["play_pv"] hour_cover_url = r.json()["data"]["url"] hour_video_url = r.json()["data"]["v_url"] hour_video_duration = r.json()["data"]["du"] hour_video_comment_cnt = r.json()["data"]["comment_count"] hour_video_like_cnt = r.json()["data"]["favor"]["total"] hour_video_share_cnt = r.json()["data"]["share"] hour_video_width = r.json()["data"]["w"] hour_video_height = r.json()["data"]["h"] hour_video_resolution = str(hour_video_width) + "*" + str(hour_video_height) hour_video_send_time = r.json()["data"]["t"] hour_user_name = r.json()["data"]["user"]["nick"] hour_head_url = r.json()["data"]["user"]["hurl"] Common.logger().info("视频详情，当前播放量:{}", hour_play_cnt) # 下载封面 Common.download_method("logs", "cover", v_title, hour_cover_url) # 下载视频 Common.download_method("logs", "video", v_title, hour_video_url) # 保存视频信息至 "./videos/{download_video_title}/info.txt" with open("./videos/" + v_title + "/" + "info.txt", "a", encoding="UTF-8") as f_a: f_a.write(str(v_id) + "\n" + str(v_title) + "\n" + str(int(int(hour_video_duration) / 1000)) + "\n" + str(hour_play_cnt) + "\n" + str(hour_video_comment_cnt) + "\n" + str(hour_video_like_cnt) + "\n" + str(hour_video_share_cnt) + "\n" + str(hour_video_resolution) + "\n" + str(hour_video_send_time) + "\n" + str(hour_user_name) + "\n" + str(hour_head_url) + "\n" + str(hour_video_url) + "\n" + str(hour_cover_url) + "\n" + str("90747742180aeb22c0fe3a3c6a38f3d9")) Common.logger().info("==========视频信息已保存至info.txt==========") Common.logger().info("开始上传视频") Publish.upload_and_publish("prod", "up") Common.logger().info("视频上传完成:{}", v_title) # 上传完成时间 upload_time = int(time.time()) # 保存视频信息到云文档 Common.logger().info("添加视频到云文档:{}", v_title) # 插入空行 time.sleep(1) Feishu.insert_columns("logs", "xiaoniangao", "yatRv2", "ROWS", 1, 2) # 视频信息写入云文档 values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(upload_time))), "小时级上升榜", v_id, v_title, hour_play_cnt, hour_video_comment_cnt, hour_video_like_cnt, hour_video_share_cnt, int(hour_video_duration)/1000, hour_video_resolution, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(hour_video_send_time) / 1000)), hour_user_name, p_id, p_mid, hour_head_url, hour_cover_url, hour_video_url]] time.sleep(1) Feishu.update_values("logs", "xiaoniangao", "yatRv2", "A2:Q2", values) except Exception as e: Common.logger().error("下载视频异常:{}", e) elif int(fifteen_cnt) >= 2000 and int(twenty_cnt) >= 2000: Common.logger().info("15:00 and 20:00 数据上升量:{} and {} >= 2000", fifteen_cnt, twenty_cnt) Common.logger().info("满足下载规则，开始下载视频") try: url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id" headers = { "x-b3-traceid": cls.hour_x_b3_traceid, "X-Token-Id": cls.hour_x_token_id, "uid": cls.hour_uid, "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": cls.hour_referer } data = { "play_src": "1", "profile_id": int(p_id), "profile_mid": int(p_mid), "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/" "!400x400r/crop/400x400/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail" "/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "no_comments": True, "no_follow": True, "vid": v_id, "hot_l1_comment": True, "token": cls.hour_token, "uid": cls.hour_uid, "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0", "log_common_params": { "e": [{ "data": { "page": "dynamicSharePage" } }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" } } urllib3.disable_warnings() r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False) hour_play_cnt = r.json()["data"]["play_pv"] hour_cover_url = r.json()["data"]["url"] hour_video_url = r.json()["data"]["v_url"] hour_video_duration = r.json()["data"]["du"] hour_video_comment_cnt = r.json()["data"]["comment_count"] hour_video_like_cnt = r.json()["data"]["favor"]["total"] hour_video_share_cnt = r.json()["data"]["share"] hour_video_width = r.json()["data"]["w"] hour_video_height = r.json()["data"]["h"] hour_video_resolution = str(hour_video_width) + "*" + str(hour_video_height) hour_video_send_time = r.json()["data"]["t"] hour_user_name = r.json()["data"]["user"]["nick"] hour_head_url = r.json()["data"]["user"]["hurl"] Common.logger().info("视频详情，当前播放量:{}", hour_play_cnt) # 下载封面 Common.download_method("logs", "cover", v_title, hour_cover_url) # 下载视频 Common.download_method("logs", "video", v_title, hour_video_url) # 保存视频信息至 "./videos/{download_video_title}/info.txt" with open("./videos/" + v_title + "/" + "info.txt", "a", encoding="UTF-8") as f_a: f_a.write(str(v_id) + "\n" + str(v_title) + "\n" + str(int(int(hour_video_duration) / 1000)) + "\n" + str(hour_play_cnt) + "\n" + str(hour_video_comment_cnt) + "\n" + str(hour_video_like_cnt) + "\n" + str(hour_video_share_cnt) + "\n" + str(hour_video_resolution) + "\n" + str(hour_video_send_time) + "\n" + str(hour_user_name) + "\n" + str(hour_head_url) + "\n" + str(hour_video_url) + "\n" + str(hour_cover_url) + "\n" + str("90747742180aeb22c0fe3a3c6a38f3d9")) Common.logger().info("==========视频信息已保存至info.txt==========") Common.logger().info("开始上传视频") Publish.upload_and_publish("prod", "up") Common.logger().info("视频上传完成:{}", v_title) # 上传完成时间 upload_time = int(time.time()) # 保存视频信息到云文档 Common.logger().info("添加视频到云文档:{}", v_title) # 插入空行 time.sleep(1) Feishu.insert_columns("logs", "xiaoniangao", "yatRv2", "ROWS", 1, 2) # 视频信息写入云文档 values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(upload_time))), "小时级上升榜", v_id, v_title, hour_play_cnt, hour_video_comment_cnt, hour_video_like_cnt, hour_video_share_cnt, int(hour_video_duration)/1000, hour_video_resolution, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(hour_video_send_time) / 1000)), hour_user_name, p_id, p_mid, hour_head_url, hour_cover_url, hour_video_url]] time.sleep(1) Feishu.update_values("logs", "xiaoniangao", "yatRv2", "A2:Q2", values) except Exception as e: Common.logger().error("下载视频异常:{}", e) elif int(yesterday_twenty_cnt) >= 2000 and int(ten_cnt) >= 2000: Common.logger().info("昨日20:00 and 今日10:00 数据上升量:{} and {} >= 2000", yesterday_twenty_cnt, ten_cnt) Common.logger().info("满足下载规则，开始下载视频") try: url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id" headers = { "x-b3-traceid": cls.hour_x_b3_traceid, "X-Token-Id": cls.hour_x_token_id, "uid": cls.hour_uid, "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": cls.hour_referer } data = { "play_src": "1", "profile_id": int(p_id), "profile_mid": int(p_mid), "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/" "!400x400r/crop/400x400/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail" "/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "no_comments": True, "no_follow": True, "vid": v_id, "hot_l1_comment": True, "token": cls.hour_token, "uid": cls.hour_uid, "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0", "log_common_params": { "e": [{ "data": { "page": "dynamicSharePage" } }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" } } urllib3.disable_warnings() r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False) hour_play_cnt = r.json()["data"]["play_pv"] hour_cover_url = r.json()["data"]["url"] hour_video_url = r.json()["data"]["v_url"] hour_video_duration = r.json()["data"]["du"] hour_video_comment_cnt = r.json()["data"]["comment_count"] hour_video_like_cnt = r.json()["data"]["favor"]["total"] hour_video_share_cnt = r.json()["data"]["share"] hour_video_width = r.json()["data"]["w"] hour_video_height = r.json()["data"]["h"] hour_video_resolution = str(hour_video_width) + "*" + str(hour_video_height) hour_video_send_time = r.json()["data"]["t"] hour_user_name = r.json()["data"]["user"]["nick"] hour_head_url = r.json()["data"]["user"]["hurl"] Common.logger().info("视频详情，当前播放量:{}", hour_play_cnt) # 下载封面 Common.download_method("logs", "cover", v_title, hour_cover_url) # 下载视频 Common.download_method("logs", "video", v_title, hour_video_url) # 保存视频信息至 "./videos/{download_video_title}/info.txt" with open("./videos/" + v_title + "/" + "info.txt", "a", encoding="UTF-8") as f_a: f_a.write(str(v_id) + "\n" + str(v_title) + "\n" + str(int(int(hour_video_duration) / 1000)) + "\n" + str(hour_play_cnt) + "\n" + str(hour_video_comment_cnt) + "\n" + str(hour_video_like_cnt) + "\n" + str(hour_video_share_cnt) + "\n" + str(hour_video_resolution) + "\n" + str(hour_video_send_time) + "\n" + str(hour_user_name) + "\n" + str(hour_head_url) + "\n" + str(hour_video_url) + "\n" + str(hour_cover_url) + "\n" + str("90747742180aeb22c0fe3a3c6a38f3d9")) Common.logger().info("==========视频信息已保存至info.txt==========") Common.logger().info("开始上传视频") Publish.upload_and_publish("prod", "up") Common.logger().info("视频上传完成:{}", v_title) # 上传完成时间 upload_time = int(time.time()) # 保存视频信息到云文档 Common.logger().info("添加视频到云文档:{}", v_title) # 插入空行 time.sleep(1) Feishu.insert_columns("logs", "xiaoniangao", "yatRv2", "ROWS", 1, 2) # 视频信息写入云文档 values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(upload_time))), "小时级上升榜", v_id, v_title, hour_play_cnt, hour_video_comment_cnt, hour_video_like_cnt, hour_video_share_cnt, int(hour_video_duration)/1000, hour_video_resolution, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(hour_video_send_time) / 1000)), hour_user_name, p_id, p_mid, hour_head_url, hour_cover_url, hour_video_url]] time.sleep(1) Feishu.update_values("logs", "xiaoniangao", "yatRv2", "A2:Q2", values) except Exception as e: Common.logger().error("下载视频异常:{}", e) else: Common.logger().info("上升量不满足下载规则") except Exception as e: Common.logger().error("下载视频时，获取飞书信息异常:{}", e) if __name__ == "__main__": hour_list = HourList() # hour_list.get_hour_list_feeds() # hour_list.del_null_rows("xiaoniangao", "ba0da4", 3) # hour_list.update_hour_list_data( # today=datetime.datetime.now().strftime("%Y-%m-%d"), # yesterday=(datetime.date.today() + datetime.timedelta(days=-1)).strftime("%Y-%m-%d"), # before_yesterday=(datetime.date.today() + datetime.timedelta(days=-2)).strftime("%Y-%m-%d")) hour_list.download_and_publish()