# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2023/3/15 import datetime import json import os import random import shutil import sys import time import requests import urllib3 sys.path.append(os.getcwd()) from common.common import Common from common.feishu import Feishu from common.publish import Publish from common.scheduling_db import MysqlHelper from common.public import get_config_from_mysql proxies = {"http": None, "https": None} class XiaoniangaoHourScheduling: platform = "小年糕" # 生成 uid、token @classmethod def get_uid_token(cls): words = "abcdefghijklmnopqrstuvwxyz0123456789" uid = f"""{"".join(random.sample(words, 8))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 12))}""" token = "".join(random.sample(words, 32)) uid_token_dict = { "uid": uid, "token": token } return uid_token_dict # 基础门槛规则 @staticmethod def download_rule(log_type, crawler, video_dict, rule_dict): """ 下载视频的基本规则 :param log_type: 日志 :param crawler: 哪款爬虫 :param video_dict: 视频信息，字典格式 :param rule_dict: 规则信息，字典格式 :return: 满足规则，返回 True；反之，返回 False """ rule_playCnt_min = rule_dict.get('playCnt', {}).get('min', 0) rule_playCnt_max = rule_dict.get('playCnt', {}).get('max', 100000000) if rule_playCnt_max == 0: rule_playCnt_max = 100000000 rule_duration_min = rule_dict.get('duration', {}).get('min', 0) rule_duration_max = rule_dict.get('duration', {}).get('max', 100000000) if rule_duration_max == 0: rule_duration_max = 100000000 rule_period_min = rule_dict.get('period', {}).get('min', 0) # rule_period_max = rule_dict.get('period', {}).get('max', 100000000) # if rule_period_max == 0: # rule_period_max = 100000000 # # rule_fans_min = rule_dict.get('fans', {}).get('min', 0) # rule_fans_max = rule_dict.get('fans', {}).get('max', 100000000) # if rule_fans_max == 0: # rule_fans_max = 100000000 # # rule_videos_min = rule_dict.get('videos', {}).get('min', 0) # rule_videos_max = rule_dict.get('videos', {}).get('max', 100000000) # if rule_videos_max == 0: # rule_videos_max = 100000000 rule_like_min = rule_dict.get('like', {}).get('min', 0) rule_like_max = rule_dict.get('like', {}).get('max', 100000000) if rule_like_max == 0: rule_like_max = 100000000 rule_videoWidth_min = rule_dict.get('videoWidth', {}).get('min', 0) rule_videoWidth_max = rule_dict.get('videoWidth', {}).get('max', 100000000) if rule_videoWidth_max == 0: rule_videoWidth_max = 100000000 rule_videoHeight_min = rule_dict.get('videoHeight', {}).get('min', 0) rule_videoHeight_max = rule_dict.get('videoHeight', {}).get('max', 100000000) if rule_videoHeight_max == 0: rule_videoHeight_max = 100000000 rule_shareCnt_min = rule_dict.get('shareCnt', {}).get('min', 0) rule_shareCnt_max = rule_dict.get('shareCnt', {}).get('max', 100000000) if rule_shareCnt_max == 0: rule_shareCnt_max = 100000000 rule_commentCnt_min = rule_dict.get('commentCnt', {}).get('min', 0) rule_commentCnt_max = rule_dict.get('commentCnt', {}).get('max', 100000000) if rule_commentCnt_max == 0: rule_commentCnt_max = 100000000 Common.logger(log_type, crawler).info( f'rule_duration_max:{rule_duration_max} >= duration:{int(float(video_dict["duration"]))} >= rule_duration_min:{int(rule_duration_min)}') Common.logger(log_type, crawler).info( f'rule_playCnt_max:{int(rule_playCnt_max)} >= play_cnt:{int(video_dict["play_cnt"])} >= rule_playCnt_min:{int(rule_playCnt_min)}') Common.logger(log_type, crawler).info( f'now:{int(time.time())} - publish_time_stamp:{int(video_dict["publish_time_stamp"])} <= {3600 * 24 * int(rule_period_min)}') Common.logger(log_type, crawler).info( f'rule_like_max:{int(rule_like_max)} >= like_cnt:{int(video_dict["like_cnt"])} >= rule_like_min:{int(rule_like_min)}') Common.logger(log_type, crawler).info( f'rule_commentCnt_max:{int(rule_commentCnt_max)} >= comment_cnt:{int(video_dict["comment_cnt"])} >= rule_commentCnt_min:{int(rule_commentCnt_min)}') Common.logger(log_type, crawler).info( f'rule_shareCnt_max:{int(rule_shareCnt_max)} >= share_cnt:{int(video_dict["share_cnt"])} >= rule_shareCnt_min:{int(rule_shareCnt_min)}') Common.logger(log_type, crawler).info( f'rule_videoWidth_max:{int(rule_videoWidth_max)} >= video_width:{int(video_dict["video_width"])} >= rule_videoWidth_min:{int(rule_videoWidth_min)}') Common.logger(log_type, crawler).info( f'rule_videoHeight_max:{int(rule_videoHeight_max)} >= video_height:{int(video_dict["video_height"])} >= rule_videoHeight_min:{int(rule_videoHeight_min)}') if int(rule_duration_max) >= int(float(video_dict["duration"])) >= int(rule_duration_min) \ and int(rule_playCnt_max) >= int(video_dict['play_cnt']) >= int(rule_playCnt_min) \ and int(time.time()) - int(video_dict["publish_time_stamp"]) <= 3600 * 24 * int(rule_period_min) \ and int(rule_like_max) >= int(video_dict['like_cnt']) >= int(rule_like_min) \ and int(rule_commentCnt_max) >= int(video_dict['comment_cnt']) >= int(rule_commentCnt_min) \ and int(rule_shareCnt_max) >= int(video_dict['share_cnt']) >= int(rule_shareCnt_min) \ and int(rule_videoWidth_max) >= int(video_dict['video_width']) >= int(rule_videoWidth_min) \ and int(rule_videoHeight_max) >= int(video_dict['video_height']) >= int(rule_videoHeight_min): return True else: return False @classmethod def repeat_video(cls, log_type, crawler, video_id, env): sql = f""" select * from crawler_video where platform="小年糕" and out_video_id="{video_id}"; """ repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env) return len(repeat_video) @classmethod def repeat_hour(cls, log_type, crawler, video_id, env): sql = f""" select * from crawler_xiaoniangao_hour where platform="小年糕" and out_video_id="{video_id}"; """ repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env) return len(repeat_video) # 获取列表 @classmethod def get_videoList(cls, log_type, crawler, rule_dict, env): uid_token_dict = cls.get_uid_token() url = "https://kapi.xiaoniangao.cn/trends/get_recommend_trends" headers = { "x-b3-traceid": '1c403a4aa72e3c', "X-Token-Id": 'ab619e96d801f1567388629260aa68ec-1202200806', "uid": uid_token_dict['uid'], "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/624/page-frame.html' } data = { "log_params": { "page": "discover_rec", "common": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.2", "net": "wifi", "scene": 1089 } }, "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!750x500r/crop/750x500/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "ext": { "fmid": 0, "items": {} }, "app": "xng", "rec_scene": "discover_rec", "log_common_params": { "e": [{ "data": { "page": "discoverIndexPage", "topic": "recommend" }, "ab": {} }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" }, "refresh": False, "token": uid_token_dict["token"], "uid": uid_token_dict["uid"], "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0" } urllib3.disable_warnings() r = requests.post(url=url, headers=headers, json=data, proxies=proxies, verify=False) if 'data' not in r.text or r.status_code != 200: Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n") return elif "data" not in r.json(): Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()}\n") return elif "list" not in r.json()["data"]: Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']}\n") return elif len(r.json()['data']['list']) == 0: Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']['list']}\n") return else: # 视频列表数据 feeds = r.json()["data"]["list"] for i in range(len(feeds)): # 标题，表情随机加在片头、片尾，或替代句子中间的标点符号 xiaoniangao_title = feeds[i].get("title", "").strip().replace("\n", "") \ .replace("/", "").replace("\r", "").replace("#", "") \ .replace(".", "。").replace("\\", "").replace("&NBSP", "") \ .replace(":", "").replace("*", "").replace("？", "") \ .replace("?", "").replace('"', "").replace("<", "") \ .replace(">", "").replace("|", "").replace(" ", "")\ .replace('"', '').replace("'", '') # 随机取一个表情/符号 emoji = random.choice(get_config_from_mysql(log_type, crawler, env, "emoji")) # 生成最终标题，标题list[表情+title, title+表情]随机取一个 video_title = random.choice([f"{emoji}{xiaoniangao_title}", f"{xiaoniangao_title}{emoji}"]) # 视频 ID video_id = feeds[i].get("vid", "") # 播放量 play_cnt = feeds[i].get("play_pv", 0) # 点赞量 like_cnt = feeds[i].get("favor", {}).get("total", 0) # 评论数 comment_cnt = feeds[i].get("comment_count", 0) # 分享量 share_cnt = feeds[i].get("share", 0) # 时长 duration = int(feeds[i].get("du", 0)/1000) # 宽和高 video_width = int(feeds[i].get("w", 0)) video_height = int(feeds[i].get("h", 0)) # 发布时间 publish_time_stamp = int(int(feeds[i].get("t", 0))/1000) publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp)) # 用户名 / 头像 user_name = feeds[i].get("user", {}).get("nick", "").strip().replace("\n", "") \ .replace("/", "").replace("快手", "").replace(" ", "") \ .replace(" ", "").replace("&NBSP", "").replace("\r", "") avatar_url = feeds[i].get("user", {}).get("hurl", "") # 用户 ID profile_id = feeds[i]["id"] # 用户 mid profile_mid = feeds[i]["user"]["mid"] # 视频封面 cover_url = feeds[i].get("url", "") # 视频播放地址 video_url = feeds[i].get("v_url", "") video_dict = { "video_title": video_title, "video_id": video_id, "duration": duration, "play_cnt": play_cnt, "like_cnt": like_cnt, "comment_cnt": comment_cnt, "share_cnt": share_cnt, "user_name": user_name, "publish_time_stamp": publish_time_stamp, "publish_time_str": publish_time_str, "video_width": video_width, "video_height": video_height, "avatar_url": avatar_url, "profile_id": profile_id, "profile_mid": profile_mid, "cover_url": cover_url, "video_url": video_url, "session": f"xiaoniangao-hour-{int(time.time())}" } for k, v in video_dict.items(): Common.logger(log_type, crawler).info(f"{k}:{v}") # 过滤无效视频 if video_title == "" or video_id == "" or video_url == "": Common.logger(log_type, crawler).warning("无效视频\n") # 抓取基础规则过滤 elif cls.download_rule(log_type, crawler, video_dict, rule_dict) is False: Common.logger(log_type, crawler).info("不满足抓取规则\n") elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env) != 0: Common.logger(log_type, crawler).info('视频已下载\n') # 过滤敏感词 elif any(str(word) if str(word) in video_title else False for word in get_config_from_mysql(log_type, crawler, env, "filter", action="")) is True: Common.logger(log_type, crawler).info("视频已中过滤词\n") else: # 写入飞书小时级feeds数据库表 insert_sql = f""" insert into crawler_xiaoniangao_hour(profile_id, profile_mid, platform, out_video_id, video_title, user_name, cover_url, video_url, duration, publish_time, play_cnt, crawler_time_stamp, crawler_time) values({profile_id}, {profile_mid}, "{cls.platform}", "{video_id}", "{video_title}", "{user_name}", "{cover_url}", "{video_url}", {duration}, "{publish_time_str}", {play_cnt}, {int(time.time())}, "{time.strftime("%Y-%y-%d %H:%M:%S", time.localtime(int(time.time())))}" )""" Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}") MysqlHelper.update_values(log_type, crawler, insert_sql, env) Common.logger(log_type, crawler).info('视频信息插入数据库成功！\n') @classmethod def get_video_info(cls, log_type, crawler, p_id, p_mid, v_title, v_id): uid_token_dict = cls.get_uid_token() url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id" headers = { "x-b3-traceid": '1c403a4aa72e3c', "X-Token-Id": 'ab619e96d801f1567388629260aa68ec-1202200806', "uid": uid_token_dict['uid'], "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/624/page-frame.html' } data = { "play_src": "1", "profile_id": int(p_id), "profile_mid": int(p_mid), "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/" "!400x400r/crop/400x400/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail" "/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "no_comments": True, "no_follow": True, "vid": v_id, "hot_l1_comment": True, "token": uid_token_dict['token'], "uid": uid_token_dict['uid'], "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0", "log_common_params": { "e": [{ "data": { "page": "dynamicSharePage" } }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" } } urllib3.disable_warnings() r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False) if r.status_code != 200 or 'data' not in r.text: Common.logger(log_type, crawler).warning(f"get_videoInfo:{r.text}\n") else: hour_play_cnt = r.json()["data"]["play_pv"] hour_cover_url = r.json()["data"]["url"] hour_video_url = r.json()["data"]["v_url"] hour_video_duration = r.json()["data"]["du"] hour_video_comment_cnt = r.json()["data"]["comment_count"] hour_video_like_cnt = r.json()["data"]["favor"]["total"] hour_video_share_cnt = r.json()["data"]["share"] hour_video_width = r.json()["data"]["w"] hour_video_height = r.json()["data"]["h"] hour_video_send_time = r.json()["data"]["t"] publish_time_stamp = int(int(hour_video_send_time) / 1000) publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp)) hour_user_name = r.json()["data"]["user"]["nick"] hour_head_url = r.json()["data"]["user"]["hurl"] video_info_dict = { "video_id": v_id, "video_title": v_title, "duration": hour_video_duration, "play_cnt": hour_play_cnt, "like_cnt": hour_video_like_cnt, "comment_cnt": hour_video_comment_cnt, "share_cnt": hour_video_share_cnt, "user_name": hour_user_name, "publish_time_stamp": publish_time_stamp, "publish_time_str": publish_time_str, "video_width": hour_video_width, "video_height": hour_video_height, "avatar_url": hour_head_url, "profile_id": p_id, "profile_mid": p_mid, "cover_url": hour_cover_url, "video_url": hour_video_url, "session": f"xiaoniangao-hour-{int(time.time())}" } return video_info_dict # 更新小时榜数据 @classmethod def update_videoList(cls, log_type, crawler, rule_dict, strategy, oss_endpoint, env): """ 更新小时榜数据 """ befor_yesterday = (datetime.date.today() + datetime.timedelta(days=-3)).strftime("%Y-%m-%d %H:%M:%S") update_time_stamp = int(time.mktime(time.strptime(befor_yesterday, "%Y-%m-%d %H:%M:%S"))) select_sql = f""" select * from crawler_xiaoniangao_hour where crawler_time_stamp >= {update_time_stamp} GROUP BY out_video_id """ update_video_list = MysqlHelper.get_values(log_type, crawler, select_sql, env) if len(update_video_list) == 0: Common.logger(log_type, crawler).info("暂无需要更新的小时榜数据\n") return for update_video_info in update_video_list: profile_id = update_video_info["profile_id"] profile_mid = update_video_info["profile_mid"] video_title = update_video_info["video_title"] video_id = update_video_info["out_video_id"] if datetime.datetime.now().hour == 10 and datetime.datetime.now().minute <= 10: video_info_dict = cls.get_video_info(log_type=log_type, crawler=crawler, p_id=profile_id, p_mid=profile_mid, v_title=video_title, v_id=video_id) ten_play_cnt = video_info_dict['play_cnt'] Common.logger(log_type, crawler).info(f"ten_play_cnt:{ten_play_cnt}") update_sql = f""" update crawler_xiaoniangao_hour set ten_play_cnt={ten_play_cnt} WHERE out_video_id="{video_id}"; """ # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}") MysqlHelper.update_values(log_type, crawler, update_sql, env) cls.download_publish(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, update_video_info=update_video_info, strategy=strategy, oss_endpoint=oss_endpoint, env=env) elif datetime.datetime.now().hour == 15 and datetime.datetime.now().minute <= 10: video_info_dict = cls.get_video_info(log_type=log_type, crawler=crawler, p_id=profile_id, p_mid=profile_mid, v_title=video_title, v_id=video_id) fifteen_play_cnt = video_info_dict['play_cnt'] Common.logger(log_type, crawler).info(f"fifteen_play_cnt:{fifteen_play_cnt}") update_sql = f""" update crawler_xiaoniangao_hour set fifteen_play_cnt={fifteen_play_cnt} WHERE out_video_id="{video_id}"; """ # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}") MysqlHelper.update_values(log_type, crawler, update_sql, env) cls.download_publish(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, update_video_info=update_video_info, strategy=strategy, oss_endpoint=oss_endpoint, env=env) elif datetime.datetime.now().hour == 20 and datetime.datetime.now().minute <= 10: video_info_dict = cls.get_video_info(log_type=log_type, crawler=crawler, p_id=profile_id, p_mid=profile_mid, v_title=video_title, v_id=video_id) twenty_play_cnt = video_info_dict['play_cnt'] Common.logger(log_type, crawler).info(f"twenty_play_cnt:{twenty_play_cnt}") update_sql = f""" update crawler_xiaoniangao_hour set twenty_play_cnt={twenty_play_cnt} WHERE out_video_id="{video_id}"; """ # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}") MysqlHelper.update_values(log_type, crawler, update_sql, env) cls.download_publish(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, update_video_info=update_video_info, strategy=strategy, oss_endpoint=oss_endpoint, env=env) else: pass @classmethod def download(cls, log_type, crawler, video_info_dict, rule_dict, strategy, oss_endpoint, env): # 下载封面 Common.download_method(log_type=log_type, crawler=crawler, text="cover", title=video_info_dict["video_title"], url=video_info_dict["cover_url"]) # 下载视频 Common.download_method(log_type=log_type, crawler=crawler, text="video", title=video_info_dict["video_title"], url=video_info_dict["video_url"]) # 保存视频信息至 "./videos/{download_video_title}/info.txt" Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_info_dict) # 上传视频 Common.logger(log_type, crawler).info("开始上传视频...") our_video_id = Publish.upload_and_publish(log_type=log_type, crawler=crawler, strategy=strategy, our_uid="hour", env=env, oss_endpoint=oss_endpoint) if env == "dev": our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info" else: our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info" Common.logger(log_type, crawler).info("视频上传完成") if our_video_id is None: # 删除视频文件夹 shutil.rmtree(f"./{crawler}/videos/{video_info_dict['video_title']}") return # # 视频信息保存数据库 # rule_dict = { # "duration": {"min": 40}, # "play_cnt": {"min": 4000}, # "publish_day": {"min": 10} # } insert_sql = f""" insert into crawler_video(video_id, out_user_id, platform, strategy, out_video_id, video_title, cover_url, video_url, duration, publish_time, play_cnt, crawler_rule, width, height) values({our_video_id}, "{video_info_dict['profile_id']}", "{cls.platform}", "小时榜爬虫策略", "{video_info_dict['video_id']}", "{video_info_dict['video_title']}", "{video_info_dict['cover_url']}", "{video_info_dict['video_url']}", {int(video_info_dict['duration'])}, "{video_info_dict['publish_time_str']}", {int(video_info_dict['play_cnt'])}, '{json.dumps(rule_dict)}', {int(video_info_dict['video_width'])}, {int(video_info_dict['video_height'])}) """ Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}") MysqlHelper.update_values(log_type, crawler, insert_sql, env) Common.logger(log_type, crawler).info('视频信息插入数据库成功！') # 视频写入飞书 Feishu.insert_columns(log_type, crawler, "yatRv2", "ROWS", 1, 2) # 视频ID工作表，首行写入数据 upload_time = int(time.time()) values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)), "小时级上升榜", str(video_info_dict['video_id']), str(video_info_dict['video_title']), our_video_link, video_info_dict['play_cnt'], video_info_dict['comment_cnt'], video_info_dict['like_cnt'], video_info_dict['share_cnt'], video_info_dict['duration'], f"{video_info_dict['video_width']}*{video_info_dict['video_height']}", str(video_info_dict['publish_time_str'].replace("-", "/")), str(video_info_dict['user_name']), str(video_info_dict['profile_id']), str(video_info_dict['profile_mid']), str(video_info_dict['avatar_url']), str(video_info_dict['cover_url']), str(video_info_dict['video_url'])]] time.sleep(1) Feishu.update_values(log_type, crawler, "yatRv2", "F2:Z2", values) Common.logger(log_type, crawler).info('视频信息写入飞书成功\n') # 下载/上传 @classmethod def download_publish(cls, log_type, crawler, video_info_dict, rule_dict, update_video_info, strategy, oss_endpoint, env): if cls.repeat_video(log_type, crawler, video_info_dict["video_id"], env) != 0: Common.logger(log_type, crawler).info('视频已下载\n') # 播放量大于 50000，直接下载 elif int(video_info_dict["play_cnt"]) >= 50000: Common.logger(log_type, crawler).info( f"播放量:{video_info_dict['play_cnt']} >= 50000，满足下载规则，开始下载视频") cls.download(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, strategy=strategy, oss_endpoint=oss_endpoint, env=env) # 上升榜判断逻辑，任意时间段上升量>=5000，连续两个时间段上升量>=2000 elif int(update_video_info['ten_play_cnt']) >= 5000 or int( update_video_info['fifteen_play_cnt']) >= 5000 or int(update_video_info['twenty_play_cnt']) >= 5000: Common.logger(log_type, crawler).info( f"10:00 or 15:00 or 20:00 数据上升量:{int(update_video_info['ten_play_cnt'])} or {int(update_video_info['fifteen_play_cnt'])} or {int(update_video_info['twenty_play_cnt'])} >= 5000") Common.logger(log_type, crawler).info("满足下载规则，开始下载视频") cls.download(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, strategy=strategy, oss_endpoint=oss_endpoint, env=env) elif int(update_video_info['ten_play_cnt']) >= 2000 and int(update_video_info['fifteen_play_cnt']) >= 2000: Common.logger(log_type, crawler).info( f"10:00 and 15:00 数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['fifteen_play_cnt'])} >= 2000") Common.logger(log_type, crawler).info("满足下载规则，开始下载视频") cls.download(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, strategy=strategy, oss_endpoint=oss_endpoint, env=env) elif int(update_video_info['fifteen_play_cnt']) >= 2000 and int(update_video_info['twenty_play_cnt']) >= 2000: Common.logger(log_type, crawler).info( f"15:00 and 20:00 数据上升量:{int(update_video_info['fifteen_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 2000") Common.logger(log_type, crawler).info("满足下载规则，开始下载视频") cls.download(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, strategy=strategy, oss_endpoint=oss_endpoint, env=env) elif int(update_video_info['ten_play_cnt']) >= 2000 and int(update_video_info['twenty_play_cnt']) >= 2000: Common.logger(log_type, crawler).info( f"今日10:00 / 20:00数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 2000") Common.logger(log_type, crawler).info("满足下载规则，开始下载视频") cls.download(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, strategy=strategy, oss_endpoint=oss_endpoint, env=env) else: Common.logger(log_type, crawler).info("上升量不满足下载规则") if __name__ == "__main__": print(get_config_from_mysql(log_type='hour', source='xiaoniangao', env='dev', text='filter')) # print(XiaoniangaoHour.get_uid_token()) # XiaoniangaoHour.get_videoList("test", "xiaoniangao", "dev") # XiaoniangaoHour.update_videoList("test", "xiaoniangao", "小时榜爬虫策略", "out", "dev") pass