# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2023/3/15 import datetime import json import os import random import shutil import sys import time from hashlib import md5 import requests import urllib3 from common.mq import MQ sys.path.append(os.getcwd()) from common.common import Common from common.feishu import Feishu from common.publish import Publish from common.scheduling_db import MysqlHelper from common.public import get_config_from_mysql, download_rule proxies = {"http": None, "https": None} class XiaoniangaoHourScheduling: platform = "小年糕" words = "abcdefghijklmnopqrstuvwxyz0123456789" uid_token_dict = { "uid": f"""{"".join(random.sample(words, 8))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 4))}-{"".join(random.sample(words, 12))}""", "token": "".join(random.sample(words, 32)) } @classmethod def repeat_video(cls, log_type, crawler, video_id, env): # sql = f""" select * from crawler_video where platform="小年糕" and out_video_id="{video_id}"; """ sql = f""" select * from crawler_video where platform in ("{crawler}","{cls.platform}") and out_video_id="{video_id}"; """ repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env) return len(repeat_video) @classmethod def repeat_hour(cls, log_type, crawler, video_id, env): sql = f""" select * from crawler_xiaoniangao_hour where platform="小年糕" and out_video_id="{video_id}"; """ repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env) return len(repeat_video) # 获取列表 @classmethod def get_videoList(cls, log_type, crawler, rule_dict, env): uid_token_dict = cls.uid_token_dict url = "https://kapi-xng-app.xiaoniangao.cn/v2/trends/recommend" payload = "{\"topic_name\":\"recommend\",\"ext\":{\"current_item\":0,\"items\":[]},\"tag_id\":0,\"refresh\":false,\"share_width\":300,\"share_height\":240,\"staggered_style\":0,\"qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!750x500r\\/interlace\\/1\\/format\\/jpg\",\"topic_id\":2,\"h_qs\":\"imageMogr2\\/gravity\\/center\\/rotate\\/$\\/thumbnail\\/!80x80r\\/interlace\\/1\\/format\\/jpg\",\"log\":{\"net\":\"wifi\",\"product\":\"xng\",\"uid\":\"2F310D09-5E32-5985-8644-3BCB6920E76F\",\"brand\":\"iPad\",\"page\":\"\",\"session_id\":\"71C77648-3224-4083-894C-B6282131F286\",\"resolution\":\"750*1334\",\"pf\":\"4\",\"app_version\":\"1.22.5\",\"device\":\"iPad Pro (12.9-inch) (3rd generation)\",\"os_version\":\"15.7\",\"idfa\":\"\",\"channel\":\"ios_app_store\"},\"token\":\"\"}" headers = { 'Host': 'kapi-xng-app.xiaoniangao.cn', 'content-type': 'application/json; charset=utf-8', 'accept': 'application/json', 'authorization': 'PsrUTBCQ5G7UVZdgx+JxymPHcKU=', 'verb': 'POST', 'content-md5': '08fa0e6bf725fd6ef83c16d2ceb8a544', 'x-b3-traceid': '45a6c5b4c471eecc', 'accept-language': 'zh-Hans-CN;q=1.0', 'date': 'Mon, 19 Jun 2023 09:47:40 GMT', 'x-token-id': '', 'x-signaturemethod': 'hmac-sha1', 'user-agent': 'xngapp/1.22.5 (cn.xiaoniangao.xngapp; build:157; iOS 15.7.0) Alamofire/5.2.2' } urllib3.disable_warnings() r = requests.post(url=url, headers=headers, data=payload, proxies=proxies, verify=False) if 'data' not in r.text or r.status_code != 200: Common.logger(log_type, crawler).warning(f"get_videoList:{r.text}\n") Common.logging(log_type, crawler, env, f"get_videoList:{r.text}\n") return elif "data" not in r.json(): Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()}\n") Common.logging(log_type, crawler, env, f"get_videoList:{r.json()}\n") return elif "list" not in r.json()["data"]: Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']}\n") Common.logging(log_type, crawler, env, f"get_videoList:{r.json()['data']}\n") return elif len(r.json()['data']['list']) == 0: Common.logger(log_type, crawler).warning(f"get_videoList:{r.json()['data']['list']}\n") Common.logging(log_type, crawler, env, f"get_videoList:{r.json()['data']['list']}\n") return else: # 视频列表数据 feeds = r.json()["data"]["list"] for i in range(len(feeds)): try: # 标题，表情随机加在片头、片尾，或替代句子中间的标点符号 xiaoniangao_title = feeds[i].get("title", "").strip().replace("\n", "") \ .replace("/", "").replace("\r", "").replace("#", "") \ .replace(".", "。").replace("\\", "").replace("&NBSP", "") \ .replace(":", "").replace("*", "").replace("？", "") \ .replace("?", "").replace('"', "").replace("<", "") \ .replace(">", "").replace("|", "").replace(" ", "")\ .replace('"', '').replace("'", '') # 随机取一个表情/符号 emoji = random.choice(get_config_from_mysql(log_type, crawler, env, "emoji")) # 生成最终标题，标题list[表情+title, title+表情]随机取一个 video_title = random.choice([f"{emoji}{xiaoniangao_title}", f"{xiaoniangao_title}{emoji}"]) # 发布时间 publish_time_stamp = int(int(feeds[i].get("t", 0))/1000) publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp)) # 用户名 / 头像 user_name = feeds[i].get("user", {}).get("nick", "").strip().replace("\n", "") \ .replace("/", "").replace(" ", "") \ .replace(" ", "").replace("&NBSP", "").replace("\r", "") video_dict = { "video_title": video_title, "video_id": feeds[i].get("vid", ""), "duration": int(feeds[i].get("du", 0)/1000), "play_cnt": feeds[i].get("play_pv", 0), "like_cnt": feeds[i].get("favor", {}).get("total", 0), "comment_cnt": feeds[i].get("comment_count", 0), "share_cnt": feeds[i].get("share", 0), "user_name": user_name, "publish_time_stamp": publish_time_stamp, "publish_time_str": publish_time_str, "video_width": int(feeds[i].get("vw", 0)), "video_height": int(feeds[i].get("vh", 0)), "avatar_url": feeds[i].get("user", {}).get("hurl", ""), "profile_id": feeds[i]["id"], "profile_mid": feeds[i]["user"]["mid"], "cover_url": feeds[i].get("url", ""), "video_url": feeds[i].get("v_url", ""), "session": f"xiaoniangao-hour-{int(time.time())}" } for k, v in video_dict.items(): Common.logger(log_type, crawler).info(f"{k}:{v}") Common.logging(log_type, crawler, env, f"{video_dict}") # 过滤无效视频 if video_title == "" or video_dict["video_id"] == "" or video_dict["video_url"] == "": Common.logger(log_type, crawler).warning("无效视频\n") Common.logging(log_type, crawler, env, "无效视频\n") # 抓取基础规则过滤 elif download_rule(log_type=log_type, crawler=crawler, video_dict=video_dict, rule_dict=rule_dict) is False: Common.logger(log_type, crawler).info("不满足抓取规则\n") Common.logging(log_type, crawler, env, "不满足抓取规则\n") elif any(str(word) if str(word) in video_dict["video_title"] else False for word in get_config_from_mysql(log_type=log_type, source=crawler, env=env, text="filter", action="")) is True: Common.logger(log_type, crawler).info('已中过滤词\n') Common.logging(log_type, crawler, env, '已中过滤词\n') elif cls.repeat_video(log_type, crawler, video_dict['video_id'], env) != 0: Common.logger(log_type, crawler).info('视频已下载\n') Common.logging(log_type, crawler, env, '视频已下载\n') else: # 写入飞书小时级feeds数据库表 insert_sql = f""" insert into crawler_xiaoniangao_hour(profile_id, profile_mid, platform, out_video_id, video_title, user_name, cover_url, video_url, duration, publish_time, play_cnt, crawler_time_stamp, crawler_time) values({video_dict["profile_id"]}, {video_dict["profile_mid"]}, "{cls.platform}", "{video_dict["video_id"]}", "{video_title}", "{user_name}", "{video_dict["cover_url"]}", "{video_dict["video_url"]}", {video_dict["duration"]}, "{publish_time_str}", {video_dict["play_cnt"]}, {int(time.time())}, "{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time())))}" )""" Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}") Common.logging(log_type, crawler, env, f"insert_sql:{insert_sql}") MysqlHelper.update_values(log_type, crawler, insert_sql, env) Common.logger(log_type, crawler).info('视频信息写入小时级数据库成功！\n') Common.logging(log_type, crawler, env, '视频信息写入小时级数据库成功！\n') except Exception as e: Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n") Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n") @classmethod def get_video_info(cls, log_type, crawler, p_id, p_mid, v_title, v_id): uid_token_dict = cls.uid_token_dict url = "https://kapi.xiaoniangao.cn/profile/get_profile_by_id" headers = { "x-b3-traceid": '1c403a4aa72e3c', "X-Token-Id": 'ab619e96d801f1567388629260aa68ec-1202200806', "uid": uid_token_dict['uid'], "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X)' ' AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ' 'MicroMessenger/8.0.20(0x18001432) NetType/WIFI Language/zh_CN', "Referer": 'https://servicewechat.com/wxd7911e4c177690e4/624/page-frame.html' } data = { "play_src": "1", "profile_id": int(p_id), "profile_mid": int(p_mid), "qs": "imageMogr2/gravity/center/rotate/$/thumbnail/" "!400x400r/crop/400x400/interlace/1/format/jpg", "h_qs": "imageMogr2/gravity/center/rotate/$/thumbnail" "/!80x80r/crop/80x80/interlace/1/format/jpg", "share_width": 625, "share_height": 500, "no_comments": True, "no_follow": True, "vid": v_id, "hot_l1_comment": True, "token": uid_token_dict['token'], "uid": uid_token_dict['uid'], "proj": "ma", "wx_ver": "8.0.20", "code_ver": "3.62.0", "log_common_params": { "e": [{ "data": { "page": "dynamicSharePage" } }], "ext": { "brand": "iPhone", "device": "iPhone 11", "os": "iOS 14.7.1", "weixinver": "8.0.20", "srcver": "2.24.3", "net": "wifi", "scene": "1089" }, "pj": "1", "pf": "2", "session_id": "7bcce313-b57d-4305-8d14-6ebd9a1bad29" } } urllib3.disable_warnings() r = requests.post(headers=headers, url=url, json=data, proxies=proxies, verify=False) if r.status_code != 200 or 'data' not in r.text: Common.logger(log_type, crawler).warning(f"get_videoInfo:{r.text}\n") else: hour_play_cnt = r.json()["data"]["play_pv"] hour_cover_url = r.json()["data"]["url"] hour_video_url = r.json()["data"]["v_url"] hour_video_duration = r.json()["data"]["du"] hour_video_comment_cnt = r.json()["data"]["comment_count"] hour_video_like_cnt = r.json()["data"]["favor"]["total"] hour_video_share_cnt = r.json()["data"]["share"] hour_video_width = r.json()["data"]["w"] hour_video_height = r.json()["data"]["h"] hour_video_send_time = r.json()["data"]["t"] publish_time_stamp = int(int(hour_video_send_time) / 1000) publish_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp)) hour_user_name = r.json()["data"]["user"]["nick"] hour_head_url = r.json()["data"]["user"]["hurl"] video_info_dict = { "video_id": v_id, "video_title": v_title, "duration": hour_video_duration, "play_cnt": hour_play_cnt, "like_cnt": hour_video_like_cnt, "comment_cnt": hour_video_comment_cnt, "share_cnt": hour_video_share_cnt, "user_name": hour_user_name, "publish_time_stamp": publish_time_stamp, "publish_time_str": publish_time_str, "video_width": hour_video_width, "video_height": hour_video_height, "avatar_url": hour_head_url, "profile_id": p_id, "profile_mid": p_mid, "cover_url": hour_cover_url, "video_url": hour_video_url, "session": f"xiaoniangao-hour-{int(time.time())}" } return video_info_dict # 更新小时榜数据 @classmethod def update_videoList(cls, log_type, crawler, rule_dict, our_uid, env): """ 更新小时榜数据 """ befor_yesterday = (datetime.date.today() + datetime.timedelta(days=-3)).strftime("%Y-%m-%d %H:%M:%S") update_time_stamp = int(time.mktime(time.strptime(befor_yesterday, "%Y-%m-%d %H:%M:%S"))) select_sql = f""" select * from crawler_xiaoniangao_hour where crawler_time_stamp>={update_time_stamp} GROUP BY out_video_id DESC """ update_video_list = MysqlHelper.get_values(log_type, crawler, select_sql, env) if len(update_video_list) == 0: Common.logger(log_type, crawler).info("暂无需要更新的小时榜数据\n") Common.logging(log_type, crawler, env, "暂无需要更新的小时榜数据\n") return for update_video_info in update_video_list: try: profile_id = update_video_info["profile_id"] profile_mid = update_video_info["profile_mid"] video_title = update_video_info["video_title"] video_id = update_video_info["out_video_id"] if datetime.datetime.now().hour == 10 and datetime.datetime.now().minute <= 10: video_info_dict = cls.get_video_info(log_type=log_type, crawler=crawler, p_id=profile_id, p_mid=profile_mid, v_title=video_title, v_id=video_id) ten_play_cnt = video_info_dict['play_cnt'] Common.logger(log_type, crawler).info(f"ten_play_cnt:{ten_play_cnt}") Common.logging(log_type, crawler, env, f"ten_play_cnt:{ten_play_cnt}") update_sql = f""" update crawler_xiaoniangao_hour set ten_play_cnt={ten_play_cnt} WHERE out_video_id="{video_id}"; """ # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}") MysqlHelper.update_values(log_type, crawler, update_sql, env) cls.download_publish(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, update_video_info=update_video_info, our_uid=our_uid, env=env) elif datetime.datetime.now().hour == 15 and datetime.datetime.now().minute <= 10: video_info_dict = cls.get_video_info(log_type=log_type, crawler=crawler, p_id=profile_id, p_mid=profile_mid, v_title=video_title, v_id=video_id) fifteen_play_cnt = video_info_dict['play_cnt'] Common.logger(log_type, crawler).info(f"fifteen_play_cnt:{fifteen_play_cnt}") Common.logging(log_type, crawler, env, f"fifteen_play_cnt:{fifteen_play_cnt}") update_sql = f""" update crawler_xiaoniangao_hour set fifteen_play_cnt={fifteen_play_cnt} WHERE out_video_id="{video_id}"; """ # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}") MysqlHelper.update_values(log_type, crawler, update_sql, env) cls.download_publish(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, update_video_info=update_video_info, our_uid=our_uid, env=env) elif datetime.datetime.now().hour == 20 and datetime.datetime.now().minute <= 10: video_info_dict = cls.get_video_info(log_type=log_type, crawler=crawler, p_id=profile_id, p_mid=profile_mid, v_title=video_title, v_id=video_id) twenty_play_cnt = video_info_dict['play_cnt'] Common.logger(log_type, crawler).info(f"twenty_play_cnt:{twenty_play_cnt}") Common.logging(log_type, crawler, env, f"twenty_play_cnt:{twenty_play_cnt}") update_sql = f""" update crawler_xiaoniangao_hour set twenty_play_cnt={twenty_play_cnt} WHERE out_video_id="{video_id}"; """ # Common.logger(log_type, crawler).info(f"update_sql:{update_sql}") MysqlHelper.update_values(log_type, crawler, update_sql, env) cls.download_publish(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, update_video_info=update_video_info, our_uid=our_uid, env=env) else: pass except Exception as e: Common.logger(log_type, crawler).error(f'更新{update_video_info["video_title"]}时异常:{e}\n') Common.logging(log_type, crawler, env, f'更新{update_video_info["video_title"]}时异常:{e}\n') @classmethod def send_to_mq(cls, log_type, crawler, video_info_dict, rule_dict, env): video_info_dict["out_user_id"] = video_info_dict["profile_id"] video_info_dict["platform"] = crawler video_info_dict["strategy"] = log_type video_info_dict["out_video_id"] = video_info_dict["video_id"] video_info_dict["width"] = video_info_dict["video_width"] video_info_dict["height"] = video_info_dict["video_height"] video_info_dict["crawler_rule"] = json.dumps(rule_dict) video_info_dict["user_id"] = video_info_dict["uid"] video_info_dict["publish_time"] = video_info_dict["publish_time_str"] video_info_dict["strategy_type"] = "hour" mq = MQ(topic_name="topic_crawler_etl_" + env) mq.send_msg(video_info_dict) @classmethod def download(cls, log_type, crawler, video_info_dict, rule_dict, our_uid, env): # 下载视频 Common.download_method(log_type=log_type, crawler=crawler, text="video", title=video_info_dict["video_title"], url=video_info_dict["video_url"]) md_title = md5(video_info_dict['video_title'].encode('utf8')).hexdigest() try: if os.path.getsize(f"./{crawler}/videos/{md_title}/video.mp4") == 0: # 删除视频文件夹 shutil.rmtree(f"./{crawler}/videos/{md_title}") Common.logger(log_type, crawler).info("视频size=0，删除成功\n") Common.logging(log_type, crawler, env, "视频size=0，删除成功\n") return except FileNotFoundError: # 删除视频文件夹 shutil.rmtree(f"./{crawler}/videos/{md_title}") Common.logger(log_type, crawler).info("视频文件不存在，删除文件夹成功\n") Common.logging(log_type, crawler, env, "视频文件不存在，删除文件夹成功\n") return # 下载封面 Common.download_method(log_type=log_type, crawler=crawler, text="cover", title=video_info_dict["video_title"], url=video_info_dict["cover_url"]) # 保存视频信息至 "./videos/{download_video_title}/info.txt" Common.save_video_info(log_type=log_type, crawler=crawler, video_dict=video_info_dict) # 上传视频 Common.logger(log_type, crawler).info("开始上传视频...") Common.logging(log_type, crawler, env, "开始上传视频...") if env == "dev": oss_endpoint = "out" our_video_id = Publish.upload_and_publish(log_type=log_type, crawler=crawler, strategy="上升榜抓取策略", our_uid=our_uid, env=env, oss_endpoint=oss_endpoint) our_video_link = f"https://testadmin.piaoquantv.com/cms/post-detail/{our_video_id}/info" else: oss_endpoint = "inner" our_video_id = Publish.upload_and_publish(log_type=log_type, crawler=crawler, strategy="上升榜抓取策略", our_uid=our_uid, env=env, oss_endpoint=oss_endpoint) our_video_link = f"https://admin.piaoquantv.com/cms/post-detail/{our_video_id}/info" if our_video_id is None: try: # 删除视频文件夹 shutil.rmtree(f"./{crawler}/videos/{md_title}") return except FileNotFoundError: return insert_sql = f""" insert into crawler_video(video_id, out_user_id, platform, strategy, out_video_id, video_title, cover_url, video_url, duration, publish_time, play_cnt, crawler_rule, width, height) values({our_video_id}, "{video_info_dict['profile_id']}", "{cls.platform}", "上升榜抓取策略", "{video_info_dict['video_id']}", "{video_info_dict['video_title']}", "{video_info_dict['cover_url']}", "{video_info_dict['video_url']}", {int(video_info_dict['duration'])}, "{video_info_dict['publish_time_str']}", {int(video_info_dict['play_cnt'])}, '{json.dumps(rule_dict)}', {int(video_info_dict['video_width'])}, {int(video_info_dict['video_height'])}) """ Common.logger(log_type, crawler).info(f"insert_sql:{insert_sql}") Common.logging(log_type, crawler, env, f"insert_sql:{insert_sql}") MysqlHelper.update_values(log_type, crawler, insert_sql, env) Common.logger(log_type, crawler).info('视频信息插入数据库成功！') Common.logging(log_type, crawler, env, '视频信息插入数据库成功！') # 视频写入飞书 Feishu.insert_columns(log_type, crawler, "yatRv2", "ROWS", 1, 2) # 视频ID工作表，首行写入数据 upload_time = int(time.time()) values = [[time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(upload_time)), "上升榜抓取策略", str(video_info_dict['video_id']), str(video_info_dict['video_title']), our_video_link, video_info_dict['play_cnt'], video_info_dict['comment_cnt'], video_info_dict['like_cnt'], video_info_dict['share_cnt'], video_info_dict['duration'], f"{video_info_dict['video_width']}*{video_info_dict['video_height']}", str(video_info_dict['publish_time_str'].replace("-", "/")), str(video_info_dict['user_name']), str(video_info_dict['profile_id']), str(video_info_dict['profile_mid']), str(video_info_dict['avatar_url']), str(video_info_dict['cover_url']), str(video_info_dict['video_url'])]] time.sleep(1) Feishu.update_values(log_type, crawler, "yatRv2", "F2:Z2", values) Common.logger(log_type, crawler).info('视频信息写入飞书成功\n') Common.logging(log_type, crawler, env, '视频信息写入飞书成功\n') # 下载/上传 @classmethod def download_publish(cls, log_type, crawler, video_info_dict, rule_dict, update_video_info, our_uid, env): if cls.repeat_video(log_type, crawler, video_info_dict["video_id"], env) != 0: Common.logger(log_type, crawler).info('视频已下载\n') Common.logging(log_type, crawler, env, '视频已下载\n') # 播放量大于 50000，直接下载 elif int(video_info_dict["play_cnt"]) >= 30000: Common.logger(log_type, crawler).info(f"播放量:{video_info_dict['play_cnt']} >= 30000，满足下载规则，开始下载视频") Common.logging(log_type, crawler, env, f"播放量:{video_info_dict['play_cnt']} >= 30000，满足下载规则，开始下载视频") # cls.download(log_type=log_type, # crawler=crawler, # video_info_dict=video_info_dict, # rule_dict=rule_dict, # our_uid=our_uid, # env=env) cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env) # 上升榜判断逻辑，任意时间段上升量>=5000，连续两个时间段上升量>=2000 elif int(update_video_info['ten_play_cnt']) >= 3000 or int( update_video_info['fifteen_play_cnt']) >= 3000 or int(update_video_info['twenty_play_cnt']) >= 3000: Common.logger(log_type, crawler).info(f"10:00 or 15:00 or 20:00 数据上升量:{int(update_video_info['ten_play_cnt'])} or {int(update_video_info['fifteen_play_cnt'])} or {int(update_video_info['twenty_play_cnt'])} >= 3000") Common.logging(log_type, crawler, env, f"10:00 or 15:00 or 20:00 数据上升量:{int(update_video_info['ten_play_cnt'])} or {int(update_video_info['fifteen_play_cnt'])} or {int(update_video_info['twenty_play_cnt'])} >= 3000") Common.logger(log_type, crawler).info("满足下载规则，开始下载视频") Common.logging(log_type, crawler, env, "满足下载规则，开始下载视频") # cls.download(log_type=log_type, # crawler=crawler, # video_info_dict=video_info_dict, # rule_dict=rule_dict, # our_uid=our_uid, # env=env) cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env) elif int(update_video_info['ten_play_cnt']) >= 1000 and int(update_video_info['fifteen_play_cnt']) >= 1000: Common.logger(log_type, crawler).info(f"10:00 and 15:00 数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['fifteen_play_cnt'])} >= 1000") Common.logging(log_type, crawler, env, f"10:00 and 15:00 数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['fifteen_play_cnt'])} >= 1000") Common.logger(log_type, crawler).info("满足下载规则，开始下载视频") Common.logging(log_type, crawler, env, "满足下载规则，开始下载视频") # cls.download(log_type=log_type, # crawler=crawler, # video_info_dict=video_info_dict, # rule_dict=rule_dict, # our_uid=our_uid, # env=env) cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env) elif int(update_video_info['fifteen_play_cnt']) >= 1000 and int(update_video_info['twenty_play_cnt']) >= 1000: Common.logger(log_type, crawler).info( f"15:00 and 20:00 数据上升量:{int(update_video_info['fifteen_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 1000") Common.logging(log_type, crawler, env, f"15:00 and 20:00 数据上升量:{int(update_video_info['fifteen_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 1000") Common.logger(log_type, crawler).info("满足下载规则，开始下载视频") Common.logging(log_type, crawler, env, "满足下载规则，开始下载视频") # cls.download(log_type=log_type, # crawler=crawler, # video_info_dict=video_info_dict, # rule_dict=rule_dict, # our_uid=our_uid, # env=env) cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env) elif int(update_video_info['ten_play_cnt']) >= 1000 and int(update_video_info['twenty_play_cnt']) >= 1000: Common.logger(log_type, crawler).info( f"今日10:00 / 20:00数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 1000") Common.logging(log_type, crawler, env, f"今日10:00 / 20:00数据上升量:{int(update_video_info['ten_play_cnt'])} and {int(update_video_info['twenty_play_cnt'])} >= 1000") Common.logger(log_type, crawler).info("满足下载规则，开始下载视频") Common.logging(log_type, crawler, env, "满足下载规则，开始下载视频") # cls.download(log_type=log_type, # crawler=crawler, # video_info_dict=video_info_dict, # rule_dict=rule_dict, # our_uid=our_uid, # env=env) cls.send_to_mq(log_type=log_type, crawler=crawler, video_info_dict=video_info_dict, rule_dict=rule_dict, env=env) else: Common.logger(log_type, crawler).info("上升量不满足下载规则") Common.logging(log_type, crawler, env, "上升量不满足下载规则") if __name__ == "__main__": print(get_config_from_mysql(log_type='hour', source='xiaoniangao', env='dev', text='filter')) pass