# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2023/4/13 import json import os import sys import time import uuid import requests import urllib3 from requests.adapters import HTTPAdapter from common.mq import MQ sys.path.append(os.getcwd()) from common.common import Common from common.public import download_rule from common.scheduling_db import MysqlHelper from common.aliyun_log import AliyunLogger class SuisuiniannianyingfuqiRecommendScheduling: platform = "岁岁年年迎福气" @classmethod def repeat_video(cls, log_type, crawler, video_id, env): # sql = f""" select * from crawler_video where platform="岁岁年年迎福气" and out_video_id="{video_id}"; """ sql = f""" select * from crawler_video where platform in ("{crawler}","{cls.platform}") and out_video_id="{video_id}"; """ repeat_video = MysqlHelper.get_values(log_type, crawler, sql, env) return len(repeat_video) @classmethod def get_videoList(cls, log_type, crawler, our_uid, rule_dict, env): mq = MQ(topic_name="topic_crawler_etl_" + env) page = 1 while True: try: # url = 'https://www.jzkksp.com/index/home/get_home_list.html' url = 'https://www.angjukk.cn/index/home/get_home_list.html' headers = { 'content-type': 'application/x-www-form-urlencoded', 'Accept-Encoding': 'gzip,compress,br,deflate', 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) ' 'AppleWebKit/605.1.15 (KHTML, like Gecko) ' 'Mobile/15E148 MicroMessenger/8.0.25(0x1800192b) NetType/WIFI Language/zh_CN', 'Referer': 'https://servicewechat.com/wxd4c54f60812f6f36/1/page-frame.html', } data = { 'token': '851ae159fd33f955bf433e7c47a4a298', 'time': '1667905857000', 'str_data': 'uT551tU8', 'page': str(page), 'limit': '10', 'appid': 'wxd4c54f60812f6f36', 'version': '1.4.1', 'openid': 'oDAjy5SCFe7Ml3PNgiow3ncozL1o' } urllib3.disable_warnings() s = requests.session() # max_retries=3 重试3次 s.mount('http://', HTTPAdapter(max_retries=3)) s.mount('https://', HTTPAdapter(max_retries=3)) response = s.post(url=url, headers=headers, data=data, verify=False, timeout=5) page += 1 if response.status_code != 200: Common.logger(log_type, crawler).warning(f'get_videoList:{response.status_code}, {response.text}\n') Common.logging(log_type, crawler, env, f'get_videoList:{response.status_code}, {response.text}\n') return elif 'data' not in response.json(): Common.logger(log_type, crawler).warning(f'get_videoList:{response.status_code}, {response.json()}\n') Common.logging(log_type, crawler, env, f'get_videoList:{response.status_code}, {response.text}\n') return elif len(response.json()['data']['video_list']['data']) == 0: Common.logger(log_type, crawler).info(f'没有更多数据啦~ {response.json()}\n') Common.logging(log_type, crawler, env, f'没有更多数据啦~ {response.json()}\n') return else: feeds = response.json()['data']['video_list']['data'] for i in range(len(feeds)): try: trace_id = crawler + str(uuid.uuid1()) AliyunLogger.logging( code="1001", platform=crawler, mode=log_type, env=env, data=feeds[i], message="扫描到一条视频" ) publish_time_str = feeds[i].get('createtime', '') publish_time_stamp = int(time.mktime(time.strptime(publish_time_str, "%Y-%m-%d"))) video_dict = {'video_title': feeds[i].get('title', "").replace("'", "").replace('"', ''), 'video_id': str(feeds[i].get('id', '')), 'play_cnt': feeds[i].get('browse', 0), 'comment_cnt': 0, 'like_cnt': 0, 'share_cnt': 0, 'publish_time_stamp': publish_time_stamp, 'publish_time_str': publish_time_str, 'user_name': "岁岁年年迎福气", 'user_id': "suisuiniannianyingfuqi", 'avatar_url': feeds[i].get('thumb', ''), 'cover_url': feeds[i].get('thumb', ''), 'video_url': feeds[i].get('url', ''), 'session': f"suisuiniannianyingfuqi-{int(time.time())}"} for k, v in video_dict.items(): Common.logger(log_type, crawler).info(f"{k}:{v}") Common.logging(log_type, crawler, env, f'video_dict:{video_dict}') if video_dict["video_id"] == '' or video_dict["video_title"] == '' or video_dict["cover_url"] == '' or video_dict["video_url"] == '': Common.logger(log_type, crawler).info('无效视频\n') Common.logging(log_type, crawler, env, '无效视频\n') elif download_rule(log_type=log_type, crawler=crawler, video_dict=video_dict, rule_dict=rule_dict) is False: Common.logger(log_type, crawler).info("不满足抓取规则\n") Common.logging(log_type, crawler, env, "不满足抓取规则\n") elif cls.repeat_video(log_type, crawler, video_dict["video_id"], env) != 0: Common.logger(log_type, crawler).info('视频已下载\n') Common.logging(log_type, crawler, env, '视频已下载\n') AliyunLogger.logging( code="2002", platform=crawler, mode=log_type, message="重复的视频", data=video_dict, trace_id=trace_id, env=env ) else: video_dict["out_user_id"] = video_dict["user_id"] video_dict["platform"] = crawler video_dict["strategy"] = log_type video_dict["out_video_id"] = video_dict["video_id"] video_dict["width"] = 0 video_dict["height"] = 0 video_dict["crawler_rule"] = json.dumps(rule_dict) video_dict["user_id"] = our_uid video_dict["publish_time"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(publish_time_stamp)) mq.send_msg(video_dict) AliyunLogger.logging( code="1002", platform=crawler, mode=log_type, message="成功发送至 ETL", data=video_dict, trace_id=trace_id, env=env ) except Exception as e: Common.logger(log_type, crawler).error(f"抓取单条视频异常:{e}\n") Common.logging(log_type, crawler, env, f"抓取单条视频异常:{e}\n") AliyunLogger.logging( code="3000", platform=crawler, mode=log_type, message=f"抓取单条视频异常:{e}\n", env=env ) except Exception as e: Common.logger(log_type, crawler).error(f"抓取第{page}页时异常:{e}\n") Common.logging(log_type, crawler, env, f"抓取第{page}页时异常:{e}\n") AliyunLogger.logging( code="3000", platform=crawler, mode=log_type, message=f"抓取第{page}页时异常:{e}\n", env=env ) if __name__ == '__main__': SuisuiniannianyingfuqiRecommendScheduling.get_videoList(log_type='recommend', crawler='suisuiniannianyingfuqi', our_uid=6267140, rule_dict={}, env='dev')