# -*- coding: utf-8 -*- # @Author: wangkun # @Time: 2022/4/18 """ 获取看一看+小程序，首页推荐视频列表 """ import json import os import random import sys import time import requests import urllib3 sys.path.append(os.getcwd()) from main.common import Common proxies = {"http": None, "https": None} def get_feeds(): """ 获取视频信息后： 1.先在 video.txt 中去重 2.再从 basic.txt 中去重 3.添加视频信息至 basic.txt """ host = "https://search.weixin.qq.com" url = '/cgi-bin/recwxa/recwxavideolist?' get_video_list_session = Common.get_session() Common.crawler_log().info("获取视频list时，session：{}".format(get_video_list_session)) header = { "Connection": "keep-alive", "content-type": "application/json", "Accept-Encoding": "gzip,compress,br,deflate", "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) " "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.18(0x18001236) " "NetType/WIFI Language/zh_CN", "Referer": "https://servicewechat.com/wxbb9a805eb4f9533c/234/page-frame.html", } params = { 'session': get_video_list_session, "offset": 0, "wxaVersion": "3.9.2", "count": "10", "channelid": "208", "scene": '310', "subscene": '1089', "clientVersion": '8.0.18', "sharesearchid": '0', "nettype": 'wifi', "switchprofile": "0", "switchnewuser": "0", } try: urllib3.disable_warnings() r = requests.get(host + url, headers=header, params=params, proxies=proxies, verify=False) response = json.loads(r.content.decode("utf8")) if "data" not in response: Common.crawler_log().info("获取视频list时，session过期，随机睡眠 31-50 秒") # 如果返回空信息，则随机睡眠 50-100 秒 time.sleep(random.randint(31, 50)) get_feeds() elif "items" not in response["data"]: Common.crawler_log().info("获取视频list时，返回空信息，随机睡眠 1-3 分钟") # 如果返回空信息，则随机睡眠 1-3 分钟 time.sleep(random.randint(60, 180)) get_feeds() else: items = response["data"]["items"] for i in range(len(items)): # 如果该视频没有视频信息，则忽略 if "videoInfo" not in items[i]: Common.crawler_log().info("无视频信息") else: # 获取视频ID video_id = items[i]["videoId"] Common.crawler_log().info('视频ID：{}'.format(video_id)) # 获取视频标题 video_title = items[i]["title"].strip().replace("\n", "")\ .replace("/", "").replace("\\", "").replace("\r", "")\ .replace(":", "").replace("*", "").replace("？", "")\ .replace("?", "").replace('"', "").replace("<", "")\ .replace(">", "").replace("|", "") Common.crawler_log().info('视频标题：{}'.format(video_title)) # 获取视频播放次数 video_play_cnt = items[i]["playCount"] Common.crawler_log().info('视频播放次数：{}'.format(video_play_cnt)) # 获取视频点赞数 video_liked_cnt = items[i]["liked_cnt"] Common.crawler_log().info('视频点赞数：{}'.format(video_liked_cnt)) # 获取视频时长 video_duration = items[i]["mediaDuration"] Common.crawler_log().info('视频时长：{}秒'.format(video_duration)) # 获取视频评论数 video_comment_cnt = items[i]["comment_cnt"] Common.crawler_log().info('视频评论数：{}'.format(video_comment_cnt)) # 获取视频分享数 video_shared_cnt = items[i]["shared_cnt"] Common.crawler_log().info('视频分享数：{}'.format(video_shared_cnt)) # 获取视频发布时间 video_send_date = items[i]["date"] Common.crawler_log().info('视频发布时间：{}'.format( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(video_send_date)))) # 获取视频用户名 video_user = items[i]["source"].strip().replace("\n", "") Common.crawler_log().info('视频用户名：{}'.format(video_user)) # 获取视频宽高 if "short_video_info" not in items[i]: video_width = "0" video_height = "0" video_resolution = str(video_width) + "*" + str(video_height) Common.crawler_log().info("无分辨率：{}".format(video_resolution)) elif len(items[i]["short_video_info"]) == 0: video_width = "0" video_height = "0" video_resolution = str(video_width) + "*" + str(video_height) Common.crawler_log().info("无分辨率：{}".format(video_resolution)) else: # 视频宽 video_width = items[i]["short_video_info"]["width"] # 视频高 video_height = items[i]["short_video_info"]["height"] video_resolution = str(video_width) + "*" + str(video_height) Common.crawler_log().info('视频宽高：{}'.format(video_resolution)) # 获取视频用户头像 video_user_cover = items[i]["bizIcon"] Common.crawler_log().info('视频用户头像：{}'.format(video_user_cover)) # 获取视频封面 if "smartCoverUrl" in items[i]: video_cover = items[i]["smartCoverUrl"] Common.crawler_log().info('视频封面：{}'.format(video_cover)) else: video_cover = items[i]["thumbUrl"] Common.crawler_log().info('视频封面：{}'.format(video_cover)) # 获取播放地址 if "mpInfo" in items[i]["videoInfo"]["videoCdnInfo"].keys(): if len(items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"]) > 2: url = items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"][2]["url"] Common.crawler_log().info('视频播放地址：{}'.format(url)) else: url = items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"][0]["url"] Common.crawler_log().info('视频播放地址：{}'.format(url)) elif "ctnInfo" in items[i]["videoInfo"]["videoCdnInfo"]: url = items[i]["videoInfo"]["videoCdnInfo"]["ctnInfo"]["urlInfo"][0]["url"] Common.crawler_log().info('视频播放地址：{}'.format(url)) else: url = items[i]["videoInfo"]["videoCdnInfo"]["urlInfo"][0]["url"] Common.crawler_log().info('视频播放地址：{}'.format(url)) # 过滤无效视频 if video_id == "" \ or video_send_date == "" \ or video_title.strip() == "" \ or video_play_cnt == "" \ or video_liked_cnt == "" \ or video_duration == "" \ or video_comment_cnt == "" \ or video_shared_cnt == "" \ or video_user == "" \ or video_user_cover == "" \ or video_cover == "" \ or url == "": Common.crawler_log().info("无效视频") else: # 从 kanyikan_videoid.txt 去重 videoids = Common.read_txt("kanyikan_videoid.txt") if video_id in [vid.strip() for vid in videoids]: Common.crawler_log().info("该视频已下载：{}".format(video_title)) else: Common.crawler_log().info("该视频未下载：{}".format(video_title)) # 获取当前时间 basic_time = int(time.time()) # 从 kanyikan_feeds.txt 去重 contents = Common.read_txt("kanyikan_feeds.txt") # 文件为空时，直接添加该视频 if len(contents) == 0: Common.crawler_log().info("添加该视频信息至kanyikan_feeds.txt:{}".format(video_title)) # 当前时间、视频 ID、播放量 存储到 kanyikan_feeds.txt with open(r"./txt/kanyikan_feeds.txt", "a", encoding="utf8") as f: f.write(str(basic_time) + " + " + str(video_id) + " + " + str(video_play_cnt) + " + " + str(video_title) + " + " + str(video_duration) + " + " + str(video_comment_cnt) + " + " + str(video_liked_cnt) + " + " + str(video_shared_cnt) + " + " + str(video_resolution) + " + " + str(video_send_date) + " + " + str(video_user) + " + " + str(video_user_cover) + " + " + str(video_cover) + " + " + str(url) + " + " + Common.get_session() + "\n") else: # 文件不为空时，再做去重 if video_id in [content.split(" + ")[1] for content in contents]: Common.crawler_log().info("该视频已在kanyikan_feeds.txt中:{}".format(video_title)) else: Common.crawler_log().info("添加该视频信息至kanyikan_feeds.txt:{}".format(video_title)) # 当前时间、视频 ID、播放量 存储到 kanyikan_feeds.txt with open(r"./txt/kanyikan_feeds.txt", "a", encoding="utf8") as f: f.write(str(basic_time) + " + " + str(video_id) + " + " + str(video_play_cnt) + " + " + str(video_title) + " + " + str(video_duration) + " + " + str(video_comment_cnt) + " + " + str(video_liked_cnt) + " + " + str(video_shared_cnt) + " + " + str(video_resolution) + " + " + str(video_send_date) + " + " + str(video_user) + " + " + str(video_user_cover) + " + " + str(video_cover) + " + " + str(url) + " + " + Common.get_session() + "\n") except Exception as e: Common.crawler_log().error("获取视频 list 时异常：{}".format(e)) if __name__ == "__main__": get_feeds()