crawler_kanyikan/main/get_feeds.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/4/18
"""
获取看一看+小程序，首页推荐视频列表
"""
import json
import os
import random
import sys
import time
import requests
import urllib3

from main.feishu_lib import Feishu

sys.path.append(os.getcwd())
from main.common import Common

proxies = {"http": None, "https": None}


def get_feeds():
    """
    1.从看一看+小程序首页推荐，获取视频列表
    2.先在 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c 中去重
    3.再从 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=Y8N3Vl 中去重
    4.添加视频信息至 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=Y8N3Vl
    """
    host = "https://search.weixin.qq.com"
    url = '/cgi-bin/recwxa/recwxavideolist?'
    video_list_session = Common.get_session()
    Common.crawler_log().info("获取视频list时，session：{}".format(video_list_session))
    header = {
        "Connection": "keep-alive",
        "content-type": "application/json",
        "Accept-Encoding": "gzip,compress,br,deflate",
        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) "
                      "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.18(0x18001236) "
                      "NetType/WIFI Language/zh_CN",
        "Referer": "https://servicewechat.com/wxbb9a805eb4f9533c/234/page-frame.html",
    }
    params = {
        'session': video_list_session,
        "offset": 0,
        "wxaVersion": "3.9.2",
        "count": "10",
        "channelid": "208",
        "scene": '310',
        "subscene": '1089',
        "clientVersion": '8.0.18',
        "sharesearchid": '0',
        "nettype": 'wifi',
        "switchprofile": "0",
        "switchnewuser": "0",
    }
    try:
        urllib3.disable_warnings()
        r = requests.get(host + url, headers=header, params=params, proxies=proxies, verify=False)
        response = json.loads(r.content.decode("utf8"))

        if "data" not in response:
            Common.crawler_log().info("获取视频list时，session过期，随机睡眠 31-50 秒")
            # 如果返回空信息，则随机睡眠 31-40 秒
            time.sleep(random.randint(31, 40))
            get_feeds()
        elif "items" not in response["data"]:
            Common.crawler_log().info("获取视频list时，返回空信息，随机睡眠 1-3 分钟")
            # 如果返回空信息，则随机睡眠 1-3 分钟
            time.sleep(random.randint(60, 180))
            get_feeds()
        else:
            items = response["data"]["items"]
            for i in range(len(items)):
                # 如果该视频没有视频信息，则忽略
                if "videoInfo" not in items[i]:
                    Common.crawler_log().info("无视频信息")
                else:
                    # 获取视频ID
                    video_id = items[i]["videoId"]
                    Common.crawler_log().info('视频ID：{}'.format(video_id))
                
                    # 获取视频标题
                    video_title = items[i]["title"].strip().replace("\n", "")\
                        .replace("/", "").replace("\\", "").replace("\r", "")\
                        .replace(":", "").replace("*", "").replace("？", "")\
                        .replace("?", "").replace('"', "").replace("<", "")\
                        .replace(">", "").replace("|", "").replace(" ", "")\
                        .replace("&NBSP", "").replace(".", "。").replace(" ", "")
                    Common.crawler_log().info('视频标题：{}'.format(video_title))
                
                    # 获取视频播放次数
                    video_play_cnt = items[i]["playCount"]
                    Common.crawler_log().info('视频播放次数：{}'.format(video_play_cnt))
                
                    # 获取视频点赞数
                    video_liked_cnt = items[i]["liked_cnt"]
                    Common.crawler_log().info('视频点赞数：{}'.format(video_liked_cnt))
                
                    # 获取视频时长
                    video_duration = items[i]["mediaDuration"]
                    Common.crawler_log().info('视频时长：{}秒'.format(video_duration))
                
                    # 获取视频评论数
                    video_comment_cnt = items[i]["comment_cnt"]
                    Common.crawler_log().info('视频评论数：{}'.format(video_comment_cnt))
                
                    # 获取视频分享数
                    video_shared_cnt = items[i]["shared_cnt"]
                    Common.crawler_log().info('视频分享数：{}'.format(video_shared_cnt))
                
                    # 获取视频发布时间
                    video_send_date = items[i]["date"]
                    Common.crawler_log().info('视频发布时间：{}'.format(
                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(video_send_date))))
                
                    # 获取视频用户名
                    video_user = items[i]["source"].strip().replace("\n", "")
                    Common.crawler_log().info('视频用户名：{}'.format(video_user))
                
                    # 获取视频宽高
                    if "short_video_info" not in items[i]:
                        video_width = "0"
                        video_height = "0"
                        video_resolution = str(video_width) + "*" + str(video_height)
                        Common.crawler_log().info("无分辨率：{}".format(video_resolution))
                    elif len(items[i]["short_video_info"]) == 0:
                        video_width = "0"
                        video_height = "0"
                        video_resolution = str(video_width) + "*" + str(video_height)
                        Common.crawler_log().info("无分辨率：{}".format(video_resolution))

                    else:
                        # 视频宽
                        video_width = items[i]["short_video_info"]["width"]
                        # 视频高
                        video_height = items[i]["short_video_info"]["height"]
                        video_resolution = str(video_width) + "*" + str(video_height)
                        Common.crawler_log().info('视频宽高：{}'.format(video_resolution))
                
                    # 获取视频用户头像
                    video_user_cover = items[i]["bizIcon"]
                    Common.crawler_log().info('视频用户头像：{}'.format(video_user_cover))
                
                    # 获取视频封面
                    if "smartCoverUrl" in items[i]:
                        video_cover = items[i]["smartCoverUrl"]
                        Common.crawler_log().info('视频封面：{}'.format(video_cover))
                    else:
                        video_cover = items[i]["thumbUrl"]
                        Common.crawler_log().info('视频封面：{}'.format(video_cover))
                
                    # 获取播放地址
                    if "mpInfo" in items[i]["videoInfo"]["videoCdnInfo"].keys():
                        if len(items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"]) > 2:
                            url = items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"][2]["url"]
                            Common.crawler_log().info('视频播放地址：{}'.format(url))
                        else:
                            url = items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"][0]["url"]
                            Common.crawler_log().info('视频播放地址：{}'.format(url))
                    elif "ctnInfo" in items[i]["videoInfo"]["videoCdnInfo"]:
                        url = items[i]["videoInfo"]["videoCdnInfo"]["ctnInfo"]["urlInfo"][0]["url"]
                        Common.crawler_log().info('视频播放地址：{}'.format(url))
                    else:
                        url = items[i]["videoInfo"]["videoCdnInfo"]["urlInfo"][0]["url"]
                        Common.crawler_log().info('视频播放地址：{}'.format(url))

                    # 过滤无效视频
                    if video_id == "" \
                            or video_send_date == "" \
                            or video_title.strip() == "" \
                            or video_play_cnt == "" \
                            or video_liked_cnt == "" \
                            or video_duration == "" \
                            or video_comment_cnt == "" \
                            or video_shared_cnt == "" \
                            or video_user == "" \
                            or video_user_cover == "" \
                            or video_cover == "" \
                            or url == "":
                        Common.crawler_log().info("无效视频")
                    else:
                        # 从 云文档 去重:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
                        if video_id in [j for i in Feishu.get_values_batch("20ce0c") for j in i]:
                            Common.crawler_log().info("该视频已下载：{}".format(video_title))
                        else:
                            Common.crawler_log().info("该视频未下载：{}".format(video_title))

                            # 从 云文档 去重:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=Y8N3Vl
                            if video_id in [j for i in Feishu.get_values_batch("Y8N3Vl") for j in i]:
                                Common.crawler_log().info("该视频已在kanyikan_feeds_1中:{}".format(video_title))
                            else:
                                Common.crawler_log().info("添加该视频信息至kanyikan_feeds_1:{}".format(video_title))

                                # 看一看+工作表，插入首行
                                print(Feishu.insert_columns("Y8N3Vl"))

                                # 获取当前时间
                                get_feeds_time = int(time.time())
                                # 看一看云文档，工作表 kanyikan_feeds_1 中写入数据
                                Feishu.update_values("Y8N3Vl",
                                                     a1=str(get_feeds_time),
                                                     b1=str(video_id),
                                                     c1=str(video_play_cnt),
                                                     d1=str(video_title),
                                                     e1=str(video_duration),
                                                     f1=str(video_comment_cnt),
                                                     g1=str(video_liked_cnt),
                                                     h1=str(video_shared_cnt),
                                                     i1=str(video_resolution),
                                                     j1=str(video_send_date),
                                                     k1=str(video_user),
                                                     l1=str(video_user_cover),
                                                     m1=str(video_cover),
                                                     n1=str(url),
                                                     o1=str(video_list_session))
    except Exception as e:
        Common.crawler_log().error("获取视频 list 时异常：{}".format(e))


if __name__ == "__main__":
    get_feeds()