crawler_kuaishou/main/download_weishi.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/4/8
import json
import time

import requests
import urllib3
from main.common import Common
from main.publish import Publish

proxies = {"http": None, "https": None}


class Weishi:
    @staticmethod
    def weishi_download_rule(d_duration, d_width, d_height, d_play_cnt):
        """
        下载视频的基本规则
        :param d_duration: 时长
        :param d_width: 宽
        :param d_height: 高
        :param d_play_cnt: 播放量
        :return: 满足规则，返回 True；反之，返回 False
        """
        if 600 >= int(float(d_duration)) >= 60:
            if int(d_width) >= 720 or int(d_height) >= 720:
                if int(d_play_cnt) >= 100000:
                    return True
                else:
                    return False
            return False
        return False

    @classmethod
    def get_weishi_recommend(cls):
        """
        从微视小程序首页推荐获取视频list:
            1.在 weishi_videoid.txt 中去重
            2.在 weishi_feeds.txt 中去重
            3.添加视频信息到 weishi_feeds.txt
        """
        url = "https://api.weishi.qq.com/trpc.weishi.weishi_h5_proxy.weishi_h5_proxy/WxminiGetFeedList"
        cookies = {
            "wesee_authtype": "3",
            "wesee_openid": "oWGa05FrwkuUvT-4n1qGeQuhVsc8",
            "wesee_openkey": "8c3ec202f5d679fb5ee6d9f643640d9a2580ba504612e2d979a881d3169caf189e2a5c1d532eeff172bc21cf2"
                             "6230941ccbc10243a7879e8165ca608c17060de606a6d08afe0a3abd5250629314f9a99e9d1003b201bf5ec",
            "wesee_personid": "1593522421826902",
            "wesee_refresh_token": "",
            "wesee_access_token": "8c3ec202f5d679fb5ee6d9f643640d9a2580ba504612e2d979a881d3169caf18"
                                  "9e2a5c1d532eeff172bc21cf26230941ccbc10243a7879e8165ca608c17060de6"
                                  "06a6d08afe0a3abd5250629314f9a99e9d1003b201bf5ec",
            "wesee_thr_appid": "wx75ee9f19b93e5c46",
            "wesee_ichid": "8"
        }
        json_data = {
            "req_body": {
                "requestType": 16,
                "isrefresh": 0,
                "isfirst": 0,
                "attachInfo": "",
                "scene_id": 22,
                "requestExt": {
                    "mini_openid": "oWGa05FrwkuUvT-4n1qGeQuhVsc8",
                    "notLogin-personid": "1593522421826902"
                }
            },
            "req_header": {
                "mapExt": "{\"imageSize\":\"480\",\"adaptScene\":\"PicHDWebpLimitScene\"}"
            }
        }

        try:
            urllib3.disable_warnings()
            r = requests.post(url=url, cookies=cookies, json=json_data, proxies=proxies, verify=False)
            response = json.loads(r.content.decode("utf8"))
            if "rsp_body" not in response:
                Common.crawler_log().info("获取微视视频 list 出错:{}，休眠 10s".format(response))
                time.sleep(10)
            else:
                feeds = response["rsp_body"]["feeds"]
                for i in range(len(feeds)):
                    if "video" not in feeds[i]:
                        Common.crawler_log().info("无视频信息")
                    else:
                        # 视频 ID
                        if "id" not in feeds[i]["video"]:
                            video_id = "0"
                            Common.crawler_log().info("video_id:{}".format(video_id))
                        else:
                            video_id = feeds[i]["video"]["id"]
                            Common.crawler_log().info("video_id:{}".format(video_id))

                        # 视频标题
                        video_title = feeds[i]["desc"].strip().replace("\n", "") \
                            .replace("/", "").replace("快手", "").replace(" ", "") \
                            .replace(" ", "").replace("&NBSP", "").replace("\r", "")
                        Common.crawler_log().info("video_title:{}".format(video_title))

                        # 视频发布时间
                        if "createTime" not in feeds[i]:
                            video_send_time = "0"
                            Common.crawler_log().info("video_send_time:不存在")
                        else:
                            video_send_time = int(feeds[i]["createTime"])*1000
                            Common.crawler_log().info(
                                "video_send_time:{}".format(time.strftime(
                                    "%Y-%m-%d %H:%M:%S", time.localtime(int(video_send_time)/1000))))

                        # 视频封面地址
                        if len(feeds[i]["images"]) == 0:
                            cover_url = "0"
                            Common.crawler_log().info("cover_url:不存在")
                        else:
                            cover_url = feeds[i]["images"][0]["url"]
                            Common.crawler_log().info("cover_url:{}".format(cover_url))

                        # 视频播放地址
                        if "url" not in feeds[i]["video"]:
                            video_url = "0"
                            Common.crawler_log().info("video_url:不存在")
                        else:
                            video_url = feeds[i]["video"]["url"]
                            Common.crawler_log().info("video_url:{}".format(video_url))

                        # 视频分辨率
                        if "width" not in feeds[i]["video"] or "height" not in feeds[i]["video"]:
                            video_width = "0"
                            video_height = "0"
                            video_resolution = str(video_width) + "*" + str(video_height)
                            Common.crawler_log().info("无分辨率")
                        else:
                            video_width = feeds[i]["video"]["width"]
                            video_height = feeds[i]["video"]["height"]
                            video_resolution = str(video_width) + "*" + str(video_height)
                            Common.crawler_log().info("video_resolution:{}".format(video_resolution))

                        # 视频时长
                        if "duration" not in feeds[i]["video"]:
                            video_duration = "0"
                            Common.crawler_log().info("video_duration:不存在")
                        else:
                            video_duration = int(int(feeds[i]["video"]["duration"]) / 1000)
                            Common.crawler_log().info("video_duration:{}秒".format(video_duration))

                        # 播放数
                        if "playNum" not in feeds[i]["ugcData"]:
                            video_play_cnt = "0"
                            Common.crawler_log().info("video_play_cnt:{}".format(video_play_cnt))
                        else:
                            video_play_cnt = feeds[i]["ugcData"]["playNum"]
                            Common.crawler_log().info("video_play_cnt:{}".format(video_play_cnt))

                        # 点赞数
                        if "dingCount" not in feeds[i]["ugcData"]:
                            video_like_cnt = "0"
                            Common.crawler_log().info("video_like_cnt:{}".format(video_like_cnt))
                        else:
                            video_like_cnt = feeds[i]["ugcData"]["dingCount"]
                            Common.crawler_log().info("video_like_cnt:{}".format(video_like_cnt))

                        # 分享数
                        if "shareNum" not in feeds[i]["ugcData"]:
                            video_share_cnt = "0"
                            Common.crawler_log().info("video_share_cnt:{}".format(video_share_cnt))
                        else:
                            video_share_cnt = feeds[i]["ugcData"]["shareNum"]
                            Common.crawler_log().info("video_share_cnt:{}".format(video_share_cnt))

                        # 评论数
                        if "totalCommentNum" not in feeds[i]["ugcData"]:
                            video_comment_cnt = "0"
                            Common.crawler_log().info("video_comment_cnt:{}".format(video_comment_cnt))
                        else:
                            video_comment_cnt = feeds[i]["ugcData"]["totalCommentNum"]
                            Common.crawler_log().info("video_comment_cnt:{}".format(video_comment_cnt))

                        # 用户 ID
                        user_id = feeds[i]["poster"]["id"]
                        Common.crawler_log().info("user_id:{}".format(user_id))

                        # 用户昵称
                        user_name = feeds[i]["poster"]["nick"].strip().replace("\n", "") \
                            .replace("/", "").replace("快手", "").replace(" ", "") \
                            .replace(" ", "").replace("&NBSP", "").replace("\r", "")
                        Common.crawler_log().info("user_name:{}".format(user_name))

                        # 用户头像地址
                        if "thumbURL" not in feeds[i]["material"] and "avatar" not in feeds[i]["poster"]:
                            head_url = "0"
                            Common.crawler_log().info("head_url:不存在")
                        elif "thumbURL" in feeds[i]["material"]:
                            head_url = feeds[i]["material"]["thumbURL"]
                            Common.crawler_log().info("head_url:{}".format(head_url))
                        else:
                            head_url = feeds[i]["poster"]["avatar"]
                            Common.crawler_log().info("head_url:{}".format(head_url))

                        # 从 weishi_videoid.txt 中去重
                        videos_ids = Common.read_txt("weishi_videoid.txt")
                        if video_id in [v_id.strip() for v_id in videos_ids]:
                            Common.crawler_log().info("该视频已下载:{}".format(video_title))
                            pass
                        else:
                            Common.crawler_log().info("该视频未下载:{}".format(video_title))

                            # 从 weishi_feeds.txt 中去重
                            contents = Common.read_txt("weishi_feeds.txt")
                            # 若 weishi_feeds.txt 为空时，直接保存
                            if len(contents) == 0 and head_url != "0" \
                                    and cover_url != "0" and video_url != "0" \
                                    and video_duration != "0" and video_id != "0":
                                basic_time = int(time.time())
                                Common.crawler_log().info("添加视频信息至weishi_feeds.txt:{}".format(video_title))
                                with open(r"./txt/weishi_feeds.txt", "a", encoding="UTF-8") as f_a:
                                    f_a.write(str(basic_time) + " + " +
                                              str(video_id) + " + " +
                                              str(video_play_cnt) + " + " +
                                              str(video_title) + " + " +
                                              str(video_duration) + " + " +
                                              str(video_comment_cnt) + " + " +
                                              str(video_like_cnt) + " + " +
                                              str(video_share_cnt) + " + " +
                                              str(video_resolution) + " + " +
                                              str(video_send_time) + " + " +
                                              str(user_name) + " + " +
                                              str(head_url) + " + " +
                                              str(cover_url) + " + " +
                                              str(video_url) + " + " +
                                              str(user_id) + " + " +
                                              str("oWGa05FrwkuUvT-4n1qGeQuhVsc8") + "\n")
                            else:
                                if video_id in [content.split(" + ")[1] for content in contents]:
                                    Common.crawler_log().info("该视频已在 weishi_feeds.txt 中:{}".format(video_title))
                                elif head_url == "0" or cover_url == "0" \
                                        or video_url == "0" or video_duration == "0" or video_id == "0":
                                    Common.crawler_log().info("视频封面/播放地址/播放时长/用户头像不存在")
                                else:
                                    basic_time = int(time.time())
                                    Common.crawler_log().info("添加视频信息至weishi_feeds.txt:{}".format(video_title))
                                    with open(r"./txt/weishi_feeds.txt", "a", encoding="UTF-8") as f_a:
                                        f_a.write(str(basic_time) + " + " +
                                                  str(video_id) + " + " +
                                                  str(video_play_cnt) + " + " +
                                                  str(video_title) + " + " +
                                                  str(video_duration) + " + " +
                                                  str(video_comment_cnt) + " + " +
                                                  str(video_like_cnt) + " + " +
                                                  str(video_share_cnt) + " + " +
                                                  str(video_resolution) + " + " +
                                                  str(video_send_time) + " + " +
                                                  str(user_name) + " + " +
                                                  str(head_url) + " + " +
                                                  str(cover_url) + " + " +
                                                  str(video_url) + " + " +
                                                  str(user_id) + " + " +
                                                  str("oWGa05FrwkuUvT-4n1qGeQuhVsc8") + "\n")
        except Exception as e:
            Common.crawler_log().error("获取微视视频 list 异常:{}".format(e))

    @classmethod
    def download_weishi_play_video(cls, env):
        """
        下载播放量视频
        测试环境:env == dev
        正式环境:env == prod
        """
        videos = Common.read_txt("weishi_feeds.txt")
        for video in videos:
            download_video_id = video.strip().split(" + ")[1]
            download_video_title = video.strip().split(" + ")[3]
            download_video_duration = video.strip().split(" + ")[4]
            download_video_play_cnt = video.strip().split(" + ")[2]
            download_video_comment_cnt = video.strip().split(" + ")[5]
            download_video_like_cnt = video.strip().split(" + ")[6]
            download_video_share_cnt = video.strip().split(" + ")[7]
            download_video_resolution = video.strip().split(" + ")[8]
            download_video_width = download_video_resolution.split("*")[0]
            download_video_height = download_video_resolution.split("*")[-1]
            download_video_send_time = video.strip().split(" + ")[9]
            download_user_name = video.strip().split(" + ")[10]
            download_head_url = video.strip().split(" + ")[11]
            download_cover_url = video.strip().split(" + ")[12]
            download_video_url = video.strip().split(" + ")[13]
            download_video_session = video.strip().split(" + ")[-1]

            if cls.weishi_download_rule(download_video_duration, download_video_width,
                                        download_video_height, download_video_play_cnt) is True:
                Common.crawler_log().info("开始下载视频:{}".format(download_video_title))
                # 下载封面
                Common.download_method(text="cover", d_name=download_video_title, d_url=download_cover_url)
                # 下载视频
                Common.download_method(text="video", d_name=download_video_title, d_url=download_video_url)
                # 保存视频信息至 weishi_videoid.txt
                with open("./txt/weishi_videoid.txt", "a", encoding="UTF-8") as fa:
                    fa.write(download_video_id + "\n")
                # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                with open("./videos/" + download_video_title + "/info.txt", "a", encoding="UTF-8") as f_a:
                    f_a.write(str(download_video_id) + "\n" +
                              str(download_video_title) + "\n" +
                              str(download_video_duration) + "\n" +
                              str(download_video_play_cnt) + "\n" +
                              str(download_video_comment_cnt) + "\n" +
                              str(download_video_like_cnt) + "\n" +
                              str(download_video_share_cnt) + "\n" +
                              str(download_video_resolution) + "\n" +
                              str(download_video_send_time) + "\n" +
                              str(download_user_name) + "\n" +
                              str(download_head_url) + "\n" +
                              str(download_video_url) + "\n" +
                              str(download_cover_url) + "\n" +
                              str(download_video_session))

                # 上传视频
                if env == "dev":
                    Common.crawler_log().info("开始上传视频:{}".format(download_video_title))
                    Publish.upload_and_publish("dev", "play")
                elif env == "prod":
                    Common.crawler_log().info("开始上传视频:{}".format(download_video_title))
                    Publish.upload_and_publish("prod", "play")

                # 删除该视频在weishi_feeds.txt中的信息
                Common.crawler_log().info("删除该视频在weishi_feeds.txt中的信息:{}".format(download_video_title))
                with open(r"./txt/weishi_feeds.txt", "r", encoding="UTF-8") as f_r:
                    lines = f_r.readlines()
                with open(r"./txt/weishi_feeds.txt", "w", encoding="utf-8") as f_w:
                    for line in lines:
                        if download_video_id in line.split(" + ")[1]:
                            continue
                        f_w.write(line)
            else:
                # 删除该视频在weishi_feeds.txt中的信息
                Common.crawler_log().info("该视频不满足下载规则，删除在weishi_feeds.txt中的信息:{}".format(download_video_title))
                with open(r"./txt/weishi_feeds.txt", "r", encoding="UTF-8") as f_r:
                    lines = f_r.readlines()
                with open(r"./txt/weishi_feeds.txt", "w", encoding="utf-8") as f_w:
                    for line in lines:
                        if download_video_id in line.split(" + ")[1]:
                            continue
                        f_w.write(line)


if __name__ == "__main__":
    weishi = Weishi()
    weishi.get_weishi_recommend()