crawler_weiqun_video/main/weiqun_music.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/8/11
import os
import random
# import shutil
import sys
import time
import ffmpeg
import requests
import urllib3
sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu
from main.publish import Publish


class Music:

    # 获取已下载视频宽高、时长等信息
    @classmethod
    def get_video_info_from_local(cls, video_path):
        probe = ffmpeg.probe(video_path)
        # print('video_path: {}'.format(video_path))
        # format1 = probe['format']
        # bit_rate = int(format1['bit_rate']) / 1000
        # duration = format['duration']
        # size = int(format1['size']) / 1024 / 1024
        video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
        if video_stream is None:
            print('No video stream found!')
            return
        width = int(video_stream['width'])
        height = int(video_stream['height'])
        # num_frames = int(video_stream['nb_frames'])
        # fps = int(video_stream['r_frame_rate'].split('/')[0]) / int(video_stream['r_frame_rate'].split('/')[1])
        duration = float(video_stream['duration'])
        # print('width: {}'.format(width))
        # print('height: {}'.format(height))
        # print('num_frames: {}'.format(num_frames))
        # print('bit_rate: {}k'.format(bit_rate))
        # print('fps: {}'.format(fps))
        # print('size: {}MB'.format(size))
        # print('duration: {}'.format(duration))
        return width, height, duration

    # 获取视频ID
    @classmethod
    def get_video_list(cls, log_type):
        try:
            # 翻10页
            for num in range(1, 5):
                Common.logger(log_type).info("正在抓取第{}页\n", num)
                url = "https://ayg.818ao.com/app/index.php?"
                headers = {
                    "content-type": "application/x-www-form-urlencoded",
                    "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) "
                                  "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 "
                                  "MicroMessenger/8.0.26(0x18001a2e) NetType/WIFI Language/zh_CN",
                    "Accept-Encoding": "gzip,compress,br,deflate",
                    "Referer": "https://servicewechat.com/wx06417203b6ba3e25/1/page-frame.html"
                }
                query_string = {
                    "i": "13",
                    "t": "0",
                    "m": "jyt_txvideo",
                    "v": "1.4",
                    "from": "wxapp",
                    "c": "entry",
                    "a": "wxapp",
                    "do": "videolist",
                    "sign": "ffccddd4e767cf98e39222026b938015",
                }
                form = {
                    "category": "246",
                    "page": num,
                    "israndom": "",
                    "type": "0",
                    "isview": "",
                    "noauth": "true"
                }
                urllib3.disable_warnings()
                r = requests.post(url=url, headers=headers, params=query_string, data=form, verify=False)
                if "data" not in r.json() or len(r.json()["data"]) == 0:
                    Common.logger(log_type).warning("get_video_list:response:{}", r.text)
                else:
                    data = r.json()["data"]
                    for i in range(len(data)):
                        if "vid" in data[i]:
                            video_id = data[i]["vid"]
                            cls.get_video_info(log_type, video_id)
                        else:
                            Common.logger(log_type).info("get_video_list获取到无效视频，略过")
        except Exception as e:
            Common.logger(log_type).error("get_video_list异常:{}", e)

    # 根据 video_id 获取 video_url
    @classmethod
    def get_video_url(cls, logtype, vid):
        try:
            url = "https://ayg.818ao.com/app/index.php?"
            headers = {
                "content-type": "application/x-www-form-urlencoded",
                "Accept-Encoding": "gzip,compress,br,deflate",
                "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) "
                              "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 "
                              "MicroMessenger/8.0.26(0x18001a2e) NetType/WIFI Language/zh_CN",
                "Referer": "https://servicewechat.com/wx06417203b6ba3e25/1/page-frame.html"
            }
            query_string = {
                "i": "13",
                "t": "0",
                "m": "jyt_txvideo",
                "v": "1.4",
                "from": "wxapp",
                "c": "entry",
                "a": "wxapp",
                "do": "videoinfo",
                "state": "we7sid-da63afcc7fbfdeb15d94c9683f83dc1f",
                "sign": "a449ac59c215b8cab2fbcbb6c3a609e2",
                "vid": vid,
                "version": "1.0.3",
            }
            urllib3.disable_warnings()
            r = requests.get(url=url, headers=headers, params=query_string, verify=False)
            if "data" not in r.json():
                Common.logger(logtype).warning("get_video_url:response:{}", r.json())
            else:
                video_url = r.json()["data"]["res"]
                return video_url
        except Exception as e:
            Common.logger(logtype).error("get_video_url异常:{}", e)

    # 获取视频详情
    @classmethod
    def get_video_info(cls, log_type, vid):
        try:
            url = "https://ayg.818ao.com/app/index.php?"
            headers = {
                "content-type": "application/x-www-form-urlencoded",
                "user-agent": "User-Agent	Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) "
                              "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.26(0x18001a2e)"
                              " NetType/WIFI Language/zh_CN",
                "Accept-Encoding": "gzip,compress,br,deflate",
                "referer": "https://servicewechat.com/wx06417203b6ba3e25/1/page-frame.html"
            }
            query_string = {
                "i": "13",
                "t": "0",
                "m": "jyt_txvideo",
                "v": "1.4",
                "from": "wxapp",
                "c": "entry",
                "a": "wxapp",
                "do": "videodetail",
                "state": "we7sid-da63afcc7fbfdeb15d94c9683f83dc1f",
                "sign": "9b5c05a00fc03363daa1a48622b03bf3",
            }
            form = {
                "vid": vid,
                "cate": "246",
                "userad": ""
            }
            urllib3.disable_warnings()
            r = requests.post(url=url, headers=headers, params=query_string, data=form, verify=False)
            if r.json()["message"] != "视频详情":
                Common.logger(log_type).warning("get_video_info:response:{}", r.text)
            else:

                # video_title
                if "vtitle" in r.json()["data"]:
                    video_title = r.json()["data"]["vtitle"]
                else:
                    video_title = 0

                # video_id
                video_id = vid

                # play_cnt
                if "realview" not in r.json()["data"]:
                    play_cnt = 0
                else:
                    play_cnt = r.json()["data"]["realview"]

                # like_cnt
                like_cnt = 0

                # share_cnt
                share_cnt = 0

                # comment_cnt
                comment_cnt = 0

                # send_time
                if "create_time" in r.json()["data"]:
                    send_time = r.json()["data"]["create_time"]
                else:
                    send_time = 0

                # video_duration
                if "vtime" not in r.json()["data"]:
                    video_duration = 0
                elif r.json()["data"]["vtime"] == "" or r.json()["data"]["vtime"] is None:
                    video_duration = 0
                else:
                    video_duration = r.json()["data"]["vtime"]

                # user_id
                if "uid" not in r.json()["data"]:
                    user_id = 0
                else:
                    user_id = r.json()["data"]["uid"]

                # user_name
                user_name = "微群视频"

                # cover_url
                if "poster" not in r.json()["data"]:
                    cover_url = 0
                elif "http" not in r.json()["data"]["poster"]:
                    cover_url = "http://qiniu.818ao.com/" + r.json()["data"]["poster"]
                else:
                    cover_url = r.json()["data"]["poster"]

                # head_url
                head_url = cover_url

                # video_url
                if "vid" not in r.json()["data"]:
                    video_url = 0
                elif r.json()["data"]["vid"] == vid:
                    video_url = cls.get_video_url(log_type, vid)
                else:
                    video_url = r.json()["data"]["vid"]

                Common.logger(log_type).info("video_title:{}", video_title)
                Common.logger(log_type).info("video_id:{}", video_id)
                # Common.logger(log_type).info("play_cnt:{}", play_cnt)
                # Common.logger(log_type).info("like_cnt:{}", like_cnt)
                # Common.logger(log_type).info("share_cnt:{}", share_cnt)
                # Common.logger(log_type).info("comment_cnt:{}", comment_cnt)
                Common.logger(log_type).info("send_time:{}",
                                             time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(send_time))))
                Common.logger(log_type).info("video_duration:{}", video_duration)
                # Common.logger(log_type).info("user_name:{}", user_name)
                # Common.logger(log_type).info("user_id:{}", user_id)
                # Common.logger(log_type).info("head_url:{}", head_url)
                # Common.logger(log_type).info("cover_url:{}", cover_url)
                Common.logger(log_type).info("video_url:{}", video_url)

                # 过滤无效视频
                if video_title == 0 or video_id == 0 or send_time == 0 or cover_url == 0 or video_url == 0:
                    Common.logger(log_type).info("无效视频\n")
                # 已下载视频表去重
                elif str(video_id) in [n for m in Feishu.get_values_batch(log_type, "weiqun", "3cd128") for n in m]:
                    Common.logger(log_type).info("该视频已下载\n")
                # music_feeds去重
                elif str(video_id) in [n for m in Feishu.get_values_batch(log_type, "weiqun", "JK6npf") for n in m]:
                    Common.logger(log_type).info("该视频已在music_feeds中\n")
                else:
                    time.sleep(1)
                    Feishu.insert_columns(log_type, "weiqun", "JK6npf", "ROWS", 1, 2)
                    get_feeds_time = int(time.time())
                    values = [[str(time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(get_feeds_time))),
                               "音乐榜",
                               str(video_id),
                               video_title,
                               int(play_cnt),
                               comment_cnt,
                               like_cnt,
                               share_cnt,
                               int(video_duration),
                               time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(send_time))),
                               user_name,
                               user_id,
                               head_url,
                               cover_url,
                               video_url]]
                    time.sleep(1)
                    Feishu.update_values(log_type, "weiqun", "JK6npf", "A2:Q2", values)
                    Common.logger(log_type).info("添加至music_feeds成功\n")
        except Exception as e:
            Common.logger(log_type).error("get_video_info异常:{}", e)

    # 下载/上传视频
    @classmethod
    def download_publish(cls, log_type, env):
        try:
            music_feeds_sheet = Feishu.get_values_batch(log_type, "weiqun", "JK6npf")
            for i in range(1, len(music_feeds_sheet)):
                download_video_id = music_feeds_sheet[i][2]
                download_video_title = music_feeds_sheet[i][3]
                download_video_play_cnt = music_feeds_sheet[i][4]
                download_video_comment_cnt = music_feeds_sheet[i][5]
                download_video_like_cnt = music_feeds_sheet[i][6]
                download_video_share_cnt = music_feeds_sheet[i][7]
                # download_video_duration = music_feeds_sheet[i][8]
                download_video_send_time = music_feeds_sheet[i][9]
                download_user_name = music_feeds_sheet[i][10]
                download_user_id = music_feeds_sheet[i][11]
                download_head_url = music_feeds_sheet[i][12]
                download_cover_url = music_feeds_sheet[i][13]
                download_video_url = music_feeds_sheet[i][14]

                Common.logger(log_type).info("正在判断第{}行", i + 1)
                Common.logger(log_type).info("download_video_title:{}", download_video_title)
                # Common.logger(log_type).info("download_video_play_cnt:{}", download_video_play_cnt)
                # Common.logger(log_type).info("download_video_id:{}", download_video_id)
                # Common.logger(log_type).info("download_video_comment_cnt:{}", download_video_comment_cnt)
                # Common.logger(log_type).info("download_video_like_cnt:{}", download_video_like_cnt)
                # Common.logger(log_type).info("download_video_share_cnt:{}", download_video_share_cnt)
                Common.logger(log_type).info("download_video_send_time:{}", download_video_send_time)
                # Common.logger(log_type).info("download_user_name:{}", download_user_name)
                # Common.logger(log_type).info("download_user_id:{}", download_user_id)
                # Common.logger(log_type).info("download_head_url:{}", download_head_url)
                # Common.logger(log_type).info("download_cover_url:{}", download_cover_url)
                Common.logger(log_type).info("download_video_url:{}", download_video_url)

                # 过滤空行
                if download_video_id is None or download_video_title is None or download_video_play_cnt is None:
                    Common.logger(log_type).warning("空行，略过\n")
                # 已下载视频表去重
                elif str(download_video_id) in [n for m in Feishu.get_values_batch(log_type, "weiqun", "3cd128")
                                                for n in m]:
                    Feishu.dimension_range(log_type, "weiqun", "JK6npf", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("该视频已下载，删除成功\n")
                    return
                # 满足下载规则
                else:
                    # 下载视频
                    Common.download_method(log_type=log_type, text="video",
                                           d_name=str(download_video_title), d_url=str(download_video_url))
                    # 下载封面
                    Common.download_method(log_type=log_type, text="cover",
                                           d_name=str(download_video_title), d_url=str(download_cover_url))
                    # 获取视频宽高/时长
                    video_info = cls.get_video_info_from_local("./videos/" + download_video_title + "/video.mp4")
                    download_video_resolution = str(video_info[0]) + "*" + str(video_info[1])
                    download_video_duration = video_info[2]

                    # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                    with open("./videos/" + download_video_title
                              + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
                        f_a.write(str(download_video_id) + "\n" +
                                  str(download_video_title) + "\n" +
                                  str(int(download_video_duration)) + "\n" +
                                  str(download_video_play_cnt) + "\n" +
                                  str(download_video_comment_cnt) + "\n" +
                                  str(download_video_like_cnt) + "\n" +
                                  str(download_video_share_cnt) + "\n" +
                                  str(download_video_resolution) + "\n" +
                                  str(int(time.mktime(
                                      time.strptime(download_video_send_time, "%Y/%m/%d %H:%M:%S")))) + "\n" +
                                  str(download_user_name) + "\n" +
                                  str(download_head_url) + "\n" +
                                  str(download_video_url) + "\n" +
                                  str(download_cover_url) + "\n" +
                                  "weiqunvideo")
                    Common.logger(log_type).info("==========视频信息已保存至info.txt==========")

                    # 上传视频
                    Common.logger(log_type).info("开始上传视频:{}".format(download_video_title))
                    our_video_id = Publish.upload_and_publish(log_type, env, "play")
                    our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
                    Common.logger(log_type).info("视频上传完成:{}", download_video_title)

                    # 保存视频 ID 到云文档
                    Common.logger(log_type).info("保存视频ID至已下载表:{}", download_video_title)
                    # 视频ID工作表，插入首行
                    Feishu.insert_columns(log_type, "weiqun", "3cd128", "ROWS", 1, 2)
                    # 视频ID工作表，首行写入数据
                    upload_time = int(time.time())
                    values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
                               "音乐榜",
                               str(download_video_id),
                               str(download_video_title),
                               our_video_link,
                               download_video_play_cnt,
                               download_video_comment_cnt,
                               download_video_like_cnt,
                               download_video_share_cnt,
                               int(download_video_duration),
                               str(download_video_resolution),
                               str(download_video_send_time),
                               str(download_user_name),
                               str(download_user_id),
                               str(download_head_url),
                               str(download_cover_url),
                               str(download_video_url)]]
                    time.sleep(1)
                    Feishu.update_values(log_type, "weiqun", "3cd128", "F2:W2", values)

                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range(log_type, "weiqun", "JK6npf", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("视频:{}，下载/上传成功\n", download_video_title)
                    return

        except Exception as e:
            Feishu.dimension_range(log_type, "weiqun", "JK6npf", "ROWS", 2, 2)
            Common.logger(log_type).error("download_publish异常:{}，删除成功", e)

    # 执行下载/上传
    @classmethod
    def run_download_publish(cls, log_type, env):
        try:
            while True:
                time.sleep(1)
                music_feeds_sheet = Feishu.get_values_batch(log_type, "weiqun", "JK6npf")
                if len(music_feeds_sheet) == 1:
                    Common.logger(log_type).info("下载/上传完成\n")
                    break
                else:
                    cls.download_publish(log_type, env)
                    time.sleep(random.randint(1, 3))
        except Exception as e:
            Common.logger(log_type).error("run_download_publish异常:{}", e)


if __name__ == "__main__":
    # Music.get_video_list("weiqun")
    # Music.get_video_info("weiqun", "wxv_2228210032582639621")
    # print(Music.get_video_url("weiqun", "w3243xgp9i1"))
    Music.download_publish("weiqun", "dev")