automatic_crawler/application/etl/download.py

"""
下载视频
"""
import os
import json
import time
import asyncio
from hashlib import md5
import datetime

import httpx
import requests


class VideoDownloader(object):
    """
    视频下载功能
    """

    def __init__(self, video_obj):
        self.platform = video_obj['platform']
        self.video_id = video_obj['video_id']
        self.video_url = video_obj['video_url']
        self.cover_url = video_obj['cover_url']
        self.proxy = {
            "http://": "http://t17772369458618:5zqcjkmy@q796.kdltps.com:15818/",
            "https://": "http://t17772369458618:5zqcjkmy@q796.kdltps.com:15818/"
        }
        self.max_retry = 5

    def generate_video_path(self):
        """
        通过视频信息生成唯一视频地址
        :return:
        """
        index = "{}-{}".format(self.platform, self.video_id)
        index = md5(index.encode()).hexdigest()
        temp_dir = "/Users/luojunhui/cyber/automatic_crawler"
        file_name = "{}.mp4".format(index)
        date_info = datetime.datetime.today().strftime("%Y%m%d")
        video_path = os.path.join(temp_dir, date_info, file_name)
        if os.path.exists(video_path):
            return
        else:
            os.makedirs(os.path.dirname(video_path), exist_ok=True)
        return video_path

    async def download_video(self):
        """
        download video from the web
        :return:
        """
        if self.platform == "fuqiwang":
            download_path = self.generate_video_path()
            if download_path:
                headers = {
                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.6(0x13080610) XWEB/1156',
                    'Accept-Encoding': 'identity;q=1, *;q=0',
                    'Accept': '*/*',
                    'Sec-Fetch-Site': 'cross-site',
                    'Sec-Fetch-Mode': 'no-cors',
                    'Sec-Fetch-Dest': 'video',
                    'Referer': 'https://servicewechat.com/wxa1431c6e7acdd32d/2/page-frame.html',
                    'Accept-Language': 'en-US,en;q=0.9',
                    'Range': 'bytes=0-',
                }
                async with httpx.AsyncClient(http2=True, proxies=self.proxy, headers=headers) as client:
                    try:
                        response = await client.get(self.video_url, headers=headers)
                        if response.status_code == 206:
                            with open(download_path, "wb") as f:
                                f.write(response.content)
                        else:
                            for _ in range(self.max_retry):
                                response = await client.get(self.video_url, headers=headers, follow_redirects=True)
                                if response.status_code == 206:
                                    with open(download_path, "wb") as f:
                                        f.write(response.content)
                                    break
                    except httpx.HTTPError as e:
                        print(f"An error occurred while downloading: {e}")
            else:
                print("视频已经存在")

    def get_by_request(self):
        """
        req
        :return:
        """
        download_path = self.generate_video_path()
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.6(0x13080610) XWEB/1156',
            'Accept-Encoding': 'identity;q=1, *;q=0',
            'Accept': '*/*',
            'Sec-Fetch-Site': 'cross-site',
            'Sec-Fetch-Mode': 'no-cors',
            'Sec-Fetch-Dest': 'video',
            'Referer': 'https://servicewechat.com/wxa1431c6e7acdd32d/2/page-frame.html',
            'Accept-Language': 'en-US,en;q=0.9',
            'Range': 'bytes=0-',
        }
        r = requests.get(
            url=self.video_url,
            headers=headers,
            proxies=self.proxy
        )
        print(r.status_code)
        with open("test.mp4", "wb") as f:
            f.write(r.content)


async def main(video_obj):
    """
    异步执行函数
    :param video_obj:
    :return:
    """
    downloader = VideoDownloader(video_obj)
    await downloader.download_video()


if __name__ == '__main__':
    video_o = {
        "update_time": 1709784300,
        "platform": "fuqiwang",
        "video_id": 142599,
        "title": "🔴3·8妇女节，最美的祝福，送给全天下的女神！",
        "type": 1,
        "video_type": 2,
        "cover_url": "https://znl-video-bos.cdn.bcebos.com/c6f12b49992ef638342065439f55b444/65e93632/picture/20240306/b8b0c1cc262c2394f111650c9f82e35a_thumb.jpg",
        "video_url": "https://znl-video-bos.cdn.bcebos.com/e368801a814c548e443835086d37caaf/65e93632/video/20240306/820ee1498e3ed2a59d37aed54d39ae95_1.mp4",
    }
    VideoDownloader(video_obj=video_o).get_by_request()
    # asyncio.run(main(video_obj=video_o))