crawler_haokan/main/common.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/11/23
"""
公共方法，包含：生成log / 删除log / 下载方法 / 读取文件 / 统计下载数
"""
from datetime import date, timedelta
from loguru import logger
import datetime
import os
import time
import requests
import urllib3


class Common:
    # 统一获取当前时间 <class 'datetime.datetime'>  2022-04-14 20:13:51.244472
    now = datetime.datetime.now()
    # 昨天 <class 'str'>  2022-04-13
    yesterday = (date.today() + timedelta(days=-1)).strftime("%Y/%m/%d")
    # 今天 <class 'datetime.date'>  2022-04-14
    today = date.today()
    # 明天 <class 'str'>  2022-04-15
    tomorrow = (date.today() + timedelta(days=1)).strftime("%Y/%m/%d")

    # 使用 logger 模块生成日志
    @staticmethod
    def logger(log_type):
        """
        使用 logger 模块生成日志
        """
        # 日志路径
        log_dir = r"./logs/"
        log_path = os.getcwd() + os.sep + log_dir
        if not os.path.isdir(log_path):
            os.makedirs(log_path)

        # 日志文件名
        log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '-haokan-'+str(log_type)+'.log'

        # 日志不打印到控制台
        logger.remove(handler_id=None)

        # rotation="500 MB"，实现每 500MB 存储一个文件
        # rotation="12:00"，实现每天 12:00 创建一个文件
        # rotation="1 week"，每周创建一个文件
        # retention="10 days"，每隔10天之后就会清理旧的日志
        # 初始化日志
        logger.add(log_dir + log_name, level="INFO", rotation='00:00')

        return logger

    # 清除日志，保留最近 10 个文件
    @classmethod
    def del_logs(cls, log_type):
        """
        清除冗余日志文件
        :return: 保留最近 10 个日志
        """
        log_dir = "./logs/"
        all_files = sorted(os.listdir(log_dir))
        all_logs = []
        for log in all_files:
            name = os.path.splitext(log)[-1]
            if name == ".log":
                all_logs.append(log)

        if len(all_logs) <= 10:
            pass
        else:
            for file in all_logs[:len(all_logs) - 10]:
                os.remove(log_dir + file)
        cls.logger(log_type).info("清除日志成功")

    # 删除 charles 缓存文件，只保留最近的两个文件
    @classmethod
    def del_charles_files(cls, log_type):
        # 目标文件夹下所有文件
        all_file = sorted(os.listdir("./chlsfiles/"))
        for file in all_file[0:-3]:
            os.remove(r"./chlsfiles/" + file)
        cls.logger(log_type).info("删除 charles 缓存文件成功")

    # 封装下载视频或封面的方法
    @classmethod
    def download_method(cls, log_type, text, d_name, d_url):
        """
        下载封面：text == "cover" ； 下载视频：text == "video"
        需要下载的视频标题：d_title
        视频封面，或视频播放地址：d_url
        下载保存路径："./files/{d_title}/"
        """
        videos_dir = r"./videos/"
        if not os.path.exists(videos_dir):
            os.mkdir(videos_dir)
        # 首先创建一个保存该视频相关信息的文件夹
        video_dir = "./videos/" + str(d_name) + "/"
        if not os.path.exists(video_dir):
            os.mkdir(video_dir)

        # 下载视频
        if text == "video":
            # 需要下载的视频地址
            video_url = str(d_url).replace('http://', 'https://')
            # 视频名
            video_name = "video.mp4"

            # 下载视频
            urllib3.disable_warnings()
            response = requests.get(video_url, stream=True, proxies=cls.tunnel_proxies(), verify=False)
            try:
                with open(video_dir + video_name, "wb") as f:
                    for chunk in response.iter_content(chunk_size=10240):
                        f.write(chunk)
                cls.logger(log_type).info("==========视频下载完成==========")
            except Exception as e:
                cls.logger(log_type).error(f"视频下载失败：{e}\n")

        # 下载音频
        elif text == "audio":
            # 需要下载的视频地址
            audio_url = str(d_url).replace('http://', 'https://')
            # 音频名
            audio_name = "audio.mp4"

            # 下载视频
            urllib3.disable_warnings()
            response = requests.get(audio_url, stream=True, proxies=cls.tunnel_proxies(), verify=False)
            try:
                with open(video_dir + audio_name, "wb") as f:
                    for chunk in response.iter_content(chunk_size=10240):
                        f.write(chunk)
                cls.logger(log_type).info("==========音频下载完成==========")
            except Exception as e:
                cls.logger(log_type).error(f"音频下载失败：{e}\n")

        # 下载封面
        elif text == "cover":
            # 需要下载的封面地址
            cover_url = str(d_url)
            # 封面名
            cover_name = "image.jpg"

            # 下载封面
            urllib3.disable_warnings()
            response = requests.get(cover_url, proxies=cls.tunnel_proxies(), verify=False)
            try:
                with open(video_dir + cover_name, "wb") as f:
                    f.write(response.content)
                cls.logger(log_type).info("==========封面下载完成==========")
            except Exception as e:
                cls.logger(log_type).error(f"封面下载失败：{e}\n")

    # 快代理
    @classmethod
    def tunnel_proxies(cls):
        # 隧道域名:端口号
        tunnel = "q796.kdltps.com:15818"

        # 用户名密码方式
        username = "t17772369458618"
        password = "5zqcjkmy"
        proxies = {
            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
        }

        # 白名单方式（需提前设置白名单）
        # proxies = {
        #     "http": "http://%(proxy)s/" % {"proxy": tunnel},
        #     "https": "http://%(proxy)s/" % {"proxy": tunnel}
        # }

        # 要访问的目标网页
        # target_url = "https://www.kuaishou.com/profile/3xk9tkk6kkwkf7g"

        # # 使用隧道域名发送请求
        # response = requests.get(target_url, proxies=proxies)
        # print(response.status_code)
        return proxies


if __name__ == "__main__":
    common = Common()