crawler_shipinhao/main/demo.py

# @Author: wangkun
# @Time: 2月 25, 2022
import datetime
import logging
import os
import sys
import time
sys.path.append(os.getcwd())
from main.feishu_lib import Feishu
from main.publish import Publish
from main.common import Common
path = "../videoinfo/"


class Demo:

    @classmethod
    def read_videoname(cls):
        with open(path + "videoname.txt", "r", encoding="utf8") as f:
            content = f.read()
            name = content
            return name

    @classmethod
    def write_videoname(cls):
        n = int(float(cls.read_videoname()))
        with open(path + "videoname.txt", "w", encoding="utf8") as f:
            f.write(str(n + 1))

    @classmethod
    def split_text(cls):
        path1 = "./chlsfiles/"
        files = os.listdir(path1)
        for file in files:
            print(file)
            print("\n==========\n")
            a = os.path.splitext(file)
            print(a[-1])

            # print(file[0])
            # print(file[-1])

    @classmethod
    def split_query(cls):
        url = "encfilekey=Cvvj5Ix3eez3Y79SxtvVL0L7CkPM6dFibFeI6caGYwFHPibicjZB7UrwpKsG9wQrl01" \
              "IlTMtIjicibib9iaGoaL2sxR7vGt4sofODkBGnic9EOCZjgCQTlO7ZTNjribxkDmPq40E2RMdXxzGlWDwYfmkWz4Q15g" \
              "&token=AxricY7RBHdVbVKZOkvCOWwYWwj9zBqet1eia6GZib8GhmdrytdBu5HcPARtxW7l0AAzppsekDUXXQ&idx=1&adap" \
              "tivelytrans=943&bizid=1023&dotrans=2991&hy=SH&m=d04052b0ab0e3dd64567fa60b16d0898&scene=0&t=1&taskid=0"
        a = url.split("encfilekey=")[-1].split("&token=")[0]
        print(a)

    @classmethod
    def distent(cls):
        filekey = "Cvvj5Ix3eez3Y79SxtvVL0L7CkPM6dFibFeI6caGYwFEiaKINib2TA0ibV5JuXfIA5JysRWeytU3kTufCr7KFPLgwMiauCic" \
                  "UypoEbA31w7w3sibnraPEwcMcG6LqxSmdFN5S8AfhuibFicv86oBfSuSXcibrCqw"
        filekey_txt_path = "../videoinfo/filekey.txt"

        with open(filekey_txt_path, 'r', encoding="utf8") as f:
            content = f.read()
            print(content)
            for line in content:
                if filekey in line.strip():
                    print("yes")
                elif line.strip() == "":
                    print("null")
                else:
                    print("no")

    # 创建日志
    @classmethod
    def create_log(cls):
        # 创建日志文件夹。如果不存在，则创建
        log_dir = "../logs/"  # 日志文件夹名称
        log_path = os.getcwd() + os.sep + log_dir  # 日志路径
        if not os.path.isdir(log_path):
            os.makedirs(log_path)

        log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
        date_format = "%Y-%m-%d %p %H:%M:%S"
        # 初始化
        logging.basicConfig(filename=log_path + 'logs.logs', level=logging.INFO, format=log_format, datefmt=date_format)

        logger = logging.getLogger('shipinhao')

        logger.info("这是第一条日志")

    @classmethod
    def split_title(cls):
        title = Feishu.get_values_batch('demo', 'shipinhao', 'FSDlBy')[3][2]
        print(title)
        print('\n')
        # print(title.split('\n')[0].split('#')[0])
        print(title.split('\n')[0].replace('#', ''))

    # 下载 、上传
    @classmethod
    def download_publish(cls, log_type, env):
        try:
            recommend_feeds_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')
            for i in range(1, len(recommend_feeds_sheet)):
                download_title = recommend_feeds_sheet[i][2].strip().replace('"', '') \
                    .replace('“', '').replace('“', '…').replace("\n", "") \
                    .replace("/", "").replace("\r", "").replace("#", "") \
                    .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                    .replace(":", "").replace("*", "").replace("？", "") \
                    .replace("?", "").replace('"', "").replace("<", "") \
                    .replace(">", "").replace("|", "").replace(" ", "")
                download_duration = recommend_feeds_sheet[i][3]
                download_like_cnt = recommend_feeds_sheet[i][4]
                download_share_cnt = recommend_feeds_sheet[i][5]
                download_favorite_cnt = recommend_feeds_sheet[i][6]
                download_comment_cnt = recommend_feeds_sheet[i][7]
                download_username = recommend_feeds_sheet[i][8]
                download_head_url = recommend_feeds_sheet[i][9]
                download_cover_url = recommend_feeds_sheet[i][10]
                download_video_url = recommend_feeds_sheet[i][11]

                Common.logger(log_type).info("download_title:{}", download_title)
                Common.logger(log_type).info("download_username:{}", download_username)
                Common.logger(log_type).info("download_video_url:{}", download_video_url)

                if download_title is None or download_duration is None or download_video_url is None:
                    Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', i + 1, i + 1)
                    Common.logger(log_type).info('空行，删除成功\n')
                    return
                elif str(download_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x
                                             in y]:
                    Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', i + 1, i + 1)
                    Common.logger(log_type).info('视频已下载，删除成功\n')
                    return
                else:
                    # 下载封面
                    Common.download_method(log_type=log_type, text="cover",
                                           d_name=str(download_title), d_url=str(download_cover_url))
                    # 下载视频
                    Common.download_method(log_type=log_type, text="video",
                                           d_name=str(download_title), d_url=str(download_video_url))
                    # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                    with open("./videos/" + download_title
                              + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
                        f_a.write('shipinhao' + str(int(time.time())) + "\n" +
                                  str(download_title) + "\n" +
                                  str(download_duration) + "\n" +
                                  str(download_favorite_cnt) + "\n" +
                                  str(download_comment_cnt) + "\n" +
                                  str(download_like_cnt) + "\n" +
                                  str(download_share_cnt) + "\n" +
                                  str(1920 * 1080) + "\n" +
                                  str(int(time.time())) + "\n" +
                                  str(download_username) + "\n" +
                                  str(download_head_url) + "\n" +
                                  str(download_video_url) + "\n" +
                                  str(download_cover_url) + "\n" +
                                  "shipinhao"+str(int(time.time())))
                    Common.logger(log_type).info("==========视频信息已保存至info.txt==========")

                    Common.logger(log_type).info("开始上传视频:{}".format(download_title))
                    our_video_id = Publish.upload_and_publish(log_type, env, "play")
                    if env == 'dev':
                        our_video_link = "https://testadmin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
                    else:
                        our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
                    Common.logger(log_type).info("视频上传完成:{}", our_video_link)

                    # # 视频ID工作表，插入首行
                    # Feishu.insert_columns(log_type, "shipinhao", "c77cf9", "ROWS", 1, 2)
                    # # 视频ID工作表，首行写入数据
                    # upload_time = int(time.time())
                    # values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
                    #            "话题榜",
                    #            str(download_title),
                    #            our_video_link,
                    #            download_duration,
                    #            download_like_cnt,
                    #            download_share_cnt,
                    #            download_favorite_cnt,
                    #            download_comment_cnt,
                    #            download_username,
                    #            str(download_head_url),
                    #            str(download_cover_url),
                    #            str(download_video_url)]]
                    # time.sleep(1)
                    # Feishu.update_values(log_type, "shipinhao", "c77cf9", "F2:V2", values)

                    # 删除行或列，可选 ROWS、COLUMNS
                    time.sleep(1)
                    Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("下载/上传成功:{}\n", download_title)
                    return
        except Exception as e:
            Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", 2, 2)
            Common.logger(log_type).error('download_publish异常，删除视频信息成功:{}\n', e)

    @classmethod
    def run_download_publish(cls, log_type, env):
        while True:
            feeds_sht = Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')
            if len(feeds_sht) == 1:
                Common.logger(log_type).info('下载/上传完成\n')
                break
            else:
                cls.download_publish(log_type, env)

    @classmethod
    def get_sht(cls):
        sht = Feishu.get_values_batch('recommend', 'shipinhao', 'c77cf9')
        print(len(sht))


if __name__ == "__main__":
    # print(str(datetime.datetime.now()).split('.')[0])
    # print(time.time())
    # Demo.run_download_publish('topic', 'prod')
    Demo.get_sht()
    pass