crawler_shipinhao/main/download_publish.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/9/1
import os
import random
import sys
import time
sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu
from main.publish import Publish


class Download:
    @classmethod
    def download_publish(cls, log_type, env):
        try:
            recommend_sheet = Feishu.get_values_batch(log_type, 'shipinhao', 'FSDlBy')
            for i in range(1, len(recommend_sheet)):
                download_title = recommend_sheet[i][2].strip().replace('"', '') \
                    .replace('“', '').replace('“', '…').replace("\n", "") \
                    .replace("/", "").replace("\r", "").replace("#", "") \
                    .replace(".", "。").replace("\\", "").replace("&NBSP", "") \
                    .replace(":", "").replace("*", "").replace("？", "") \
                    .replace("?", "").replace('"', "").replace("<", "") \
                    .replace(">", "").replace("|", "").replace(" ", "")
                download_duration = recommend_sheet[i][3]
                download_like_cnt = recommend_sheet[i][4]
                download_share_cnt = recommend_sheet[i][5]
                download_favorite_cnt = recommend_sheet[i][6]
                download_comment_cnt = recommend_sheet[i][7]
                download_username = recommend_sheet[i][8]
                download_head_url = recommend_sheet[i][9]
                download_cover_url = recommend_sheet[i][10]
                download_video_url = recommend_sheet[i][11]

                Common.logger(log_type).info("download_title:{}", download_title)
                Common.logger(log_type).info("download_username:{}", download_username)
                Common.logger(log_type).info("download_video_url:{}", download_video_url)

                # 判断空行
                if download_video_url is None or download_title is None:
                    Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', i + 1, i + 1)
                    Common.logger(log_type).info("空行，删除成功\n")
                    return
                # 已下载判断
                elif str(download_title) in [x for y in Feishu.get_values_batch(log_type, 'shipinhao', 'c77cf9') for x in y]:
                    Feishu.dimension_range(log_type, 'shipinhao', 'FSDlBy', 'ROWS', i + 1, i + 1)
                    Common.logger(log_type).info("视频已下载\n")
                    return
                else:
                    # 下载封面
                    Common.download_method(log_type=log_type, text="cover",
                                           d_name=str(download_title), d_url=str(download_cover_url))
                    # 下载视频
                    Common.download_method(log_type=log_type, text="video",
                                           d_name=str(download_title), d_url=str(download_video_url))

                    # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                    with open("./videos/" + download_title
                              + "/" + "info.txt", "a", encoding="UTF-8") as f_a:
                        f_a.write('shipinhao' + str(int(time.time())) + "\n" +
                                  str(download_title) + "\n" +
                                  str(download_duration) + "\n" +
                                  str(download_favorite_cnt) + "\n" +
                                  str(download_comment_cnt) + "\n" +
                                  str(download_like_cnt) + "\n" +
                                  str(download_share_cnt) + "\n" +
                                  str(1920*1080) + "\n" +
                                  str(int(time.time())) + "\n" +
                                  str(download_username) + "\n" +
                                  str(download_head_url) + "\n" +
                                  str(download_video_url) + "\n" +
                                  str(download_cover_url) + "\n" +
                                  "shipinhao")
                    Common.logger(log_type).info("==========视频信息已保存至info.txt==========")

                    # 上传视频
                    Common.logger(log_type).info("开始上传视频:{}".format(download_title))
                    our_video_id = Publish.upload_and_publish(log_type, env, "play")
                    if env == 'dev':
                        our_video_link = "https://testadmin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
                    else:
                        our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
                    Common.logger(log_type).info("视频上传完成:{}", download_title)

                    # 视频ID工作表，插入首行
                    Feishu.insert_columns(log_type, "shipinhao", "c77cf9", "ROWS", 1, 2)
                    # 视频ID工作表，首行写入数据
                    upload_time = int(time.time())
                    values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
                               "推荐榜",
                               str(download_title),
                               our_video_link,
                               download_duration,
                               download_like_cnt,
                               download_share_cnt,
                               download_favorite_cnt,
                               download_comment_cnt,
                               str(download_username),
                               str(download_head_url),
                               str(download_cover_url),
                               str(download_video_url)]]
                    time.sleep(1)
                    Feishu.update_values(log_type, "shipinhao", "c77cf9", "F2:Z2", values)

                    # 删除行或列，可选 ROWS、COLUMNS
                    Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", i + 1, i + 1)
                    Common.logger(log_type).info("视频:{}，下载/上传成功\n", download_title)
                    return
        except Exception as e:
            Common.logger(log_type).error("download_publish异常:{}\n", e)
            Feishu.dimension_range(log_type, "shipinhao", "FSDlBy", "ROWS", 2, 2)

    @classmethod
    def run_download_publish(cls, log_type, env):
        try:
            while True:
                time.sleep(1)
                recommend_sheet = Feishu.get_values_batch(log_type, "shipinhao", "FSDlBy")
                if len(recommend_sheet) == 1:
                    Common.logger(log_type).info("下载/上传完成\n")
                    break
                else:
                    cls.download_publish(log_type, env)
                    time.sleep(random.randint(1, 3))
        except Exception as e:
            Common.logger(log_type).error("run_download_publish异常:{}", e)


if __name__ == '__main__':
    Download.run_download_publish('recommend', 'dev')