已下载视频表:增加站内视频链接；已下载视频信息同步至监控表

README.md

@@ -7,69 +7,129 @@ requests==2.27.1
 urllib3==1.26.9
 
 执行入口：
+
 cd ./crawler-kanyikan-Windows
+
 python3 main/run_XXX.py
 
+
+==========2022/6/29===========
+
+已下载视频表:预留前 5 列备用
+
+已下载视频表增加列:站内视频链接
+
+已下载视频，同步信息至监控表:https://w42nne6hzg.feishu.cn/sheets/shtcnlZWYazInhf7Z60jkbLRJyd?sheet=6fed97
+
+
+
 ==========2022/6/17===========
+
 （recommend_feeds）0-24点
+
 - 7日内播放大于2万
+
 - 或播放大于15万
+
 - 新增:download_play_sendtime.py
+
 - 执行入口:run_download_play_sendtime.py
 
 
+
 ==========2022/6/16===========
+
 - 1.凌晨0点-15点      3日内播放大于2万 爬取
+
 - 2.晚上15点-24点     15万播放爬取
+
 - 视频发布7日内，播放量大于2万
+
 - 视频时长40秒以上
+
 - 分辨率 宽或高大于720  
+
 - 分享量>0
+
 - 站内标题=看一看视频原标题 
+
 - 站内封面图=看一看视频原封面图
 
 
 ==========2022/4/21===========
+
 - 视频发布7日内，播放量大于1万（当前时间 - 发布时间 <= 7 天）
+
 - 任务执行规则：
+
     1.凌晨0点-10点      7日内播放大于1万 爬取
+
     2.早上10点-20点     内容上升榜 爬取
+
     3.晚上20点-24点     15万播放爬取
 
 
+
 ==========2022/4/15===========
+
 - 视频发布3日内，播放量大于2万（当前时间 - 发布时间 <= 3 天）
+
 - 视频时长1分钟以上，10分钟以下
+
 - 分辨率 宽或高大于720
+
 - 分享量>0
+
 - 站内标题=看一看视频原标题
+
 - 站内封面图=看一看视频原封面图
+
 - 任务执行规则：
+
     1.凌晨0点-10点      3日内播放大于2万 爬取
+
     2.早上10点-20点     内容上升榜 爬取
+
     3.晚上20点-24点     15万播放爬取
 
 
+
 ==========2022/3/29===========
+
 1.凌晨5:00 - 21:00，跑上升榜爬虫（循环隔 1 小时，检查播放量>=1000）
+
 2.晚上21:00 - 5:00，跑播放量爬虫（播放量>=200000）
+
 3.视频下载后，立即上传
 
 
 ==========2022/3/24===========
+
 1.周一至周五，跑上升榜爬虫（循环隔 1 小时，检查播放量>=1000）
+
 2.周六至周日，跑播放量爬虫（播放量>=200000）
+
 3.视频下载后，立即上传
+
 4.下周一时，暂停播放量爬虫，恢复上升榜爬虫
 
 
 ==========2022/3/15===========
+
 1.拿到外网视频 list
+
 2.去重
+
 3.获取当前抓取时间、以及播放量、加上基本规则，存储本地：
+
 	3.1 分辨率，宽或者高 >= 720 或 无分辨
+
 	3.2 600 >= 时长 >= 60
+
 4.循环隔 1 小时，检查播放量 >=1000（当前播放量 - 1 小时前的播放量），开始抓取。同时从本地存储中删除，加入到去重文本中。
+
 5.下载总条数，先不限制
+
 6.下载时间：早上 8 点 - 晚上 21 点截止
+
 7.下载完成后立即上传

main/common.py

@@ -29,18 +29,21 @@ class Common:
 
     # 使用 logger 模块生成日志
     @staticmethod
-    def logger():
+    def logger(log_type):
         """
         使用 logger 模块生成日志
         """
         # 日志路径
-        log_dir = r"./logs/"
+        log_dir = "./logs/"
         log_path = os.getcwd() + os.sep + log_dir
         if not os.path.isdir(log_path):
             os.makedirs(log_path)
 
         # 日志文件名
-        log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '.log'
+        if log_type == "recommend":
+            log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '-kanyikan-recommend.log'
+        else:
+            log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '-kanyikan.log'
 
         # 日志不打印到控制台
         logger.remove(handler_id=None)
@@ -54,8 +57,9 @@ class Common:
 
         return logger
 
+    # 清除日志，保留最近 7 个文件
     @classmethod
-    def del_logs(cls):
+    def del_logs(cls, log_type):
         """
         清除冗余日志文件
         :return: 保留最近 7 个日志
@@ -73,24 +77,26 @@ class Common:
         else:
             for file in all_logs[:len(all_logs) - 7]:
                 os.remove(log_dir + file)
-        cls.logger().info("清除冗余日志成功")
+        cls.logger(log_type).info("清除冗余日志成功")
+
+        # 删除 charles 缓存文件，只保留最近的两个文件
 
-    # 删除 charles 缓存文件，只保留最近的两个文件
     @classmethod
-    def del_charles_files(cls):
+    def del_charles_files(cls, log_type):
         # 目标文件夹下所有文件
-        all_file = sorted(os.listdir(r"./chlsfiles/"))
+        all_file = sorted(os.listdir("./chlsfiles/"))
         for file in all_file[0:-2]:
             os.remove(r"./chlsfiles/" + file)
-        cls.logger().info("删除 charles 缓存文件成功")
+        cls.logger(log_type).info("删除 charles 缓存文件成功")
 
+    # 封装下载视频或封面的方法
     @classmethod
-    def download_method(cls, text, d_name, d_url):
+    def download_method(cls, log_type, text, d_name, d_url):
         """
         下载封面：text == "cover" ； 下载视频：text == "video"
         需要下载的视频标题：d_title
         视频封面，或视频播放地址：d_url
-        下载保存路径："./videos/{d_title}/"
+        下载保存路径："./files/{d_title}/"
         """
         # 首先创建一个保存该视频相关信息的文件夹
         video_dir = "./videos/" + d_name + "/"
@@ -111,9 +117,9 @@ class Common:
                 with open(video_dir + video_name, "wb") as f:
                     for chunk in response.iter_content(chunk_size=10240):
                         f.write(chunk)
-                cls.logger().info("==========视频下载完成==========")
+                cls.logger(log_type).info("==========视频下载完成==========")
             except Exception as e:
-                cls.logger().exception("视频下载失败：{}", e)
+                cls.logger(log_type).exception("视频下载失败：{}", e)
 
         # 下载封面
         elif text == "cover":
@@ -128,27 +134,17 @@ class Common:
             try:
                 with open(video_dir + cover_name, "wb") as f:
                     f.write(response.content)
-                cls.logger().info("==========封面下载完成==========")
+                cls.logger(log_type).info("==========封面下载完成==========")
             except Exception as e:
-                cls.logger().exception("封面下载失败：{}", e)
-
-    @staticmethod
-    def read_txt(t_name):
-        """
-        读取 txt 文件
-        :param t_name: 文件名
-        :return: 文件内容
-        """
-        with open(r"./txt/" + t_name, "r", encoding="utf8") as f:
-            return f.readlines()
+                cls.logger(log_type).exception("封面下载失败：{}", e)
 
     @classmethod
-    def get_session(cls):
+    def get_session(cls, log_type):
         # charles 抓包文件保存目录
         charles_file_dir = r"./chlsfiles/"
 
         if int(len(os.listdir(charles_file_dir))) == 1:
-            Common.logger().info("未找到chlsfile文件，等待60s")
+            Common.logger(log_type).info("未找到chlsfile文件，等待60s")
             time.sleep(60)
         else:
             try:
@@ -214,13 +210,13 @@ class Common:
                             else:
                                 return sessions
                 else:
-                    cls.logger().info("未找到 session，10s后重新获取")
+                    cls.logger(log_type).info("未找到 session，10s后重新获取")
                     time.sleep(10)
-                    cls.get_session()
+                    cls.get_session(log_type)
             except Exception as e:
-                cls.logger().exception("获取 session 异常，30s后重试：{}", e)
+                cls.logger(log_type).exception("获取 session 异常，30s后重试：{}", e)
                 time.sleep(30)
-                cls.get_session()
+                cls.get_session(log_type)
 
 
 if __name__ == "__main__":

main/download_play.py

@@ -44,61 +44,65 @@ class DownloadPlay:
         正式环境:env == prod
         """
         try:
-            for i in range(1, len(Feishu.get_values_batch("SdCHOM"))+1):
+            recommend_feeds = Feishu.get_values_batch("recommend", "kanyikan", "SdCHOM")
+            for i in range(1, len(recommend_feeds)+1):
                 time.sleep(1)
                 # download_push_time = Feishu.get_values_batch("SdCHOM")[i][0]
-                download_video_id = Feishu.get_values_batch("SdCHOM")[i][2]
-                download_video_title = Feishu.get_values_batch("SdCHOM")[i][3]
-                download_video_play_cnt = Feishu.get_values_batch("SdCHOM")[i][4]
-                download_video_comment_cnt = Feishu.get_values_batch("SdCHOM")[i][5]
-                download_video_like_cnt = Feishu.get_values_batch("SdCHOM")[i][6]
-                download_video_share_cnt = Feishu.get_values_batch("SdCHOM")[i][7]
-                download_video_duration = Feishu.get_values_batch("SdCHOM")[i][8]
-                download_video_resolution = Feishu.get_values_batch("SdCHOM")[i][9]
+                download_video_id = recommend_feeds[i][2]
+                download_video_title = recommend_feeds[i][3]
+                download_video_play_cnt = recommend_feeds[i][4]
+                download_video_comment_cnt = recommend_feeds[i][5]
+                download_video_like_cnt = recommend_feeds[i][6]
+                download_video_share_cnt = recommend_feeds[i][7]
+                download_video_duration = recommend_feeds[i][8]
+                download_video_resolution = recommend_feeds[i][9]
                 download_video_width = download_video_resolution.split("*")[0]
                 download_video_height = download_video_resolution.split("*")[-1]
-                download_video_send_time = Feishu.get_values_batch("SdCHOM")[i][10]
-                download_user_name = Feishu.get_values_batch("SdCHOM")[i][11]
-                download_user_id = Feishu.get_values_batch("SdCHOM")[i][12]
-                download_head_url = Feishu.get_values_batch("SdCHOM")[i][13][0]["link"]
-                download_cover_url = Feishu.get_values_batch("SdCHOM")[i][14][0]["link"]
-                download_video_url = Feishu.get_values_batch("SdCHOM")[i][15][0]["link"]
+                download_video_send_time = recommend_feeds[i][10]
+                download_user_name = recommend_feeds[i][11]
+                download_user_id = recommend_feeds[i][12]
+                download_head_url = recommend_feeds[i][13]
+                download_cover_url = recommend_feeds[i][14]
+                download_video_url = recommend_feeds[i][15]
 
-                # Common.logger().info("download_video_id:{}", download_video_id)
-                # Common.logger().info("download_video_title:{}", download_video_title)
-                # Common.logger().info("download_video_play_cnt:{}", download_video_play_cnt)
-                # Common.logger().info("download_video_comment_cnt:{}", download_video_comment_cnt)
-                # Common.logger().info("download_video_like_cnt:{}", download_video_like_cnt)
-                # Common.logger().info("download_video_share_cnt:{}", download_video_share_cnt)
-                # Common.logger().info("download_video_duration:{}", download_video_duration)
-                # Common.logger().info("download_video_resolution:{}", download_video_resolution)
-                # Common.logger().info("download_video_send_time:{}", download_video_send_time)
-                # Common.logger().info("download_user_name:{}", download_user_name)
-                # Common.logger().info("download_user_id:{}", download_user_id)
-                # Common.logger().info("download_head_url:{}", download_head_url)
-                # Common.logger().info("download_cover_url:{}", download_cover_url)
-                # Common.logger().info("download_video_url:{}", download_video_url)
+                # Common.logger("recommend").info("download_video_id:{}", download_video_id)
+                # Common.logger("recommend").info("download_video_title:{}", download_video_title)
+                # Common.logger("recommend").info("download_video_play_cnt:{}", download_video_play_cnt)
+                # Common.logger("recommend").info("download_video_comment_cnt:{}", download_video_comment_cnt)
+                # Common.logger("recommend").info("download_video_like_cnt:{}", download_video_like_cnt)
+                # Common.logger("recommend").info("download_video_share_cnt:{}", download_video_share_cnt)
+                # Common.logger("recommend").info("download_video_duration:{}", download_video_duration)
+                # Common.logger("recommend").info("download_video_resolution:{}", download_video_resolution)
+                # Common.logger("recommend").info("download_video_send_time:{}", download_video_send_time)
+                # Common.logger("recommend").info("download_user_name:{}", download_user_name)
+                # Common.logger("recommend").info("download_user_id:{}", download_user_id)
+                # Common.logger("recommend").info("download_head_url:{}", download_head_url)
+                # Common.logger("recommend").info("download_cover_url:{}", download_cover_url)
+                # Common.logger("recommend").info("download_video_url:{}", download_video_url)
 
-                Common.logger().info("正在判断第{}行，视频:{}", i, download_video_title)
+                Common.logger("recommend").info("正在判断第{}行，视频:{}", i, download_video_title)
 
                 if cls.play_rule(
                         play_width=download_video_width, play_height=download_video_height,
                         play_duration=download_video_duration, play_play_cnt=download_video_play_cnt) is False:
-                    Common.logger().info("不满足播放量下载规则，删除该视频信息:{}", download_video_title)
+                    Common.logger("recommend").info("不满足播放量下载规则，删除该视频信息:{}", download_video_title)
                     # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                    Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                    Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                     return
-                elif download_video_id in [j for m in Feishu.get_values_batch("20ce0c") for j in m]:
-                    Common.logger().info("视频已下载，删除该视频信息:{}", download_video_title)
+                elif download_video_id in [j for m in Feishu.get_values_batch(
+                        "recommend", "kanyikan", "20ce0c") for j in m]:
+                    Common.logger("recommend").info("视频已下载，删除该视频信息:{}", download_video_title)
                     # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                    Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                    Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                     return
                 else:
-                    Common.logger().info("开始下载视频:{}", download_video_title)
+                    Common.logger("recommend").info("开始下载视频:{}", download_video_title)
                     # 下载封面
-                    Common.download_method(text="cover", d_name=download_video_title, d_url=download_cover_url)
+                    Common.download_method(log_type="recommend", text="cover",
+                                           d_name=download_video_title, d_url=download_cover_url)
                     # 下载视频
-                    Common.download_method(text="video", d_name=download_video_title, d_url=download_video_url)
+                    Common.download_method(log_type="recommend", text="video",
+                                           d_name=download_video_title, d_url=download_video_url)
                     # 保存视频信息至 "./videos/{download_video_title}/info.txt"
                     with open(r"./videos/" + download_video_title
                               + "/" + "info.txt", "a", encoding="utf8") as f_a:
@@ -116,17 +120,17 @@ class DownloadPlay:
                                   str(download_head_url) + "\n" +
                                   str(download_video_url) + "\n" +
                                   str(download_cover_url) + "\n" +
-                                  str(Common.get_session()))
-                    Common.logger().info("==========视频信息已保存至info.txt==========")
+                                  str(Common.get_session("recommend")))
+                    Common.logger("recommend").info("==========视频信息已保存至info.txt==========")
 
                     # 上传视频
-                    Common.logger().info("开始上传视频:{}", download_video_title)
-                    Publish.upload_and_publish(env, "play")
+                    Common.logger("recommend").info("开始上传视频:{}", download_video_title)
+                    Publish.upload_and_publish("recommend", env, "play")
 
                     # 保存视频 ID 到云文档:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
-                    Common.logger().info("保存视频ID至云文档:{}", download_video_title)
+                    Common.logger("recommend").info("保存视频ID至云文档:{}", download_video_title)
                     # 看一看+ ，视频ID工作表，插入首行
-                    Feishu.insert_columns("20ce0c", "ROWS", 1, 2)
+                    Feishu.insert_columns("recommend", "kanyikan", "20ce0c", "ROWS", 1, 2)
                     # 看一看+ ，视频ID工作表，首行写入数据
                     upload_time = int(time.time())
                     values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
@@ -146,32 +150,32 @@ class DownloadPlay:
                                str(download_cover_url),
                                str(download_video_url)]]
                     time.sleep(1)
-                    Feishu.update_values("20ce0c", "A2:Q2", values)
+                    Feishu.update_values("recommend", "kanyikan", "20ce0c", "A2:Q2", values)
 
                     # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                    Common.logger().info("从云文档删除该视频信息:{}", download_video_title)
+                    Common.logger("recommend").info("从云文档删除该视频信息:{}", download_video_title)
                     # 删除行或列，可选 ROWS、COLUMNS
-                    Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                    Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                     return
         except Exception as e:
-            Common.logger().error("视频info异常:{}", e)
-            Feishu.dimension_range("SdCHOM", "ROWS", 2, 2)
+            Common.logger("recommend").error("视频info异常:{}", e)
+            Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", 2, 2)
             return
 
     # 执行播放量下载及上传
     @classmethod
-    def run_download_play_video(cls):
+    def run_download_play_video(cls, env):
         try:
             while True:
-                if len(Feishu.get_values_batch("SdCHOM")) == 1:
+                if len(Feishu.get_values_batch("recommend", "kanyikan", "SdCHOM")) == 1:
                     break
                 else:
-                    cls.download_play_video("prod")
+                    cls.download_play_video(env)
         except Exception as e:
-            Common.logger().error("执行上传及下载异常:{}", e)
+            Common.logger("recommend").error("执行上传及下载异常:{}", e)
 
 
 if __name__ == "__main__":
     download_play = DownloadPlay()
-    get_feeds()
-    download_play.download_play_video("dev")
+    get_feeds("recommend")
+    download_play.run_download_play_video("dev")

main/download_play_sendtime.py

@@ -9,7 +9,7 @@ import requests
 import urllib3
 sys.path.append(os.getcwd())
 from main.common import Common
-from main.get_feeds import get_feeds
+# from main.get_feeds import get_feeds
 from main.publish import Publish
 from main.feishu_lib import Feishu
 
@@ -46,13 +46,12 @@ class DownloadPlaySendtime:
         :return: 下载并上传视频
         """
         try:
-            for i in range(1, len(Feishu.get_values_batch("SdCHOM"))+1):
+            recommend_feeds = Feishu.get_values_batch("recommend", "kanyikan", "SdCHOM")
+            for i in range(1, len(recommend_feeds)+1):
                 time.sleep(1)
-
-                sendtime_session = Common.get_session()
-                # Common.logger().info("获取视频info时，session：{}", sendtime_session)
-                download_video_id = Feishu.get_values_batch("SdCHOM")[i][2]
-                download_video_title = Feishu.get_values_batch("SdCHOM")[i][3]
+                sendtime_session = Common.get_session("recommend")
+                download_video_id = recommend_feeds[i][2]
+                download_video_title = recommend_feeds[i][3]
                 url = "https://search.weixin.qq.com/cgi-bin/recwxa/recwxagetonevideoinfo?"
                 param = {
                     "session": sendtime_session,
@@ -70,9 +69,9 @@ class DownloadPlaySendtime:
                 r = requests.get(url=url, params=param, proxies=proxies, verify=False)
                 response = json.loads(r.content.decode("utf8"))
                 if "data" not in response:
-                    Common.logger().error("获取视频info时错误，删除该视频:{}", download_video_title)
+                    Common.logger("recommend").error("获取视频info时错误，删除该视频:{}", download_video_title)
                     # 删除行或列，可选 ROWS、COLUMNS
-                    Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                    Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                 else:
                     data = response["data"]
                     v_duration = data["duration"]
@@ -99,7 +98,7 @@ class DownloadPlaySendtime:
                         else:
                             download_url_sendtime = data["play_info"]["items"][0]["play_url"]
 
-                    Common.logger().info("正在判断第{}行，视频:{}", i, download_video_title)
+                    Common.logger("recommend").info("正在判断第{}行，视频:{}", i, download_video_title)
 
                     # 判断无效视频
                     if download_video_id == "" \
@@ -112,48 +111,49 @@ class DownloadPlaySendtime:
                             and v_user_cover == "" \
                             and v_video_cover == ""\
                             and download_url_sendtime == "":
-                        Common.logger().info("无效视频，删除该视频信息:{}", download_video_title)
+                        Common.logger("recommend").info("无效视频，删除该视频信息:{}", download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     elif v_send_date < 1622476800:
-                        Common.logger().info(
+                        Common.logger("recommend").info(
                             "发布时间小于2021年6月:{}，{}", download_video_title, v_send_date)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     # 抓取基础规则
                     elif cls.send_time_rule(v_width, v_height, v_duration, v_play_cnt_sendtime) is False:
-                        Common.logger().info("不满足发布时间榜下载规则，删除该视频信息:{}", download_video_title)
+                        Common.logger("recommend").info("不满足发布时间榜下载规则，删除该视频信息:{}", download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     # 不满足规则：发布时间 > 7 天 and 播放量 < 150000
                     elif int(time.time()) - int(v_send_date) > 604800 and int(v_play_cnt_sendtime) < 150000:
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Common.logger().info("播放量:{} < 150000", int(v_play_cnt_sendtime))
+                        Common.logger("recommend").info("播放量:{} < 150000", int(v_play_cnt_sendtime))
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     # 不满足规则：发布时间 <= 7 天 and 播放量 < 20000
                     elif int(time.time()) - int(v_send_date) <= 604800 and int(v_play_cnt_sendtime) < 20000:
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Common.logger().info("视频7天播放量:{} < 20000", int(v_play_cnt_sendtime))
+                        Common.logger("recommend").info("视频7天播放量:{} < 20000", int(v_play_cnt_sendtime))
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
-                    elif download_video_id in [j for m in Feishu.get_values_batch("20ce0c") for j in m]:
-                        Common.logger().info("视频已下载，删除该视频信息:{}", download_video_title)
+                    elif download_video_id in [j for m in Feishu.get_values_batch(
+                            "recommend", "kanyikan", "20ce0c") for j in m]:
+                        Common.logger("recommend").info("视频已下载，删除该视频信息:{}", download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     else:
-                        Common.logger().info("开始下载视频:{}", download_video_title)
+                        Common.logger("recommend").info("开始下载视频:{}", download_video_title)
 
                         # 下载封面
-                        Common.download_method("cover", download_video_title, v_video_cover)
+                        Common.download_method("recommend", "cover", download_video_title, v_video_cover)
                         # 下载视频
-                        Common.download_method("video", download_video_title, download_url_sendtime)
+                        Common.download_method("recommend", "video", download_video_title, download_url_sendtime)
                         # 保存视频信息到 "./files/{视频标题}/videoinfo.txt"
                         with open(r"./videos/" + download_video_title +
                                   "/" + "info.txt", "a", encoding="utf8") as f_a2:
@@ -171,23 +171,26 @@ class DownloadPlaySendtime:
                                        str(download_url_sendtime) + "\n" +
                                        str(v_video_cover) + "\n" +
                                        str(sendtime_session))
-                        Common.logger().info("==========视频信息已保存至info.txt==========")
+                        Common.logger("recommend").info("==========视频信息已保存至info.txt==========")
 
                         # 上传该视频
-                        Common.logger().info("开始上传视频:{}", download_video_title)
-                        Publish.upload_and_publish(env, "send_time")
+                        Common.logger("recommend").info("开始上传视频:{}", download_video_title)
+                        our_video_id = Publish.upload_and_publish("recommend", env, "send_time")
+                        our_video_link = "https://admin.piaoquantv.com/cms/post-detail/" + str(our_video_id) + "/info"
+                        Common.logger("recommend").info("视频上传完成:{}", download_video_title)
 
                         # 保存视频 ID 到云文档:
                         # https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
-                        Common.logger().info("保存视频ID至云文档:{}", download_video_title)
+                        Common.logger("recommend").info("保存视频ID至云文档:{}", download_video_title)
                         # 看一看+ ，视频ID工作表，插入首行
-                        Feishu.insert_columns("20ce0c", "ROWS", 1, 2)
+                        Feishu.insert_columns("recommend", "kanyikan", "20ce0c", "ROWS", 1, 2)
                         # 看一看+ ，视频ID工作表，首行写入数据
                         upload_time = int(time.time())
                         values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
                                    "推荐榜",
                                    str(download_video_id),
                                    str(download_video_title),
+                                   our_video_link,
                                    int(v_play_cnt_sendtime),
                                    v_comment_cnt,
                                    v_liked_cnt,
@@ -201,32 +204,45 @@ class DownloadPlaySendtime:
                                    str(v_video_cover),
                                    str(download_url_sendtime)]]
                         time.sleep(1)
-                        Feishu.update_values("20ce0c", "A2:Q2", values)
+                        Feishu.update_values("recommend", "kanyikan", "20ce0c", "F2:W2", values)
+
+                        # 保存视频信息到监控表
+                        Common.logger("recommend").info("添加视频到监控表:{}", download_video_title)
+                        # 插入空行
+                        time.sleep(1)
+                        Feishu.insert_columns("recommend", "monitor", "6fed97", "ROWS", 1, 2)
+                        # 视频信息写入监控表
+                        values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(upload_time))),
+                                   str(download_video_id),
+                                   download_video_title,
+                                   our_video_link,
+                                   v_duration,
+                                   time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(v_send_date)),
+                                   int(v_play_cnt_sendtime)]]
+                        time.sleep(1)
+                        Feishu.update_values("recommend", "monitor", "6fed97", "F2:L2", values)
 
-                        # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Common.logger().info("从云文档删除该视频信息:{}", download_video_title)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
 
         except Exception as e:
-            Common.logger().error("获取视频info异常:{}", e)
-            Feishu.dimension_range("SdCHOM", "ROWS", 2, 2)
+            Common.logger("recommend").error("获取视频info异常:{}", e)
+            Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", 2, 2)
 
     # 执行上传及下载
     @classmethod
-    def run_download_play_sendtime(cls):
+    def run_download_play_sendtime(cls, env):
         try:
             while True:
-                if len(Feishu.get_values_batch("SdCHOM")) == 1:
+                if len(Feishu.get_values_batch("recommend", "kanyikan", "SdCHOM")) == 1:
                     break
                 else:
-                    cls.download_play_sendtime("prod")
+                    cls.download_play_sendtime(env)
         except Exception as e:
-            Common.logger().error("执行上传及下载异常:{}", e)
+            Common.logger("recommend").error("执行上传及下载异常:{}", e)
 
 
 if __name__ == "__main__":
     download_sendtime = DownloadPlaySendtime()
-    get_feeds()
-    download_sendtime.download_play_sendtime("prod")
+    download_sendtime.run_download_play_sendtime("dev")

main/download_sendtime.py

@@ -52,13 +52,12 @@ class DownloadSendtime:
         :return: 下载并上传视频
         """
         try:
-            for i in range(1, len(Feishu.get_values_batch("SdCHOM"))+1):
+            recommend_feeds = Feishu.get_values_batch("recommend", "kanyikan", "SdCHOM")
+            for i in range(1, len(recommend_feeds)+1):
                 time.sleep(1)
-
-                sendtime_session = Common.get_session()
-                # Common.logger().info("获取视频info时，session：{}", sendtime_session)
-                download_video_id = Feishu.get_values_batch("SdCHOM")[i][2]
-                download_video_title = Feishu.get_values_batch("SdCHOM")[i][3]
+                sendtime_session = Common.get_session("recommend")
+                download_video_id = recommend_feeds[i][2]
+                download_video_title = recommend_feeds[i][3]
                 url = "https://search.weixin.qq.com/cgi-bin/recwxa/recwxagetonevideoinfo?"
                 param = {
                     "session": sendtime_session,
@@ -76,9 +75,9 @@ class DownloadSendtime:
                 r = requests.get(url=url, params=param, proxies=proxies, verify=False)
                 response = json.loads(r.content.decode("utf8"))
                 if "data" not in response:
-                    Common.logger().error("获取视频info时错误，删除该视频:{}", download_video_title)
+                    Common.logger("recommend").error("获取视频info时错误，删除该视频:{}", download_video_title)
                     # 删除行或列，可选 ROWS、COLUMNS
-                    Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                    Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                 else:
                     data = response["data"]
                     v_duration = data["duration"]
@@ -105,7 +104,7 @@ class DownloadSendtime:
                         else:
                             download_url_sendtime = data["play_info"]["items"][0]["play_url"]
 
-                    Common.logger().info("正在判断第{}行，视频:{}", i, download_video_title)
+                    Common.logger("recommend").info("正在判断第{}行，视频:{}", i, download_video_title)
 
                     # 判断无效视频
                     if download_video_id == "" \
@@ -118,44 +117,45 @@ class DownloadSendtime:
                             and v_user_cover == "" \
                             and v_video_cover == ""\
                             and download_url_sendtime == "":
-                        Common.logger().info("无效视频，删除该视频信息:{}", download_video_title)
+                        Common.logger("recommend").info("无效视频，删除该视频信息:{}", download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     # 发布时间榜下载规则
                     elif cls.send_time_rule(v_width, v_height, v_duration, v_play_cnt_sendtime) is False:
-                        Common.logger().info("不满足发布时间榜下载规则，删除该视频信息:{}", download_video_title)
+                        Common.logger("recommend").info("不满足发布时间榜下载规则，删除该视频信息:{}", download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     # 发布时间 <=7 天
                     elif int(time.time()) - int(v_send_date) > 604800:
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Common.logger().info("视频发布时间大于7天：{}天；标题：{}",
-                                             int((int(time.time()) - int(v_send_date)) / 86400),
-                                             download_video_title)
+                        Common.logger("recommend").info("视频发布时间大于7天：{}天；标题：{}",
+                                                        int((int(time.time()) - int(v_send_date)) / 86400),
+                                                        download_video_title)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     elif int(v_play_cnt_sendtime) < 20000:
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Common.logger().info("该视频7天播放量：{}<10000 ；不满足下载规则:{}",
-                                             int(v_play_cnt_sendtime), download_video_title)
+                        Common.logger("recommend").info("该视频7天播放量：{}<10000 ；不满足下载规则:{}",
+                                                        int(v_play_cnt_sendtime), download_video_title)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
-                    elif download_video_id in [j for m in Feishu.get_values_batch("20ce0c") for j in m]:
-                        Common.logger().info("视频已下载，删除该视频信息:{}", download_video_title)
+                    elif download_video_id in [j for m in Feishu.get_values_batch(
+                            "recommend", "kanyikan", "20ce0c") for j in m]:
+                        Common.logger("recommend").info("视频已下载，删除该视频信息:{}", download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
                     else:
-                        Common.logger().info("开始下载视频:{}", download_video_title)
+                        Common.logger("recommend").info("开始下载视频:{}", download_video_title)
 
                         # 下载封面
-                        Common.download_method("cover", download_video_title, v_video_cover)
+                        Common.download_method("recommend", "cover", download_video_title, v_video_cover)
                         # 下载视频
-                        Common.download_method("video", download_video_title, download_url_sendtime)
+                        Common.download_method("recommend", "video", download_video_title, download_url_sendtime)
                         # 保存视频信息到 "./files/{视频标题}/videoinfo.txt"
                         with open(r"./videos/" + download_video_title +
                                   "/" + "info.txt", "a", encoding="utf8") as f_a2:
@@ -173,17 +173,17 @@ class DownloadSendtime:
                                        str(download_url_sendtime) + "\n" +
                                        str(v_video_cover) + "\n" +
                                        str(sendtime_session))
-                        Common.logger().info("==========视频信息已保存至info.txt==========")
+                        Common.logger("recommend").info("==========视频信息已保存至info.txt==========")
 
                         # 上传该视频
-                        Common.logger().info("开始上传视频:{}", download_video_title)
-                        Publish.upload_and_publish(env, "send_time")
+                        Common.logger("recommend").info("开始上传视频:{}", download_video_title)
+                        Publish.upload_and_publish("recommend", env, "send_time")
 
                         # 保存视频 ID 到云文档:
                         # https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
-                        Common.logger().info("保存视频ID至云文档:{}", download_video_title)
+                        Common.logger("recommend").info("保存视频ID至云文档:{}", download_video_title)
                         # 看一看+ ，视频ID工作表，插入首行
-                        Feishu.insert_columns("20ce0c", "ROWS", 1, 2)
+                        Feishu.insert_columns("recommend", "kanyikan", "20ce0c", "ROWS", 1, 2)
                         # 看一看+ ，视频ID工作表，首行写入数据
                         upload_time = int(time.time())
                         values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
@@ -203,31 +203,31 @@ class DownloadSendtime:
                                    str(v_video_cover),
                                    str(download_url_sendtime)]]
                         time.sleep(1)
-                        Feishu.update_values("20ce0c", "A2:Q2", values)
+                        Feishu.update_values("recommend", "kanyikan", "20ce0c", "A2:Q2", values)
 
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Common.logger().info("从云文档删除该视频信息:{}", download_video_title)
+                        Common.logger("recommend").info("从云文档删除该视频信息:{}", download_video_title)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                         return
 
         except Exception as e:
-            Common.logger().error("获取视频info异常:{}", e)
+            Common.logger("recommend").error("获取视频info异常:{}", e)
 
     # 执行上传及下载
     @classmethod
     def run_download_sendtime_video(cls):
         try:
             while True:
-                if len(Feishu.get_values_batch("SdCHOM")) == 1:
+                if len(Feishu.get_values_batch("recommend", "kanyikan", "SdCHOM")) == 1:
                     break
                 else:
                     cls.download_sendtime_video("prod")
         except Exception as e:
-            Common.logger().error("执行上传及下载异常:{}", e)
+            Common.logger("recommend").error("执行上传及下载异常:{}", e)
 
 
 if __name__ == "__main__":
     download_sendtime = DownloadSendtime()
-    get_feeds()
+    get_feeds("recommend")
     download_sendtime.download_sendtime_video("dev")

main/download_up.py

@@ -57,14 +57,15 @@ class DownloadUp:
             1 删除该视频在 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM 中的信息
         """
         try:
-            for i in range(1, len(Feishu.get_values_batch("SdCHOM"))+1):
+            recommend_feeds = Feishu.get_values_batch("recommend", "kanyikan", "SdCHOM")
+            for i in range(1, len(recommend_feeds)+1):
                 time.sleep(1)
-                video_info_session = Common.get_session()
-                download_time = Feishu.get_values_batch("SdCHOM")[i][0]  # 第一次获取该视频的时间
+                video_info_session = Common.get_session("recommend")
+                download_time = recommend_feeds[i][0]  # 第一次获取该视频的时间
                 download_time = int(time.mktime(time.strptime(download_time, "%Y/%m/%d %H:%M:%S")))
-                download_video_id = Feishu.get_values_batch("SdCHOM")[i][2]  # 外网视频 ID
-                download_video_title = Feishu.get_values_batch("SdCHOM")[i][3]  # 视频标题
-                download_video_play_cnt = Feishu.get_values_batch("SdCHOM")[i][4]  # 播放量
+                download_video_id = recommend_feeds[i][2]  # 外网视频 ID
+                download_video_title = recommend_feeds[i][3]  # 视频标题
+                download_video_play_cnt = recommend_feeds[i][4]  # 播放量
 
                 url = "https://search.weixin.qq.com/cgi-bin/recwxa/recwxagetonevideoinfo?"
                 param = {
@@ -83,9 +84,9 @@ class DownloadUp:
                 r = requests.get(url=url, params=param, proxies=proxies, verify=False)
                 response = json.loads(r.content.decode("utf8"))
                 if "data" not in response:
-                    Common.logger().error("获取视频info时错误，删除该视频:{}", download_video_title)
+                    Common.logger("recommend").error("获取视频info时错误，删除该视频:{}", download_video_title)
                     # 删除行或列，可选 ROWS、COLUMNS
-                    Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                    Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
                 else:
                     data = response["data"]
                     v_duration = data["duration"]
@@ -112,7 +113,7 @@ class DownloadUp:
                         else:
                             download_url_up = data["play_info"]["items"][0]["play_url"]
 
-                    Common.logger().info("正在判断第{}行，视频:{}", i, download_video_title)
+                    Common.logger("recommend").info("正在判断第{}行，视频:{}", i, download_video_title)
 
                     # 判断无效视频
                     if download_video_id == "" \
@@ -125,47 +126,48 @@ class DownloadUp:
                             and v_user_cover == "" \
                             and v_video_cover == ""\
                             and download_url_up == "":
-                        Common.logger().info("无效视频，删除该视频信息:{}", download_video_title)
+                        Common.logger("recommend").info("无效视频，删除该视频信息:{}", download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
 
                     # 上升榜时长不足 1 小时
                     elif int(time.time()) - int(download_time) < 3600:
-                        Common.logger().info("距上次获取该视频时间：{}分钟；{}",
-                                             int((int(int(time.time()) - int(download_time))) / 60),
-                                             download_video_title)
+                        Common.logger("recommend").info("距上次获取该视频时间：{}分钟；{}",
+                                                        int((int(int(time.time()) - int(download_time))) / 60),
+                                                        download_video_title)
 
                     # 上升榜时长超过 2 小时
                     elif int(time.time()) - int(download_time) > 7200:
-                        Common.logger().info("距上次获取该视频时间：""{}分钟。超过2小时，删除该视频；标题：{}",
-                                             int((int(time.time()) - int(download_time)) / 60),
-                                             download_video_title)
+                        Common.logger("recommend").info("距上次获取该视频时间：""{}分钟。超过2小时，删除该视频；标题：{}",
+                                                        int((int(time.time()) - int(download_time)) / 60),
+                                                        download_video_title)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
 
                     # 上升榜下载规则
                     elif cls.up_rule(v_width, v_height, v_duration, v_play_cnt_up) is False:
-                        Common.logger().info("不满足上升榜下载规则，删除视频:{}", download_video_title)
+                        Common.logger("recommend").info("不满足上升榜下载规则，删除视频:{}", download_video_title)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
 
                     # 从已下载视频表中去重:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
-                    elif download_video_id in [j for m in Feishu.get_values_batch("20ce0c") for j in m]:
-                        Common.logger().info("视频已下载，删除该视频信息:{}", download_video_title)
+                    elif download_video_id in [j for m in Feishu.get_values_batch(
+                            "recommend", "kanyikan", "20ce0c") for j in m]:
+                        Common.logger("recommend").info("视频已下载，删除该视频信息:{}", download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
 
                     # 上升榜时长在 1-2 小时内，播放量增长>=1000
                     elif (7200 >= int(time.time()) - int(download_time) >= 3600)\
                             and (int(v_play_cnt_up) - int(download_video_play_cnt) >= 1000):
-                        Common.logger().info(
+                        Common.logger("recommend").info(
                             "视频:{}，在上升榜时间内的播放量{}>=1000，开始下载视频",
                             download_video_title, int(v_play_cnt_up) - int(download_video_play_cnt))
 
                         # 下载封面
-                        Common.download_method("cover", download_video_title, v_video_cover)
+                        Common.download_method("recommend", "cover", download_video_title, v_video_cover)
                         # 下载视频
-                        Common.download_method("video", download_video_title, download_url_up)
+                        Common.download_method("recommend", "video", download_video_title, download_url_up)
                         # 保存视频信息到 "./files/{视频标题}/videoinfo.txt"
                         with open(r"./videos/" + download_video_title
                                   + "/" + "info.txt", "a", encoding="utf8") as f_a2:
@@ -183,17 +185,17 @@ class DownloadUp:
                                        str(download_url_up) + "\n" +
                                        str(v_video_cover) + "\n" +
                                        str(video_info_session))
-                        Common.logger().info("==========视频信息已保存至info.txt==========")
+                        Common.logger("recommend").info("==========视频信息已保存至info.txt==========")
 
                         # 上传该视频
-                        Common.logger().info("开始上传视频:{}", download_video_title)
-                        Publish.upload_and_publish(env, "up")
+                        Common.logger("recommend").info("开始上传视频:{}", download_video_title)
+                        Publish.upload_and_publish("recommend", env, "up")
 
                         # 保存视频 ID 到云文档:
                         # https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
-                        Common.logger().info("保存视频ID至云文档:{}", download_video_title)
+                        Common.logger("recommend").info("保存视频ID至云文档:{}", download_video_title)
                         # 看一看+ ，视频ID工作表，插入首行
-                        Feishu.insert_columns("20ce0c", "ROWS", 1, 2)
+                        Feishu.insert_columns("recommend", "kanyikan", "20ce0c", "ROWS", 1, 2)
                         # 看一看+ ，视频ID工作表，首行写入数据
                         upload_time = int(time.time())
                         values = [[time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(upload_time)),
@@ -213,27 +215,27 @@ class DownloadUp:
                                    str(v_video_cover),
                                    str(download_url_up)]]
                         time.sleep(1)
-                        Feishu.update_values("20ce0c", "A2:Q2", values)
+                        Feishu.update_values("recommend", "kanyikan", "20ce0c", "A2:Q2", values)
 
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Common.logger().info("从云文档删除该视频信息:{}", download_video_title)
+                        Common.logger("recommend").info("从云文档删除该视频信息:{}", download_video_title)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
 
                     # 上升榜时长在 1-2 小时内，播放量增长<1000
                     elif (7200 >= int(time.time()) - int(download_time) >= 3600)\
                             and (int(v_play_cnt_up) - int(download_video_play_cnt) < 1000):
                         # 删除之前保存的该视频信息，并把现在的信息保存进去
-                        Common.logger().info("该视频1小时内的播放量：{}<1000；更新该视频信息:{}",
-                                             int(v_play_cnt_up) - int(download_video_play_cnt),
-                                             download_video_title)
+                        Common.logger("recommend").info("该视频1小时内的播放量：{}<1000；更新该视频信息:{}",
+                                                        int(v_play_cnt_up) - int(download_video_play_cnt),
+                                                        download_video_title)
                         # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                        Common.logger().info("从云文档删除该视频信息:{}", download_video_title)
+                        Common.logger("recommend").info("从云文档删除该视频信息:{}", download_video_title)
                         # 删除行或列，可选 ROWS、COLUMNS
-                        Feishu.dimension_range("SdCHOM", "ROWS", i + 1, i + 1)
+                        Feishu.dimension_range("recommend", "kanyikan", "SdCHOM", "ROWS", i + 1, i + 1)
 
                         # 看一看+工作表，插入首行
-                        Feishu.insert_columns("SdCHOM", "ROWS", 1, 2)
+                        Feishu.insert_columns("recommend", "kanyikan", "SdCHOM", "ROWS", 1, 2)
 
                         # 获取当前时间
                         download_up_time = int(time.time())
@@ -254,14 +256,14 @@ class DownloadUp:
                                    str(v_video_cover),
                                    str(download_url_up)]]
                         time.sleep(1)
-                        Feishu.update_values("SdCHOM", "A2:Q2", values)
+                        Feishu.update_values("recommend", "kanyikan", "SdCHOM", "A2:Q2", values)
 
         except Exception as e:
             # 从云文档删除该视频信息:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-            Common.logger().error("获取视频info异常:{}", e)
+            Common.logger("recommend").error("获取视频info异常:{}", e)
 
 
 if __name__ == "__main__":
     downloadup = DownloadUp()
-    get_feeds()
+    get_feeds("recommend")
     downloadup.download_up_video("dev")

main/feishu_lib.py

@@ -2,7 +2,6 @@
 # @Author: wangkun
 # @Time: 2022/5/6
 import json
-import time
 
 import requests
 import urllib3
@@ -16,12 +15,37 @@ class Feishu:
     """
     编辑飞书云文档
     """
-    feishu_url = "https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?"
-    spreadsheetToken = "shtcngRPoDYAi24x52j2nDuHMih"
+    # 看一看爬虫数据表
+    kanyikan_url = "https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?"
+    # 快手爬虫数据表
+    kuaishou_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnp4SaJt37q6OOOrYzPMjQkg?"
+    # 微视爬虫数据表
+    weishi_url = "https://w42nne6hzg.feishu.cn/sheets/shtcn5YSWg91JfVGzj0SFZIRRPh?"
+    # 小年糕爬虫数据表
+    xiaoniangao_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?"
+    # 数据监控表
+    crawler_monitor = "https://w42nne6hzg.feishu.cn/sheets/shtcnlZWYazInhf7Z60jkbLRJyd?"
+
+    # 飞书路径token
+    @classmethod
+    def spreadsheettoken(cls, crawler):
+        """
+        :param crawler: 哪个爬虫
+        """
+        if crawler == "kanyikan":
+            return "shtcngRPoDYAi24x52j2nDuHMih"
+        elif crawler == "kuaishou":
+            return "shtcnp4SaJt37q6OOOrYzPMjQkg"
+        elif crawler == "weishi":
+            return "shtcn5YSWg91JfVGzj0SFZIRRPh"
+        elif crawler == "xiaoniangao":
+            return "shtcnYxiyQ1wLklo1W5Kdqc9cGh"
+        elif crawler == "monitor":
+            return "shtcnlZWYazInhf7Z60jkbLRJyd"
 
     # 获取飞书api token
     @classmethod
-    def get_token(cls):
+    def get_token(cls, log_type):
         """
         获取飞书api token
         :return:
@@ -32,23 +56,24 @@ class Feishu:
 
         try:
             urllib3.disable_warnings()
-            time.sleep(1)
             response = requests.post(url=url, data=post_data, proxies=proxies, verify=False)
             tenant_access_token = response.json()["tenant_access_token"]
             return tenant_access_token
         except Exception as e:
-            Common.logger().error("获取飞书 api token 异常:{}", e)
+            Common.logger(log_type).error("获取飞书 api token 异常:{}", e)
 
     # 获取表格元数据
     @classmethod
-    def get_metainfo(cls):
+    def get_metainfo(cls, log_type, crawler):
         """
         获取表格元数据
         :return:
         """
-        url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" + cls.spreadsheetToken + "/metainfo"
+        get_metainfo_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                           + cls.spreadsheettoken(crawler) + "/metainfo"
+
         headers = {
-            "Authorization": "Bearer " + cls.get_token(),
+            "Authorization": "Bearer " + cls.get_token(log_type),
             "Content-Type": "application/json; charset=utf-8"
         }
         params = {
@@ -57,24 +82,26 @@ class Feishu:
         }
         try:
             urllib3.disable_warnings()
-            r = requests.get(url=url, headers=headers, params=params, proxies=proxies, verify=False)
+            r = requests.get(url=get_metainfo_url, headers=headers, params=params, proxies=proxies, verify=False)
             response = json.loads(r.content.decode("utf8"))
             return response
         except Exception as e:
-            Common.logger().error("获取表格元数据异常:{}", e)
+            Common.logger(log_type).error("获取表格元数据异常:{}", e)
 
     # 读取工作表中所有数据
     @classmethod
-    def get_values_batch(cls, sheetid):
+    def get_values_batch(cls, log_type, crawler, sheetid):
         """
         读取工作表中所有数据
+        :param log_type: 启用哪个 log
+        :param crawler: 哪个爬虫
         :param sheetid: 哪张表
         :return: 所有数据
         """
-
-        url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" + cls.spreadsheetToken + "/values_batch_get"
+        get_values_batch_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                               + cls.spreadsheettoken(crawler) + "/values_batch_get"
         headers = {
-            "Authorization": "Bearer " + cls.get_token(),
+            "Authorization": "Bearer " + cls.get_token(log_type),
             "Content-Type": "application/json; charset=utf-8"
         }
         params = {
@@ -85,7 +112,7 @@ class Feishu:
             # valueRenderOption=FormattedValue 计算并格式化单元格；
             # valueRenderOption=Formula单元格中含有公式时返回公式本身；
             # valueRenderOption=UnformattedValue计算但不对单元格进行格式化
-            "valueRenderOption": "FormattedValue",
+            "valueRenderOption": "ToString",
 
             # dateTimeRenderOption=FormattedString 计算并将时间日期按照其格式进行格式化，但不会对数字进行格式化，返回格式化后的字符串。
             "dateTimeRenderOption": "",
@@ -95,29 +122,29 @@ class Feishu:
         }
         try:
             urllib3.disable_warnings()
-            time.sleep(0.5)
-            r = requests.get(url=url, headers=headers, params=params, proxies=proxies, verify=False)
+            r = requests.get(url=get_values_batch_url, headers=headers, params=params, proxies=proxies, verify=False)
             response = json.loads(r.content.decode("utf8"))
             values = response["data"]["valueRanges"][0]["values"]
             return values
         except Exception as e:
-            Common.logger().error("读取工作表所有数据异常:{}", e)
+            Common.logger(log_type).error("读取工作表所有数据异常:{}", e)
 
     # 工作表，插入行或列
     @classmethod
-    def insert_columns(cls, sheetid, majordimension, startindex, endindex):
+    def insert_columns(cls, log_type, crawler, sheetid, majordimension, startindex, endindex):
         """
-        工作表，插入行或列
-        :param sheetid: 哪张表
-        :param majordimension: 行或列，默认 ROWS ，可选 ROWS、COLUMNS
-        :param startindex: 开始的位置
-        :param endindex: 结束的位置
-        :return:插入首行
+        工作表插入行或列
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫的云文档
+        :param sheetid:哪张工作表
+        :param majordimension:行或者列
+        :param startindex:开始位置
+        :param endindex:结束位置
         """
-        url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/"\
-              + cls.spreadsheetToken + "/insert_dimension_range"
+        insert_columns_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                             + cls.spreadsheettoken(crawler) + "/insert_dimension_range"
         headers = {
-            "Authorization": "Bearer " + cls.get_token(),
+            "Authorization": "Bearer " + cls.get_token(log_type),
             "Content-Type": "application/json; charset=utf-8"
         }
         body = {
@@ -131,26 +158,26 @@ class Feishu:
         }
         try:
             urllib3.disable_warnings()
-            time.sleep(0.5)
-            r = requests.post(url=url, headers=headers, json=body, proxies=proxies, verify=False)
-            Common.logger().info("插入空行或列:{}", r.json()["msg"])
+            r = requests.post(url=insert_columns_url, headers=headers, json=body, proxies=proxies, verify=False)
+            Common.logger(log_type).info("插入行或列:{}", r.json()["msg"])
         except Exception as e:
-            Common.logger().error("插入空行或列异常:{}", e)
+            Common.logger(log_type).error("插入行或列异常:{}", e)
 
-    # 工作表，写入数据
+    # 写入数据
     @classmethod
-    def update_values(cls, sheetid, ranges, values):
+    def update_values(cls, log_type, crawler, sheetid, ranges, values):
         """
         写入数据
-        :param sheetid: 哪张工作表
-        :param ranges: 单元格范围
-        :param values: 更新值
-        :return:
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫的云文档
+        :param sheetid:哪张工作表
+        :param ranges:单元格范围
+        :param values:写入的具体数据，list
         """
-
-        url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" + cls.spreadsheetToken + "/values_batch_update"
+        update_values_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                            + cls.spreadsheettoken(crawler) + "/values_batch_update"
         headers = {
-            "Authorization": "Bearer " + cls.get_token(),
+            "Authorization": "Bearer " + cls.get_token(log_type),
             "Content-Type": "application/json; charset=utf-8"
         }
         body = {
@@ -161,33 +188,65 @@ class Feishu:
                 },
             ],
         }
+
+        try:
+            urllib3.disable_warnings()
+            r = requests.post(url=update_values_url, headers=headers, json=body, proxies=proxies, verify=False)
+            Common.logger(log_type).info("写入数据:{}", r.json()["msg"])
+        except Exception as e:
+            Common.logger(log_type).error("写入数据异常:{}", e)
+
+    # 合并单元格
+    @classmethod
+    def merge_cells(cls, log_type, crawler, sheetid, ranges):
+        """
+        合并单元格
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫
+        :param sheetid:哪张工作表
+        :param ranges:需要合并的单元格范围
+        """
+        merge_cells_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                          + cls.spreadsheettoken(crawler) + "/merge_cells"
+        headers = {
+            "Authorization": "Bearer " + cls.get_token(log_type),
+            "Content-Type": "application/json; charset=utf-8"
+        }
+
+        body = {
+            "range": sheetid + "!" + ranges,
+            "mergeType": "MERGE_ROWS"
+        }
+
         try:
             urllib3.disable_warnings()
-            r = requests.post(url=url, headers=headers, json=body, proxies=proxies, verify=False)
-            Common.logger().info("写入数据:{}", r.json()["msg"])
+            r = requests.post(url=merge_cells_url, headers=headers, json=body, proxies=proxies, verify=False)
+            Common.logger(log_type).info("合并单元格:{}", r.json()["msg"])
         except Exception as e:
-            Common.logger().error("写入数据异常:{}", e)
+            Common.logger(log_type).error("合并单元格异常:{}", e)
 
     # 读取单元格数据
     @classmethod
-    def get_range_value(cls, sheetid, cell):
+    def get_range_value(cls, log_type, crawler, sheetid, cell):
         """
         读取单元格内容
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫
         :param sheetid: 哪张工作表
         :param cell: 哪个单元格
         :return: 单元格内容
         """
-        url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
-              + cls.spreadsheetToken + "/values/" + sheetid + "!" + cell
+        get_range_value_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                              + cls.spreadsheettoken(crawler) + "/values/" + sheetid + "!" + cell
         headers = {
-            "Authorization": "Bearer " + cls.get_token(),
+            "Authorization": "Bearer " + cls.get_token(log_type),
             "Content-Type": "application/json; charset=utf-8"
         }
         params = {
             # valueRenderOption=ToString 可返回纯文本的值(数值类型除外)；
             # valueRenderOption=FormattedValue 计算并格式化单元格；
-            # valueRenderOption=Formula单元格中含有公式时返回公式本身；
-            # valueRenderOption=UnformattedValue计算但不对单元格进行格式化。
+            # valueRenderOption=Formula 单元格中含有公式时返回公式本身；
+            # valueRenderOption=UnformattedValue 计算但不对单元格进行格式化。
             "valueRenderOption": "FormattedValue",
 
             # dateTimeRenderOption=FormattedString 计算并对时间日期按照其格式进行格式化，但不会对数字进行格式化，返回格式化后的字符串。
@@ -198,26 +257,28 @@ class Feishu:
         }
         try:
             urllib3.disable_warnings()
-            time.sleep(0.5)
-            r = requests.get(url=url, headers=headers, params=params, proxies=proxies, verify=False)
+            r = requests.get(url=get_range_value_url, headers=headers, params=params, proxies=proxies, verify=False)
             return r.json()["data"]["valueRange"]["values"][0]
         except Exception as e:
-            Common.logger().error("读取单元格数据异常:{}", e)
+            Common.logger(log_type).error("读取单元格数据异常:{}", e)
 
     # 删除行或列，可选 ROWS、COLUMNS
     @classmethod
-    def dimension_range(cls, sheetid, major_dimension, startindex, endindex):
+    def dimension_range(cls, log_type, crawler, sheetid, major_dimension, startindex, endindex):
         """
         删除行或列
+        :param log_type: 日志路径
+        :param crawler: 哪个爬虫
         :param sheetid:工作表
         :param major_dimension:默认 ROWS ，可选 ROWS、COLUMNS
         :param startindex:开始的位置
         :param endindex:结束的位置
         :return:
         """
-        url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" + cls.spreadsheetToken + "/dimension_range"
+        dimension_range_url = "https://open.feishu.cn/open-apis/sheets/v2/spreadsheets/" \
+                              + cls.spreadsheettoken(crawler) + "/dimension_range"
         headers = {
-            "Authorization": "Bearer " + cls.get_token(),
+            "Authorization": "Bearer " + cls.get_token(log_type),
             "Content-Type": "application/json; charset=utf-8"
         }
         body = {
@@ -230,31 +291,19 @@ class Feishu:
             }
         try:
             urllib3.disable_warnings()
-            r = requests.delete(url=url, headers=headers, json=body, proxies=proxies, verify=False)
-            Common.logger().info("删除视频数据:{}", r.json()["msg"])
+            r = requests.delete(url=dimension_range_url, headers=headers, json=body, proxies=proxies, verify=False)
+            Common.logger(log_type).info("删除视频数据:{}", r.json()["msg"])
         except Exception as e:
-            Common.logger().error("删除视频数据异常:{}", e)
+            Common.logger(log_type).error("删除视频数据异常:{}", e)
 
 
 if __name__ == "__main__":
     feishu = Feishu()
 
-    # 获取飞书api token
-    # print(feishu.get_token())
-    # # 获取表格元数据
-    # feishu.get_metainfo()
-
-    # 读取工作表中所有数据
-    # print(feishu.get_values_batch("Zt2PGQ")[1][3])
-    # print(len(feishu.get_values_batch("SdCHOM")))
-
-    # # 看一看+工作表，插入首行
-    # print(feishu.insert_columns("Y8N3Vl"))
-
-    # # 查询单元格内容
-    # print(feishu.get_range_value("Y8N3Vl", "B8:C8"))
-    #
-    # # 删除行或列，可选 ROWS、COLUMNS
-    # feishu.dimension_range("Y8N3Vl", "ROWS")
+    print(feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "B3:B3")[0])
+    print(feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "B4:B4")[0])
+    print(feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "C5:C5")[0][0]["link"])
+    print(feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "B6:B6")[0])
+    print(feishu.get_range_value("person", "xiaoniangao", "dzcWHw", "B7:B7")[0])
 
     pass

main/get_feeds.py

@@ -21,11 +21,11 @@ proxies = {"http": None, "https": None}
 
 
 # 敏感词库
-def kanyikan_sensitive_words():
+def kanyikan_sensitive_words(log_type):
     # 敏感词库列表
     word_list = []
     # 从云文档读取所有敏感词，添加到词库列表
-    lists = Feishu.get_values_batch("rofdM5")
+    lists = Feishu.get_values_batch(log_type, "kanyikan", "rofdM5")
     for i in lists:
         for j in i:
             # 过滤空的单元格内容
@@ -36,18 +36,18 @@ def kanyikan_sensitive_words():
     return word_list
 
 
-def get_feeds():
+def get_feeds(log_type):
     """
     1.从看一看+小程序首页推荐，获取视频列表
     2.先在 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c 中去重
     3.再从 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM 中去重
     4.添加视频信息至 https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
     """
-    Common.logger().info("开始从推荐页获取视频列表")
+    Common.logger(log_type).info("开始从推荐页获取视频列表")
     host = "https://search.weixin.qq.com"
     url = '/cgi-bin/recwxa/recwxavideolist?'
-    video_list_session = Common.get_session()
-    # Common.logger().info("获取视频list时，session：{}", video_list_session)
+    video_list_session = Common.get_session(log_type)
+    # Common.logger(log_type).info("获取视频list时，session：{}", video_list_session)
     header = {
         "Connection": "keep-alive",
         "content-type": "application/json",
@@ -77,21 +77,21 @@ def get_feeds():
         response = json.loads(r.content.decode("utf8"))
 
         if "data" not in response:
-            Common.logger().info("获取视频list时，session过期，随机睡眠 31-50 秒")
+            Common.logger(log_type).info("获取视频list时，session过期，随机睡眠 31-50 秒")
             # 如果返回空信息，则随机睡眠 31-40 秒
             time.sleep(random.randint(31, 40))
-            get_feeds()
+            get_feeds(log_type)
         elif "items" not in response["data"]:
-            Common.logger().info("获取视频list时，返回空信息，随机睡眠 1-3 分钟")
+            Common.logger(log_type).info("获取视频list时，response:{}，随机睡眠 1-3 分钟", response)
             # 如果返回空信息，则随机睡眠 1-3 分钟
             time.sleep(random.randint(60, 180))
-            get_feeds()
+            get_feeds(log_type)
         else:
             items = response["data"]["items"]
             for i in range(len(items)):
                 # 如果该视频没有视频信息，则忽略
                 if "videoInfo" not in items[i]:
-                    Common.logger().info("无视频信息")
+                    Common.logger(log_type).info("无视频信息")
                 else:
                     # 获取视频标题
                     video_title = items[i]["title"].strip().replace("\n", "")\
@@ -101,43 +101,43 @@ def get_feeds():
                         .replace(">", "").replace("|", "").replace(" ", "")\
                         .replace("&NBSP", "").replace(".", "。").replace(" ", "")\
                         .replace("小年糕", "").replace("#", "").replace("Merge", "")
-                    Common.logger().info('视频标题：{}', video_title)
+                    Common.logger(log_type).info('视频标题：{}', video_title)
 
                     # 获取视频ID
                     video_id = items[i]["videoId"]
-                    Common.logger().info('视频ID：{}', video_id)
+                    Common.logger(log_type).info('视频ID：{}', video_id)
                 
                     # 获取视频播放次数
                     video_play_cnt = items[i]["playCount"]
-                    Common.logger().info('视频播放次数：{}', video_play_cnt)
+                    Common.logger(log_type).info('视频播放次数：{}', video_play_cnt)
                 
                     # 获取视频点赞数
                     video_liked_cnt = items[i]["liked_cnt"]
-                    Common.logger().info('视频点赞数：{}', video_liked_cnt)
+                    Common.logger(log_type).info('视频点赞数：{}', video_liked_cnt)
                 
                     # 获取视频评论数
                     video_comment_cnt = items[i]["comment_cnt"]
-                    Common.logger().info('视频评论数：{}', video_comment_cnt)
+                    Common.logger(log_type).info('视频评论数：{}', video_comment_cnt)
                 
                     # 获取视频分享数
                     video_shared_cnt = items[i]["shared_cnt"]
-                    Common.logger().info('视频分享数：{}', video_shared_cnt)
+                    Common.logger(log_type).info('视频分享数：{}', video_shared_cnt)
 
                     # 获取视频时长
                     video_duration = items[i]["mediaDuration"]
-                    Common.logger().info('视频时长：{}秒', video_duration)
+                    Common.logger(log_type).info('视频时长：{}秒', video_duration)
 
                     # 获取视频宽高
                     if "short_video_info" not in items[i]:
                         video_width = "0"
                         video_height = "0"
                         video_resolution = str(video_width) + "*" + str(video_height)
-                        Common.logger().info("无分辨率：{}", video_resolution)
+                        Common.logger(log_type).info("无分辨率：{}", video_resolution)
                     elif len(items[i]["short_video_info"]) == 0:
                         video_width = "0"
                         video_height = "0"
                         video_resolution = str(video_width) + "*" + str(video_height)
-                        Common.logger().info("无分辨率：{}", video_resolution)
+                        Common.logger(log_type).info("无分辨率：{}", video_resolution)
 
                     else:
                         # 视频宽
@@ -145,16 +145,16 @@ def get_feeds():
                         # 视频高
                         video_height = items[i]["short_video_info"]["height"]
                         video_resolution = str(video_width) + "*" + str(video_height)
-                        Common.logger().info('视频宽高：{}', video_resolution)
+                        Common.logger(log_type).info('视频宽高：{}', video_resolution)
                 
                     # 获取视频发布时间
                     video_send_date = items[i]["date"]
-                    Common.logger().info("视频发布时间：{}",
-                                         time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(video_send_date)))
+                    Common.logger(log_type).info("视频发布时间：{}",
+                                                 time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(video_send_date)))
                 
                     # 获取视频用户名
                     video_user = items[i]["source"].strip().replace("\n", "")
-                    Common.logger().info('视频用户名：{}', video_user)
+                    Common.logger(log_type).info('视频用户名：{}', video_user)
 
                     # user_id
                     if "openid" not in items[i]:
@@ -164,30 +164,30 @@ def get_feeds():
                 
                     # 获取视频用户头像
                     video_user_cover = items[i]["bizIcon"]
-                    Common.logger().info('视频用户头像：{}', video_user_cover)
+                    Common.logger(log_type).info('视频用户头像：{}', video_user_cover)
                 
                     # 获取视频封面
                     if "smartCoverUrl" in items[i]:
                         video_cover = items[i]["smartCoverUrl"]
-                        Common.logger().info('视频封面：{}', video_cover)
+                        Common.logger(log_type).info('视频封面：{}', video_cover)
                     else:
                         video_cover = items[i]["thumbUrl"]
-                        Common.logger().info('视频封面：{}', video_cover)
+                        Common.logger(log_type).info('视频封面：{}', video_cover)
                 
                     # 获取播放地址
                     if "mpInfo" in items[i]["videoInfo"]["videoCdnInfo"].keys():
                         if len(items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"]) > 2:
                             url = items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"][2]["url"]
-                            Common.logger().info('视频播放地址：{}', url)
+                            Common.logger(log_type).info('视频播放地址：{}', url)
                         else:
                             url = items[i]["videoInfo"]["videoCdnInfo"]["mpInfo"]["urlInfo"][0]["url"]
-                            Common.logger().info('视频播放地址：{}', url)
+                            Common.logger(log_type).info('视频播放地址：{}', url)
                     elif "ctnInfo" in items[i]["videoInfo"]["videoCdnInfo"]:
                         url = items[i]["videoInfo"]["videoCdnInfo"]["ctnInfo"]["urlInfo"][0]["url"]
-                        Common.logger().info('视频播放地址：{}', url)
+                        Common.logger(log_type).info('视频播放地址：{}', url)
                     else:
                         url = items[i]["videoInfo"]["videoCdnInfo"]["urlInfo"][0]["url"]
-                        Common.logger().info('视频播放地址：{}', url)
+                        Common.logger(log_type).info('视频播放地址：{}', url)
 
                     # 过滤无效视频
                     if video_id == "" \
@@ -202,25 +202,26 @@ def get_feeds():
                             or video_user_cover == "" \
                             or video_cover == "" \
                             or url == "":
-                        Common.logger().info("无效视频")
+                        Common.logger(log_type).info("无效视频")
 
                     # 基础门槛，播放量>=20000
                     elif int(video_play_cnt) < 20000:
-                        Common.logger().info("播放量{} < 20000", video_play_cnt)
+                        Common.logger(log_type).info("播放量{} < 20000", video_play_cnt)
                     # 过滤敏感词
-                    elif any(word if word in video_title else False for word in kanyikan_sensitive_words()) is True:
-                        Common.logger().info("视频已中敏感词：{}".format(video_title))
+                    elif any(word if word in video_title else False
+                             for word in kanyikan_sensitive_words(log_type)) is True:
+                        Common.logger(log_type).info("视频已中敏感词：{}".format(video_title))
                     # 从 云文档 去重:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=20ce0c
-                    elif video_id in [j for i in Feishu.get_values_batch("20ce0c") for j in i]:
-                        Common.logger().info("该视频已下载：{}", video_title)
+                    elif video_id in [j for i in Feishu.get_values_batch(log_type, "kanyikan", "20ce0c") for j in i]:
+                        Common.logger(log_type).info("该视频已下载：{}", video_title)
                     # 从 云文档 去重:https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?sheet=SdCHOM
-                    elif video_id in [j for i in Feishu.get_values_batch("SdCHOM") for j in i]:
-                        Common.logger().info("该视频已在kanyikan_feeds中:{}", video_title)
+                    elif video_id in [j for i in Feishu.get_values_batch(log_type, "kanyikan", "SdCHOM") for j in i]:
+                        Common.logger(log_type).info("该视频已在kanyikan_feeds中:{}", video_title)
                     else:
-                        Common.logger().info("该视频未下载，添加至kanyikan_feeds:{}", video_title)
+                        Common.logger(log_type).info("该视频未下载，添加至kanyikan_feeds:{}", video_title)
 
                         # 看一看+工作表，插入首行
-                        Feishu.insert_columns("SdCHOM", "ROWS", 1, 2)
+                        Feishu.insert_columns(log_type, "kanyikan", "SdCHOM", "ROWS", 1, 2)
 
                         # 获取当前时间
                         get_feeds_time = int(time.time())
@@ -243,10 +244,10 @@ def get_feeds():
                                    url]]
                         time.sleep(1)
                         # 写入数据
-                        Feishu.update_values("SdCHOM", "A2:P2", values)
+                        Feishu.update_values(log_type, "kanyikan", "SdCHOM", "A2:P2", values)
     except Exception as e:
-        Common.logger().error("获取视频 list 时异常：{}", e)
+        Common.logger(log_type).error("获取视频 list 时异常：{}", e)
 
 
 if __name__ == "__main__":
-    get_feeds()
+    get_feeds("recommend")

main/publish.py

@@ -20,7 +20,7 @@ proxies = {"http": None, "https": None}
 
 class Publish:
     @classmethod
-    def publish_video_dev(cls, request_data):
+    def publish_video_dev(cls, log_type, request_data):
         """
         loginUid  站内uid (随机)
         appType  默认：888888
@@ -37,14 +37,16 @@ class Publish:
         :return:
         """
         result = cls.request_post('https://videotest.yishihui.com/longvideoapi/crawler/video/send', request_data)
-        Common.logger().info('publish result: {}', result)
+        # Common.logger(log_type).info('publish result: {}', result)
+        video_id = result["data"]["id"]
         if result['code'] != 0:
-            Common.logger().error('pushlish failure msg = {}', result['msg'])
+            Common.logger(log_type).error('pushlish failure msg = {}', result['msg'])
         else:
-            Common.logger().info('publish success video_id = : {}', request_data['crawlerSrcId'])
+            Common.logger(log_type).info('publish success video_id = : {}', request_data['crawlerSrcId'])
+        return video_id
 
     @classmethod
-    def publish_video_prod(cls, request_data):
+    def publish_video_prod(cls, log_type, request_data):
         """
         loginUid  站内uid (随机)
         appType  默认：888888
@@ -61,11 +63,13 @@ class Publish:
         :return:
         """
         result = cls.request_post('https://longvideoapi.piaoquantv.com/longvideoapi/crawler/video/send', request_data)
-        Common.logger().info('publish result: {}', result)
+        # Common.logger(log_type).info('publish result: {}', result)
+        video_id = result["data"]["id"]
         if result['code'] != 0:
-            Common.logger().error('pushlish failure msg = {}', result['msg'])
+            Common.logger(log_type).error('pushlish failure msg = {}', result['msg'])
         else:
-            Common.logger().info('publish success video_id = : {}', request_data['crawlerSrcId'])
+            Common.logger(log_type).info('publish success video_id = : {}', request_data['crawlerSrcId'])
+        return video_id
 
     @classmethod
     def request_post(cls, request_url, request_data):
@@ -117,21 +121,21 @@ class Publish:
     oss_file_path_image = r'longvideo/crawler_local/image/{}/{}/{}'
 
     @classmethod
-    def put_file(cls, oss_file, local_file):
+    def put_file(cls, log_type, oss_file, local_file):
         cls.bucket.put_object_from_file(oss_file, local_file)
-        Common.logger().info("put oss file = {}, local file = {} success", oss_file, local_file)
+        Common.logger(log_type).info("put oss file = {}, local file = {} success", oss_file, local_file)
 
     # 清除本地文件
     @classmethod
-    def remove_local_file(cls, local_file):
+    def remove_local_file(cls, log_type, local_file):
         os.remove(local_file)
-        Common.logger().info("remove local file = {} success", local_file)
+        Common.logger(log_type).info("remove local file = {} success", local_file)
 
     # 清除本地文件夹
     @classmethod
-    def remove_local_file_dir(cls, local_file):
+    def remove_local_file_dir(cls, log_type, local_file):
         os.rmdir(local_file)
-        Common.logger().info("remove local file dir = {} success", local_file)
+        Common.logger(log_type).info("remove local file dir = {} success", local_file)
 
     local_file_path = '.\\videos'
     video_file = 'video'
@@ -150,13 +154,14 @@ class Publish:
                            20631223, 20631224, 20631225, 20631226, 20631227]
 
     @classmethod
-    def upload_and_publish(cls, env, job):
+    def upload_and_publish(cls, log_type, env, job):
         """
         上传视频到 oss
+        :param log_type: 选择的 log
         :param env: 测试环境：dev，正式环境：prod
         :param job: 上升榜：up，播放量：play， send_time：发布时间榜
         """
-        Common.logger().info("upload_and_publish starting...")
+        Common.logger(log_type).info("upload_and_publish starting...")
         today = time.strftime("%Y%m%d", time.localtime())
         # videos 目录下的所有视频文件夹
         files = os.listdir(cls.local_file_path)
@@ -166,7 +171,7 @@ class Publish:
                 fi_d = os.path.join(cls.local_file_path, f)
                 # 确认为视频文件夹
                 if os.path.isdir(fi_d):
-                    Common.logger().info('dir = {}', fi_d)
+                    Common.logger(log_type).info('dir = {}', fi_d)
                     # 列出所有视频文件夹
                     dir_files = os.listdir(fi_d)
                     data = {'appType': '888888', 'crawlerSrcCode': 'KANYIKAN', 'viewStatus': '1', 'versionCode': '1'}
@@ -190,7 +195,7 @@ class Publish:
                     for fi in dir_files:
                         # 视频文件夹下的所有文件路径
                         fi_path = fi_d + '\\' + fi
-                        Common.logger().info('dir fi_path = {}', fi_path)
+                        Common.logger(log_type).info('dir fi_path = {}', fi_path)
                         # 读取 info.txt，赋值给 data
                         if cls.info_file in fi:
                             f = open(fi_path, "r", encoding="UTF-8")
@@ -199,7 +204,7 @@ class Publish:
                                 line = f.readline()
                                 line = line.replace('\n', '')
                                 if line is not None and len(line) != 0 and not line.isspace():
-                                    Common.logger().info("line = {}", line)
+                                    Common.logger(log_type).info("line = {}", line)
                                     if i == 0:
                                         data['crawlerSrcId'] = line
                                     elif i == 1:
@@ -209,15 +214,15 @@ class Publish:
                                     elif i == 8:
                                         data['crawlerSrcPublishTimestamp'] = line
                                 else:
-                                    Common.logger().warning("{} line is None", fi_path)
+                                    Common.logger(log_type).warning("{} line is None", fi_path)
                             f.close()
                             # remove info.txt
-                            cls.remove_local_file(fi_path)
+                            cls.remove_local_file(log_type, fi_path)
                     # 刷新数据
                     dir_files = os.listdir(fi_d)
                     for fi in dir_files:
                         fi_path = fi_d + '\\' + fi
-                        Common.logger().info('dir fi_path = {}', fi_path)
+                        Common.logger(log_type).info('dir fi_path = {}', fi_path)
                         # 上传oss
                         if cls.video_file in fi:
                             global oss_video_file
@@ -225,31 +230,34 @@ class Publish:
                                 oss_video_file = cls.oss_file_path_video.format("dev", today, data['crawlerSrcId'])
                             elif env == "prod":
                                 oss_video_file = cls.oss_file_path_video.format("prod", today, data['crawlerSrcId'])
-                            Common.logger().info("oss_video_file = {}", oss_video_file)
-                            cls.put_file(oss_video_file, fi_path)
+                            Common.logger(log_type).info("oss_video_file = {}", oss_video_file)
+                            cls.put_file(log_type, oss_video_file, fi_path)
                             data['videoPath'] = oss_video_file
-                            Common.logger().info("videoPath = {}", oss_video_file)
+                            Common.logger(log_type).info("videoPath = {}", oss_video_file)
                         elif cls.image_file in fi:
                             global oss_image_file
                             if env == "dev":
                                 oss_image_file = cls.oss_file_path_image.format("env", today, data['crawlerSrcId'])
                             elif env == "prod":
                                 oss_image_file = cls.oss_file_path_image.format("prod", today, data['crawlerSrcId'])
-                            Common.logger().info("oss_image_file = {}", oss_image_file)
-                            cls.put_file(oss_image_file, fi_path)
+                            Common.logger(log_type).info("oss_image_file = {}", oss_image_file)
+                            cls.put_file(log_type, oss_image_file, fi_path)
                             data['coverImgPath'] = oss_image_file
-                            Common.logger().info("coverImgPath = {}", oss_image_file)
+                            Common.logger(log_type).info("coverImgPath = {}", oss_image_file)
                         # 全部remove
-                        cls.remove_local_file(fi_path)
+                        cls.remove_local_file(log_type, fi_path)
 
                     # 发布
                     if env == "dev":
-                        cls.publish_video_dev(data)
+                        video_id = cls.publish_video_dev(log_type, data)
                     elif env == "prod":
-                        cls.publish_video_prod(data)
-                    cls.remove_local_file_dir(fi_d)
+                        video_id = cls.publish_video_prod(log_type, data)
+                    else:
+                        video_id = cls.publish_video_dev(log_type, data)
+                    cls.remove_local_file_dir(log_type, fi_d)
+                    return video_id
 
                 else:
-                    Common.logger().error('file not a dir = {}', fi_d)
+                    Common.logger(log_type).error('file not a dir = {}', fi_d)
             except Exception as e:
-                Common.logger().exception('upload_and_publish error', e)
+                Common.logger(log_type).exception('upload_and_publish error', e)

main/run.py

@@ -25,20 +25,20 @@ class Main:
             play_now = datetime.datetime.now()
             # 指定时间结束抓取视频
             if play_now.hour == 0:
-                Common.logger().info("结束抓取播放量视频\n")
+                Common.logger("recommend").info("结束抓取播放量视频\n")
                 time.sleep(3)
                 break
             else:
                 # 获取视频信息，并下载
-                get_feeds()
-                DownloadPlay.run_download_play_video()
+                get_feeds("recommend")
+                DownloadPlay.run_download_play_video("prod")
                 # 请求随机间隔时间
                 time.sleep(random.randint(31, 40))
 
         # 删除 charles 缓存文件
-        Common.del_charles_files()
+        Common.del_charles_files("recommend")
         # 删除多余日志
-        Common.del_logs()
+        Common.del_logs("recommend")
 
     @classmethod
     def download_up_job(cls):
@@ -49,20 +49,20 @@ class Main:
             up_now = datetime.datetime.now()
             # 指定时间结束抓取视频
             if up_now.hour > 19:
-                Common.logger().info("结束抓取上升榜视频\n")
+                Common.logger("recommend").info("结束抓取上升榜视频\n")
                 time.sleep(3)
                 break
             else:
                 # 获取视频信息，并下载
-                get_feeds()
+                get_feeds("recommend")
                 DownloadUp.download_up_video("prod")
                 # 请求随机间隔时间
                 time.sleep(random.randint(31, 40))
 
         # 删除 charles 缓存文件
-        Common.del_charles_files()
+        Common.del_charles_files("recommend")
         # 删除多余日志
-        Common.del_logs()
+        Common.del_logs("recommend")
 
     @classmethod
     def download_sendtime_job(cls):
@@ -73,20 +73,20 @@ class Main:
             sendtime_now = datetime.datetime.now()
             # 指定时间结束抓取视频
             if sendtime_now.hour > 14:
-                Common.logger().info("结束抓取7天榜单视频\n")
+                Common.logger("recommend").info("结束抓取7天榜单视频\n")
                 time.sleep(3)
                 break
             else:
                 # 获取视频信息，并下载
-                get_feeds()
+                get_feeds("recommend")
                 DownloadSendtime.run_download_sendtime_video()
                 # 请求随机间隔时间
                 time.sleep(random.randint(31, 40))
 
         # 删除 charles 缓存文件
-        Common.del_charles_files()
+        Common.del_charles_files("recommend")
         # 删除多余日志
-        Common.del_logs()
+        Common.del_logs("recommend")
 
     @classmethod
     def main(cls):
@@ -99,7 +99,7 @@ class Main:
             while True:
                 main_now = datetime.datetime.now()
                 if 14 >= main_now.hour >= 0:
-                    Common.logger().info("开始抓取7天榜单视频\n")
+                    Common.logger("recommend").info("开始抓取7天榜单视频\n")
                     time.sleep(1)
                     cls.download_sendtime_job()
                 # elif 19 >= main_now.hour >= 10:
@@ -107,11 +107,11 @@ class Main:
                 #     time.sleep(1)
                 #     cls.download_up_job()
                 elif 24 >= main_now.hour >= 15:
-                    Common.logger().info("开始抓取播放量视频\n")
+                    Common.logger("recommend").info("开始抓取播放量视频\n")
                     time.sleep(1)
                     cls.download_play_job()
                 else:
-                    Common.logger().info("结束今天抓取及上传任务\n")
+                    Common.logger("recommend").info("结束今天抓取及上传任务\n")
                     time.sleep(60)
                     break
 

main/run_recommend.py

@@ -17,12 +17,12 @@ class Main:
         while True:
             prod_job_time = datetime.datetime.now()
             if prod_job_time.hour == 0 and prod_job_time.minute <= 10:
-                Common.del_logs()
-                Common.del_charles_files()
+                Common.del_logs("recommend")
+                Common.del_charles_files("recommend")
                 time.sleep(60)
             else:
-                get_feeds()
-                DownloadPlaySendtime.run_download_play_sendtime()
+                get_feeds("recommend")
+                DownloadPlaySendtime.run_download_play_sendtime("prod")
 
 
 if __name__ == "__main__":