update

README.md

@@ -26,6 +26,9 @@ https://git.yishihui.com/Server/crawler_bot.git
 
 
 #### 需求
+2022/10/19
+1. 小年糕3种爬取方式分别报警
+
 2022/9/13
 1. 增加监控渠道: 视频号
 

main/common.py

@@ -39,8 +39,6 @@ class Common:
         # 日志文件名
         if log_type == "bot":
             log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '-bot.log'
-        elif log_type == "bot":
-            log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '-bot.log'
         else:
             log_name = time.strftime("%Y-%m-%d", time.localtime(time.time())) + '-bot.log'
 

main/feishu_lib.py

@@ -15,7 +15,6 @@ class Feishu:
     # 看一看爬虫数据表
     kanyikan_url = "https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih?"
     # 快手爬虫数据表
-    # kuaishou_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnp4SaJt37q6OOOrYzPMjQkg?"
     kuaishou_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnICEfaw9llDNQkKgdymM1xf?"
     # 微视爬虫数据表
     weishi_url = "https://w42nne6hzg.feishu.cn/sheets/shtcn5YSWg91JfVGzj0SFZIRRPh?"
@@ -49,7 +48,6 @@ class Feishu:
         if crawler == "kanyikan":
             return "shtcngRPoDYAi24x52j2nDuHMih"
         elif crawler == "kuaishou":
-            # return "shtcnp4SaJt37q6OOOrYzPMjQkg"
             return "shtcnICEfaw9llDNQkKgdymM1xf"
         elif crawler == "weishi":
             return "shtcn5YSWg91JfVGzj0SFZIRRPh"
@@ -361,9 +359,17 @@ class Feishu:
             if crawler == "kanyikan":
                 content = "看一看爬虫表"
                 sheet_url = "https://w42nne6hzg.feishu.cn/sheets/shtcngRPoDYAi24x52j2nDuHMih"
-            elif crawler == "xiaoniangao":
-                content = "小年糕爬虫表"
-                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh"
+
+            elif crawler == "xiaoniangao_hour":
+                content = "小年糕_小时级_已下载表"
+                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=yatRv2"
+            elif crawler == "xiaoniangao_person":
+                content = "小年糕_用户主页_已下载表"
+                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=Wu0CeL"
+            elif crawler == "xiaoniangao_play":
+                content = "小年糕_播放量_已下载表"
+                sheet_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnYxiyQ1wLklo1W5Kdqc9cGh?sheet=c85k1C"
+
             elif crawler == "music_album":
                 content = "音乐相册爬虫表"
                 sheet_url = "https://w42nne6hzg.feishu.cn/sheets/shtcnT6zvmfsYe1g0iv4pt7855g"
@@ -431,5 +437,8 @@ class Feishu:
 
 
 if __name__ == "__main__":
-    Feishu.bot("bot", "kuaishou", "我是快手测试内容，请忽略")
+    Feishu.bot("bot", "xiaoniangao_hour", "嘿。嘿嘿。。嘿嘿嘿")
     # Feishu.get_userid("kuaishou", "huxinxue")
+    
+    pass
+

main/run_bot.py

@@ -6,7 +6,6 @@ import datetime
 import os
 import sys
 import time
-
 sys.path.append(os.getcwd())
 from main.common import Common
 from main.feishu_lib import Feishu
@@ -17,60 +16,70 @@ class Bot:
     @classmethod
     def get_first_time(cls, log_type, crawler):
         try:
-            if crawler == "xiaoniangao":
+            if crawler == "xiaoniangao_hour":
                 sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][5]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+            elif crawler == "xiaoniangao_person":
+                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "Wu0CeL")
+                # 已下载表，最新一条视频抓取时间
+                first_download_time = sheet[1][5]
+                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+            elif crawler == "xiaoniangao_play":
+                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "c85k1C")
+                # 已下载表，最新一条视频抓取时间
+                first_download_time = sheet[1][5]
+                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             elif crawler == "kanyikan":
                 sheet = Feishu.get_values_batch(log_type, "kanyikan", "20ce0c")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][5]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             elif crawler == "music_album":
                 sheet = Feishu.get_values_batch(log_type, "music_album", "f5a76e")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][5]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             elif crawler == "bszf":
                 sheet = Feishu.get_values_batch(log_type, "bszf", "440018")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][4]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             elif crawler == "kuaishou":
                 sheet = Feishu.get_values_batch(log_type, "kuaishou", "3cd128")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][5]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             elif crawler == "gzh":
                 sheet = Feishu.get_values_batch(log_type, "gzh", "fCs3BT")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][3]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             elif crawler == "weiqun":
                 sheet = Feishu.get_values_batch(log_type, "weiqun", "3cd128")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][5]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             elif crawler == "weishi":
                 sheet = Feishu.get_values_batch(log_type, "weishi", "caa3fa")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][5]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             elif crawler == "shipinhao":
                 sheet = Feishu.get_values_batch(log_type, "shipinhao", "c77cf9")
-                # print(sheet[1])
                 # 已下载表，最新一条视频抓取时间
                 first_download_time = sheet[1][5]
                 first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
+
             else:
                 sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2")
                 # 已下载表，最新一条视频抓取时间
@@ -79,7 +88,7 @@ class Bot:
 
             return first_download_time
         except Exception as e:
-            Common.logger(log_type).error("get_first_time异常:{}", e)
+            Common.logger(log_type).error("get_first_time异常:{}\n", e)
 
     # 获取各个爬虫的 feeds 表
     @classmethod
@@ -163,49 +172,57 @@ class Bot:
             # 看一看爬虫报警
             if crawler == "kanyikan" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                 Feishu.bot(log_type, crawler, "看一看已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("看一看已下载表，超过24小时没有新视频入库了😤\n")
+                Common.logger(log_type).warning("看一看已下载表，超过24小时没有新视频入库了😤\n")
 
             # 小年糕爬虫报警
-            elif crawler == "xiaoniangao" and (
+            elif crawler == "xiaoniangao_hour" and (
+                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
+                Feishu.bot(log_type, crawler, "小年糕_小时级_已下载表，超过24小时没有新视频入库了😤")
+                Common.logger(log_type).warning("小年糕_小时级_已下载表，超过24小时没有新视频入库了😤\n")
+            elif crawler == "xiaoniangao_person" and (
+                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
+                Feishu.bot(log_type, crawler, "小年糕_用户主页_已下载表，超过24小时没有新视频入库了😤")
+                Common.logger(log_type).warning("小年糕_用户主页_已下载表，超过24小时没有新视频入库了😤\n")
+            elif crawler == "xiaoniangao_play" and (
                     int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
-                Feishu.bot(log_type, crawler, "小年糕已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("小年糕已下载表，超过24小时没有新视频入库了😤\n")
+                Feishu.bot(log_type, crawler, "小年糕_播放量_已下载表，超过24小时没有新视频入库了😤")
+                Common.logger(log_type).warning("小年糕_播放量_已下载表，超过24小时没有新视频入库了😤\n")
 
             # 音乐相册爬虫报警
             elif crawler == "music_album" and (
                     int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                 Feishu.bot(log_type, crawler, "音乐相册已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("音乐相册已下载表，超过24小时没有新视频入库了😤\n")
+                Common.logger(log_type).warning("音乐相册已下载表，超过24小时没有新视频入库了😤\n")
 
             # 本山祝福爬虫报警
             elif crawler == "bszf" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                 Feishu.bot(log_type, crawler, "本山祝福已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("本山祝福已下载表，超过24小时没有新视频入库了😤\n")
+                Common.logger(log_type).warning("本山祝福已下载表，超过24小时没有新视频入库了😤\n")
 
             # 快手爬虫报警
             elif crawler == "kuaishou" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                 Feishu.bot(log_type, crawler, "快手已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("快手已下载表，超过24小时没有新视频入库了😤\n")
+                Common.logger(log_type).warning("快手已下载表，超过24小时没有新视频入库了😤\n")
 
             # 公众号爬虫报警
             elif crawler == "gzh" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                 Feishu.bot(log_type, crawler, "公众号已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("公众号已下载表，超过24小时没有新视频入库了😤\n")
+                Common.logger(log_type).warning("公众号已下载表，超过24小时没有新视频入库了😤\n")
 
             # 微群视频爬虫报警
             elif crawler == "weiqun" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                 Feishu.bot(log_type, crawler, "微群视频已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("微群视频已下载表，超过24小时没有新视频入库了😤\n")
+                Common.logger(log_type).warning("微群视频已下载表，超过24小时没有新视频入库了😤\n")
 
             # 微视爬虫报警
             elif crawler == "weishi" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                 Feishu.bot(log_type, crawler, "微视已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("微视已下载表，超过24小时没有新视频入库了😤\n")
+                Common.logger(log_type).warning("微视已下载表，超过24小时没有新视频入库了😤\n")
 
             # 视频号爬虫报警
             elif crawler == "shipinhao" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                 Feishu.bot(log_type, crawler, "视频号已下载表，超过24小时没有新视频入库了😤")
-                Common.logger(log_type).info("视频号已下载表，超过24小时没有新视频入库了😤\n")
+                Common.logger(log_type).warning("视频号已下载表，超过24小时没有新视频入库了😤\n")
 
         except Exception as e:
             Common.logger(log_type).error("robot_alarm异常:{}", e)
@@ -220,14 +237,15 @@ class Bot:
         # 已下载表，超过 24 小时无新视频入库报警
         duration = 3600 * 24
         while True:
-            # if 21 > datetime.datetime.now().hour >= 10:
             if datetime.datetime.now().hour == 10:
 
                 Common.logger("bot").info("监控看一看已下载表")
                 Bot.robot_download_sheet("bot", "kanyikan", duration)
 
                 Common.logger("bot").info("监控小年糕已下载表")
-                Bot.robot_download_sheet("bot", "xiaoniangao", duration)
+                Bot.robot_download_sheet("bot", "xiaoniangao_hour", duration)
+                Bot.robot_download_sheet("bot", "xiaoniangao_person", duration)
+                Bot.robot_download_sheet("bot", "xiaoniangao_play", duration)
 
                 Common.logger("bot").info("监控本山祝福已下载表")
                 Bot.robot_download_sheet("bot", "bszf", duration)
@@ -238,15 +256,15 @@ class Bot:
                 Common.logger("bot").info("监控微视已下载表")
                 Bot.robot_download_sheet("bot", "weishi", duration)
 
+                Common.logger("bot").info("监控视频号已下载表")
+                Bot.robot_download_sheet("bot", "shipinhao", duration)
+
                 # Common.logger("bot").info("监控公众号已下载表")
                 # Bot.robot_download_sheet("bot", "gzh", duration)
 
                 # Common.logger("bot").info("监控音乐相册已下载表")
                 # Bot.robot_download_sheet("bot", "music_album", duration)
 
-                Common.logger("bot").info("监控视频号已下载表")
-                Bot.robot_download_sheet("bot", "shipinhao", duration)
-
                 # Common.logger("bot").info("监控微群视频已下载表")
                 # Bot.robot_download_sheet("bot", "weiqun", duration)
 
@@ -255,8 +273,6 @@ class Bot:
                 time.sleep(3600 * (24-datetime.datetime.now().hour))
             else:
                 pass
-                # Common.logger("bot").info("今日监控完毕\n")
-                # time.sleep(3600 * (24-datetime.datetime.now().hour))
 
 
 if __name__ == "__main__":