crawler_bot/main/run_bot.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/8/9
# import datetime
import datetime
import os
import sys
import time
sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu


class Bot:

    # 获取各个爬虫表最新一条抓取时间
    @classmethod
    def get_first_time(cls, log_type, crawler):
        try:
            if crawler == "xiaoniangao_hour":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == "xiaoniangao_person":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "Wu0CeL")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == "xiaoniangao_play":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "c85k1C")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'xigua_video':
                sheet = Feishu.get_values_batch(log_type, "xigua", "e075e9")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == 'xigua_little_video':
                sheet = Feishu.get_values_batch(log_type, "xigua", "hDSDnv")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'zhihu_hot':
                sheet = Feishu.get_values_batch(log_type, "zhihu", "8871e3")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == 'zhihu_follow':
                sheet = Feishu.get_values_batch(log_type, "zhihu", "4MGuux")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'jxxf':
                sheet = Feishu.get_values_batch(log_type, "jxxf", "d9e9b1")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'zmyx':
                sheet = Feishu.get_values_batch(log_type, "zmyx", "19c772")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'zhufumao':
                sheet = Feishu.get_values_batch(log_type, "zhufumao", "e13bdf")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'ssnnyfq':
                sheet = Feishu.get_values_batch(log_type, "ssnnyfq", "290bae")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "kanyikan":
                sheet = Feishu.get_values_batch(log_type, "kanyikan", "20ce0c")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "music_album":
                sheet = Feishu.get_values_batch(log_type, "music_album", "f5a76e")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "bszf":
                sheet = Feishu.get_values_batch(log_type, "bszf", "440018")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][4]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "kuaishou_recommend":
                sheet = Feishu.get_values_batch(log_type, "kuaishou", "3cd128")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == "kuaishou_follow":
                sheet = Feishu.get_values_batch(log_type, "kuaishou", "fYdA8F")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "gzh":
                sheet = Feishu.get_values_batch(log_type, "gzh", "fCs3BT")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][3]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "weiqun":
                sheet = Feishu.get_values_batch(log_type, "weiqun", "3cd128")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "weishi":
                sheet = Feishu.get_values_batch(log_type, "weishi", "caa3fa")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "shipinhao":
                sheet = Feishu.get_values_batch(log_type, "shipinhao", "c77cf9")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            else:
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            return first_download_time
        except Exception as e:
            Common.logger(log_type).error(f"get_first_time异常:{e}\n")

    # 触发机器人报警:超过24小时没有新入库的视频
    @classmethod
    def robot_download_sheet(cls, log_type, crawler, duration):
        """
        已下载视频表:超过24小时没有新入库的视频
        """
        try:
            # 看一看爬虫报警
            if crawler == "kanyikan" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "看一看已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("看一看已下载表，超过24小时没有新视频入库了😤\n")

            # 小年糕爬虫报警
            elif crawler == "xiaoniangao_hour" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_小时级_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_小时级_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "xiaoniangao_person" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_用户主页_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_用户主页_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "xiaoniangao_play" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_播放量_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_播放量_已下载表，超过24小时没有新视频入库了😤\n")

            # 西瓜视频
            elif crawler == "xigua_video" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "西瓜视频_用户主页_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("西瓜视频_用户主页_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "xigua_little_video" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "西瓜视频_小视频_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("西瓜视频_小视频_已下载表，超过24小时没有新视频入库了😤\n")

            # 知乎
            elif crawler == "zhihu_hot" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "知乎_热门_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("知乎_热门_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "zhihu_follow" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "知乎_定向_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("知乎_定向_已下载表，超过24小时没有新视频入库了😤\n")

            # 吉祥幸福
            elif crawler == "jxxf" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "吉祥幸福_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("吉祥幸福_已下载表，超过24小时没有新视频入库了😤\n")

            # 众妙音信
            elif crawler == "zmyx" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "众妙音信_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("众妙音信_已下载表，超过24小时没有新视频入库了😤\n")

            # 祝福猫视频
            elif crawler == "zhufumao" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "祝福猫_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("祝福猫_已下载表，超过24小时没有新视频入库了😤\n")

            # 岁岁年年迎福气
            elif crawler == "ssnnyfq" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "岁岁年年迎福气_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("岁岁年年迎福气_已下载表，超过24小时没有新视频入库了😤\n")

            # 音乐相册爬虫报警
            elif crawler == "music_album" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "音乐相册已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("音乐相册已下载表，超过24小时没有新视频入库了😤\n")

            # 本山祝福爬虫报警
            elif crawler == "bszf" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "本山祝福已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("本山祝福已下载表，超过24小时没有新视频入库了😤\n")

            # 快手爬虫报警
            elif crawler == "kuaishou_recommend" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "快手_推荐榜_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("快手_推荐榜_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "kuaishou_follow" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "快手_用户主页_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("快手_用户主页_已下载表，超过24小时没有新视频入库了😤\n")

            # 公众号爬虫报警
            elif crawler == "gzh" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "公众号已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("公众号已下载表，超过24小时没有新视频入库了😤\n")

            # 微群视频爬虫报警
            elif crawler == "weiqun" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "微群视频已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("微群视频已下载表，超过24小时没有新视频入库了😤\n")

            # 微视爬虫报警
            elif crawler == "weishi" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "微视已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("微视已下载表，超过24小时没有新视频入库了😤\n")

            # 视频号爬虫报警
            elif crawler == "shipinhao" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "视频号已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("视频号已下载表，超过24小时没有新视频入库了😤\n")

        except Exception as e:
            Common.logger(log_type).error(f"robot_alarm异常:{e}\n")

    # 监控运行入口
    @classmethod
    def main(cls):
        """
        每隔六小时，检查一次已下载表；
        已下载表的最新一条数据抓取时间，距当前时间超过 24 小时，则触发机器人报警，发送飞书报警消息
        """
        # 已下载表，超过 24 小时无新视频入库报警
        duration = 3600 * 24
        while True:
            if datetime.datetime.now().hour == 10:

                Common.logger("bot").info("监控看一看已下载表")
                Bot.robot_download_sheet("bot", "kanyikan", duration)

                Common.logger("bot").info("监控小年糕已下载表")
                Bot.robot_download_sheet("bot", "xiaoniangao_hour", duration)
                Bot.robot_download_sheet("bot", "xiaoniangao_person", duration)
                Bot.robot_download_sheet("bot", "xiaoniangao_play", duration)

                # Common.logger("bot").info("监控西瓜视频已下载表")
                # Bot.robot_download_sheet("bot", "xigua_video", duration)
                # Bot.robot_download_sheet("bot", "xigua_little_video", duration)

                Common.logger('bot').info('监控知乎已下载表')
                Bot.robot_download_sheet("bot", "zhihu_hot", duration)
                Bot.robot_download_sheet("bot", "zhihu_follow", duration)

                Common.logger("bot").info("监控吉祥幸福已下载表")
                Bot.robot_download_sheet("bot", "jxxf", duration)

                Common.logger("bot").info("监控众妙音信已下载表")
                Bot.robot_download_sheet("bot", "zmyx", duration)

                Common.logger("bot").info("监控祝福猫已下载表")
                Bot.robot_download_sheet("bot", "zhufumao", duration)

                Common.logger("bot").info("监控岁岁年年迎福气已下载表")
                Bot.robot_download_sheet("bot", "ssnnyfq", duration)

                Common.logger("bot").info("监控本山祝福已下载表")
                Bot.robot_download_sheet("bot", "bszf", duration)

                Common.logger("bot").info("监控快手已下载表")
                Bot.robot_download_sheet("bot", "kuaishou_recommend", duration)
                Bot.robot_download_sheet("bot", "kuaishou_follow", duration)

                Common.logger("bot").info("监控微视已下载表")
                Bot.robot_download_sheet("bot", "weishi", duration)

                Common.logger("bot").info("监控视频号已下载表")
                Bot.robot_download_sheet("bot", "shipinhao", duration)

                # Common.logger("bot").info("监控公众号已下载表")
                # Bot.robot_download_sheet("bot", "gzh", duration)

                # Common.logger("bot").info("监控音乐相册已下载表")
                # Bot.robot_download_sheet("bot", "music_album", duration)

                # Common.logger("bot").info("监控微群视频已下载表")
                # Bot.robot_download_sheet("bot", "weiqun", duration)

                Common.del_logs("bot")
                Common.logger("bot").info(f"休眠{24-datetime.datetime.now().hour}小时")
                time.sleep(3600 * (24-datetime.datetime.now().hour))
            else:
                pass


if __name__ == "__main__":

    # Bot.robot_download_sheet("bot", "zhufumao", 1)
    Bot.main()

    pass