crawler_bot/main/run_bot.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/8/9
# import datetime
import datetime
import os
import sys
import time
sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu


class Bot:
    # 获取各个爬虫表最新一条抓取时间
    @classmethod
    def get_first_time(cls, log_type, crawler):
        try:
            if crawler == "xiaoniangao_hour":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == "xiaoniangao_person":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "Wu0CeL")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == "xiaoniangao_play":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "c85k1C")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "kanyikan":
                sheet = Feishu.get_values_batch(log_type, "kanyikan", "20ce0c")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "music_album":
                sheet = Feishu.get_values_batch(log_type, "music_album", "f5a76e")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "bszf":
                sheet = Feishu.get_values_batch(log_type, "bszf", "440018")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][4]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "kuaishou":
                sheet = Feishu.get_values_batch(log_type, "kuaishou", "3cd128")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "gzh":
                sheet = Feishu.get_values_batch(log_type, "gzh", "fCs3BT")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][3]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "weiqun":
                sheet = Feishu.get_values_batch(log_type, "weiqun", "3cd128")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "weishi":
                sheet = Feishu.get_values_batch(log_type, "weishi", "caa3fa")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "shipinhao":
                sheet = Feishu.get_values_batch(log_type, "shipinhao", "c77cf9")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            else:
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            return first_download_time
        except Exception as e:
            Common.logger(log_type).error("get_first_time异常:{}\n", e)

    # 获取各个爬虫的 feeds 表
    @classmethod
    def get_feeds_sheet(cls, log_type, crawler, sheet):
        try:
            if crawler == "kanyikan" and sheet == "recommend":
                feeds_sheet = Feishu.get_values_batch(log_type, "kanyikan", "SdCHOM")
            elif crawler == "kanyikan" and sheet == "moment":
                feeds_sheet = Feishu.get_values_batch(log_type, "kanyikan", "tGqZMX")
            elif crawler == "xiaoniangao" and sheet == "hour":
                feeds_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4")
            elif crawler == "xiaoniangao" and sheet == "person":
                feeds_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "k6ldje")
            elif crawler == "music_album" and sheet == "recommend":
                feeds_sheet = Feishu.get_values_batch(log_type, "music_album", "69UxPo")
            elif crawler == "bszf" and sheet == "recommend":
                feeds_sheet = Feishu.get_values_batch(log_type, "bszf", "CcHgO7")
            elif crawler == "kuaishou" and sheet == "recommend":
                feeds_sheet = Feishu.get_values_batch(log_type, "kuaishou", "JK6npf")
            elif crawler == "kuaishou" and sheet == "follow":
                feeds_sheet = Feishu.get_values_batch(log_type, "kuaishou", "wW5cyb")
            elif crawler == "gzh" and sheet == "recommend":
                feeds_sheet = Feishu.get_values_batch(log_type, "gzh", "zWKFGb")
            elif crawler == "weishi" and sheet == "recommend":
                feeds_sheet = Feishu.get_values_batch(log_type, "weishi", "O7fCzr")
            else:
                feeds_sheet = "请输入{crawler}和{sheet}"

            return feeds_sheet
        except Exception as e:
            Common.logger(log_type).error("get_feeds_sheet异常:{}", e)

    # feeds_sheet表报警:连续 2 小时无数据
    @classmethod
    def rebot_feeds_sheet(cls, log_type, crawler, sheet):
        """
        每隔一分钟获取一次表数据的数量：
            1.中途有数据时，退出此次监控
            2.连续2小时无数据时，触发机器人报警
        """
        # kanyikan_recommend_sheet = Feishu.get_values_batch(log_type, "kanyikan", "SdCHOM")
        # kanyikan_moment_sheet = Feishu.get_values_batch(log_type, "kanyikan", "tGqZMX")
        # xiaoniangao_hour_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4")
        # xiaoniangao_person_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "k6ldje")
        # music_album_recommend_sheet = Feishu.get_values_batch(log_type, "music_album", "69UxPo")
        # bszf_recommend_sheet = Feishu.get_values_batch(log_type, "bszf", "CcHgO7")
        # kuaishou_recommend_sheet = Feishu.get_values_batch(log_type, "kuaishou", "JK6npf")
        # kuaishou_follow_sheet = Feishu.get_values_batch(log_type, "kuaishou", "wW5cyb")
        # gzh_recommend_sheet = Feishu.get_values_batch(log_type, "gzh", "zWKFGb")

        for i in range(120):
            if len(cls.get_feeds_sheet(log_type, crawler, sheet)) > 1:
                break
            else:
                time.sleep(60)
                if i == 119 and crawler == "kanyikan" and sheet == "recommend":
                    Feishu.bot(log_type, "kanyikan", "看一看推荐榜表，已经 2 小时无数据了😤")
                elif i == 119 and crawler == "kanyikan" and sheet == "moment":
                    Feishu.bot(log_type, "kanyikan", "看一看朋友圈表，已经 2 小时无数据了😤")
                elif i == 119 and crawler == "xiaoniangao" and sheet == "person":
                    Feishu.bot(log_type, "xiaoniangao", "小年糕用户主页表，已经 2 小时无数据了😤")
                elif i == 119 and crawler == "music_album" \
                        and sheet == "recommend" and datetime.datetime.now().hour < 13:
                    Feishu.bot(log_type, "music_album", "音乐相册推荐表，已经 2 小时无数据了😤")
                elif i == 119 and crawler == "bszf" and sheet == "recommend" and datetime.datetime.now().hour < 13:
                    Feishu.bot(log_type, "bszf", "本山祝福推荐表，已经 2 小时无数据了😤")
                elif i == 119 and crawler == "kuaishou" and sheet == "recommend":
                    Feishu.bot(log_type, "kuaishou", "快手推荐表，已经 2 小时无数据了😤")
                elif i == 119 and crawler == "kuaishou" and sheet == "follow":
                    Feishu.bot(log_type, "kuaishou", "快手关注表，已经 2 小时无数据了😤")
                elif i == 119 and crawler == "gzh" and sheet == "recommend":
                    Feishu.bot(log_type, "gzh", "公众号推荐表，已经 2 小时无数据了😤")

    # 触发机器人报警:超过24小时没有新入库的视频
    @classmethod
    def robot_download_sheet(cls, log_type, crawler, duration):
        """
        已下载视频表:超过24小时没有新入库的视频
        """
        try:
            # 看一看爬虫报警
            if crawler == "kanyikan" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "看一看已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("看一看已下载表，超过24小时没有新视频入库了😤\n")

            # 小年糕爬虫报警
            elif crawler == "xiaoniangao_hour" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_小时级_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_小时级_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "xiaoniangao_person" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_用户主页_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_用户主页_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "xiaoniangao_play" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_播放量_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_播放量_已下载表，超过24小时没有新视频入库了😤\n")

            # 音乐相册爬虫报警
            elif crawler == "music_album" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "音乐相册已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("音乐相册已下载表，超过24小时没有新视频入库了😤\n")

            # 本山祝福爬虫报警
            elif crawler == "bszf" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "本山祝福已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("本山祝福已下载表，超过24小时没有新视频入库了😤\n")

            # 快手爬虫报警
            elif crawler == "kuaishou" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "快手已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("快手已下载表，超过24小时没有新视频入库了😤\n")

            # 公众号爬虫报警
            elif crawler == "gzh" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "公众号已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("公众号已下载表，超过24小时没有新视频入库了😤\n")

            # 微群视频爬虫报警
            elif crawler == "weiqun" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "微群视频已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("微群视频已下载表，超过24小时没有新视频入库了😤\n")

            # 微视爬虫报警
            elif crawler == "weishi" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "微视已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("微视已下载表，超过24小时没有新视频入库了😤\n")

            # 视频号爬虫报警
            elif crawler == "shipinhao" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "视频号已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("视频号已下载表，超过24小时没有新视频入库了😤\n")

        except Exception as e:
            Common.logger(log_type).error("robot_alarm异常:{}", e)

    # 监控运行入口
    @classmethod
    def main(cls):
        """
        每隔六小时，检查一次已下载表；
        已下载表的最新一条数据抓取时间，距当前时间超过 24 小时，则触发机器人报警，发送飞书报警消息
        """
        # 已下载表，超过 24 小时无新视频入库报警
        duration = 3600 * 24
        while True:
            if datetime.datetime.now().hour == 10:

                Common.logger("bot").info("监控看一看已下载表")
                Bot.robot_download_sheet("bot", "kanyikan", duration)

                Common.logger("bot").info("监控小年糕已下载表")
                Bot.robot_download_sheet("bot", "xiaoniangao_hour", duration)
                Bot.robot_download_sheet("bot", "xiaoniangao_person", duration)
                Bot.robot_download_sheet("bot", "xiaoniangao_play", duration)

                Common.logger("bot").info("监控本山祝福已下载表")
                Bot.robot_download_sheet("bot", "bszf", duration)

                Common.logger("bot").info("监控快手已下载表")
                Bot.robot_download_sheet("bot", "kuaishou", duration)

                Common.logger("bot").info("监控微视已下载表")
                Bot.robot_download_sheet("bot", "weishi", duration)

                Common.logger("bot").info("监控视频号已下载表")
                Bot.robot_download_sheet("bot", "shipinhao", duration)

                # Common.logger("bot").info("监控公众号已下载表")
                # Bot.robot_download_sheet("bot", "gzh", duration)

                # Common.logger("bot").info("监控音乐相册已下载表")
                # Bot.robot_download_sheet("bot", "music_album", duration)

                # Common.logger("bot").info("监控微群视频已下载表")
                # Bot.robot_download_sheet("bot", "weiqun", duration)

                Common.del_logs("bot")
                Common.logger("bot").info("休眠{}小时", 24-datetime.datetime.now().hour)
                time.sleep(3600 * (24-datetime.datetime.now().hour))
            else:
                pass


if __name__ == "__main__":
    # Bot.robot_download_sheet("bot", "gzh", 10)
    Bot.main()