crawler_bot/main/run_bot.py

# -*- coding: utf-8 -*-
# @Author: wangkun
# @Time: 2022/8/9
# import datetime
import datetime
import os
import sys
import time
sys.path.append(os.getcwd())
from main.common import Common
from main.feishu_lib import Feishu


class Bot:
    # # 获取各个爬虫的 feeds 表
    # @classmethod
    # def get_feeds_sheet(cls, log_type, crawler, sheet):
    #     try:
    #         if crawler == "kanyikan" and sheet == "recommend":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "kanyikan", "SdCHOM")
    #         elif crawler == "kanyikan" and sheet == "moment":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "kanyikan", "tGqZMX")
    #         elif crawler == "xiaoniangao" and sheet == "hour":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4")
    #         elif crawler == "xiaoniangao" and sheet == "person":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "k6ldje")
    #         elif crawler == "music_album" and sheet == "recommend":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "music_album", "69UxPo")
    #         elif crawler == "bszf" and sheet == "recommend":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "bszf", "CcHgO7")
    #         elif crawler == "kuaishou" and sheet == "recommend":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "kuaishou", "JK6npf")
    #         elif crawler == "kuaishou" and sheet == "follow":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "kuaishou", "wW5cyb")
    #         elif crawler == "gzh" and sheet == "recommend":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "gzh", "zWKFGb")
    #         elif crawler == "weishi" and sheet == "recommend":
    #             feeds_sheet = Feishu.get_values_batch(log_type, "weishi", "O7fCzr")
    #         else:
    #             feeds_sheet = "请输入{crawler}和{sheet}"
    #
    #         return feeds_sheet
    #     except Exception as e:
    #         Common.logger(log_type).error("get_feeds_sheet异常:{}", e)
    #
    # # feeds_sheet表报警:连续 2 小时无数据
    # @classmethod
    # def rebot_feeds_sheet(cls, log_type, crawler, sheet):
    #     """
    #     每隔一分钟获取一次表数据的数量：
    #         1.中途有数据时，退出此次监控
    #         2.连续2小时无数据时，触发机器人报警
    #     """
    #     # kanyikan_recommend_sheet = Feishu.get_values_batch(log_type, "kanyikan", "SdCHOM")
    #     # kanyikan_moment_sheet = Feishu.get_values_batch(log_type, "kanyikan", "tGqZMX")
    #     # xiaoniangao_hour_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "ba0da4")
    #     # xiaoniangao_person_sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "k6ldje")
    #     # music_album_recommend_sheet = Feishu.get_values_batch(log_type, "music_album", "69UxPo")
    #     # bszf_recommend_sheet = Feishu.get_values_batch(log_type, "bszf", "CcHgO7")
    #     # kuaishou_recommend_sheet = Feishu.get_values_batch(log_type, "kuaishou", "JK6npf")
    #     # kuaishou_follow_sheet = Feishu.get_values_batch(log_type, "kuaishou", "wW5cyb")
    #     # gzh_recommend_sheet = Feishu.get_values_batch(log_type, "gzh", "zWKFGb")
    #
    #     for i in range(120):
    #         if len(cls.get_feeds_sheet(log_type, crawler, sheet)) > 1:
    #             break
    #         else:
    #             time.sleep(60)
    #             if i == 119 and crawler == "kanyikan" and sheet == "recommend":
    #                 Feishu.bot(log_type, "kanyikan", "看一看推荐榜表，已经 2 小时无数据了😤")
    #             elif i == 119 and crawler == "kanyikan" and sheet == "moment":
    #                 Feishu.bot(log_type, "kanyikan", "看一看朋友圈表，已经 2 小时无数据了😤")
    #             elif i == 119 and crawler == "xiaoniangao" and sheet == "person":
    #                 Feishu.bot(log_type, "xiaoniangao", "小年糕用户主页表，已经 2 小时无数据了😤")
    #             elif i == 119 and crawler == "music_album" \
    #                     and sheet == "recommend" and datetime.datetime.now().hour < 13:
    #                 Feishu.bot(log_type, "music_album", "音乐相册推荐表，已经 2 小时无数据了😤")
    #             elif i == 119 and crawler == "bszf" and sheet == "recommend" and datetime.datetime.now().hour < 13:
    #                 Feishu.bot(log_type, "bszf", "本山祝福推荐表，已经 2 小时无数据了😤")
    #             elif i == 119 and crawler == "kuaishou" and sheet == "recommend":
    #                 Feishu.bot(log_type, "kuaishou", "快手推荐表，已经 2 小时无数据了😤")
    #             elif i == 119 and crawler == "kuaishou" and sheet == "follow":
    #                 Feishu.bot(log_type, "kuaishou", "快手关注表，已经 2 小时无数据了😤")
    #             elif i == 119 and crawler == "gzh" and sheet == "recommend":
    #                 Feishu.bot(log_type, "gzh", "公众号推荐表，已经 2 小时无数据了😤")

    # 获取各个爬虫表最新一条抓取时间
    @classmethod
    def get_first_time(cls, log_type, crawler):
        try:
            if crawler == "xiaoniangao_hour":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == "xiaoniangao_person":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "Wu0CeL")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == "xiaoniangao_play":
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "c85k1C")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'xigua_video':
                sheet = Feishu.get_values_batch(log_type, "xigua", "e075e9")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == 'xigua_little_video':
                sheet = Feishu.get_values_batch(log_type, "xigua", "hDSDnv")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'zhihu_hot':
                sheet = Feishu.get_values_batch(log_type, "zhihu", "8871e3")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == 'zhihu_follow':
                sheet = Feishu.get_values_batch(log_type, "zhihu", "4MGuux")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'jxxf':
                sheet = Feishu.get_values_batch(log_type, "jxxf", "d9e9b1")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == 'zmyx':
                sheet = Feishu.get_values_batch(log_type, "zmyx", "19c772")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "kanyikan":
                sheet = Feishu.get_values_batch(log_type, "kanyikan", "20ce0c")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "music_album":
                sheet = Feishu.get_values_batch(log_type, "music_album", "f5a76e")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "bszf":
                sheet = Feishu.get_values_batch(log_type, "bszf", "440018")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][4]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "kuaishou_recommend":
                sheet = Feishu.get_values_batch(log_type, "kuaishou", "3cd128")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))
            elif crawler == "kuaishou_follow":
                sheet = Feishu.get_values_batch(log_type, "kuaishou", "fYdA8F")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "gzh":
                sheet = Feishu.get_values_batch(log_type, "gzh", "fCs3BT")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][3]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "weiqun":
                sheet = Feishu.get_values_batch(log_type, "weiqun", "3cd128")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "weishi":
                sheet = Feishu.get_values_batch(log_type, "weishi", "caa3fa")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            elif crawler == "shipinhao":
                sheet = Feishu.get_values_batch(log_type, "shipinhao", "c77cf9")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            else:
                sheet = Feishu.get_values_batch(log_type, "xiaoniangao", "yatRv2")
                # 已下载表，最新一条视频抓取时间
                first_download_time = sheet[1][5]
                first_download_time = int(time.mktime(time.strptime(first_download_time, "%Y/%m/%d %H:%M:%S")))

            return first_download_time
        except Exception as e:
            Common.logger(log_type).error("get_first_time异常:{}\n", e)

    # 触发机器人报警:超过24小时没有新入库的视频
    @classmethod
    def robot_download_sheet(cls, log_type, crawler, duration):
        """
        已下载视频表:超过24小时没有新入库的视频
        """
        try:
            # 看一看爬虫报警
            if crawler == "kanyikan" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "看一看已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("看一看已下载表，超过24小时没有新视频入库了😤\n")

            # 小年糕爬虫报警
            elif crawler == "xiaoniangao_hour" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_小时级_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_小时级_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "xiaoniangao_person" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_用户主页_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_用户主页_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "xiaoniangao_play" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "小年糕_播放量_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("小年糕_播放量_已下载表，超过24小时没有新视频入库了😤\n")

            # 西瓜视频
            elif crawler == "xigua_video" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "西瓜视频_用户主页_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("西瓜视频_用户主页_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "xigua_little_video" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "西瓜视频_小视频_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("西瓜视频_小视频_已下载表，超过24小时没有新视频入库了😤\n")

            # 知乎
            elif crawler == "zhihu_hot" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "知乎_热门_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("知乎_热门_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "zhihu_follow" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "知乎_定向_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("知乎_定向_已下载表，超过24小时没有新视频入库了😤\n")

            # 吉祥幸福
            elif crawler == "jxxf" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "吉祥幸福_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("吉祥幸福_已下载表，超过24小时没有新视频入库了😤\n")

            # 众妙音信
            elif crawler == "zmyx" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "众妙音信_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("众妙音信_已下载表，超过24小时没有新视频入库了😤\n")

            # 音乐相册爬虫报警
            elif crawler == "music_album" and (
                    int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "音乐相册已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("音乐相册已下载表，超过24小时没有新视频入库了😤\n")

            # 本山祝福爬虫报警
            elif crawler == "bszf" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "本山祝福已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("本山祝福已下载表，超过24小时没有新视频入库了😤\n")

            # 快手爬虫报警
            elif crawler == "kuaishou_recommend" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "快手_推荐榜_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("快手_推荐榜_已下载表，超过24小时没有新视频入库了😤\n")
            elif crawler == "kuaishou_follow" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "快手_用户主页_已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("快手_用户主页_已下载表，超过24小时没有新视频入库了😤\n")

            # 公众号爬虫报警
            elif crawler == "gzh" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "公众号已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("公众号已下载表，超过24小时没有新视频入库了😤\n")

            # 微群视频爬虫报警
            elif crawler == "weiqun" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "微群视频已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("微群视频已下载表，超过24小时没有新视频入库了😤\n")

            # 微视爬虫报警
            elif crawler == "weishi" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "微视已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("微视已下载表，超过24小时没有新视频入库了😤\n")

            # 视频号爬虫报警
            elif crawler == "shipinhao" and (int(time.time()) - cls.get_first_time(log_type, crawler) > int(duration)):
                Feishu.bot(log_type, crawler, "视频号已下载表，超过24小时没有新视频入库了😤")
                Common.logger(log_type).warning("视频号已下载表，超过24小时没有新视频入库了😤\n")

        except Exception as e:
            Common.logger(log_type).error("robot_alarm异常:{}", e)

    # 监控运行入口
    @classmethod
    def main(cls):
        """
        每隔六小时，检查一次已下载表；
        已下载表的最新一条数据抓取时间，距当前时间超过 24 小时，则触发机器人报警，发送飞书报警消息
        """
        # 已下载表，超过 24 小时无新视频入库报警
        duration = 3600 * 24
        while True:
            if datetime.datetime.now().hour == 10:

                Common.logger("bot").info("监控看一看已下载表")
                Bot.robot_download_sheet("bot", "kanyikan", duration)

                Common.logger("bot").info("监控小年糕已下载表")
                Bot.robot_download_sheet("bot", "xiaoniangao_hour", duration)
                Bot.robot_download_sheet("bot", "xiaoniangao_person", duration)
                Bot.robot_download_sheet("bot", "xiaoniangao_play", duration)

                Common.logger("bot").info("监控西瓜视频已下载表")
                Bot.robot_download_sheet("bot", "xigua_video", duration)
                # Bot.robot_download_sheet("bot", "xigua_little_video", duration)

                Common.logger('bot').info('监控知乎已下载表')
                Bot.robot_download_sheet("bot", "zhihu_hot", duration)
                Bot.robot_download_sheet("bot", "zhihu_follow", duration)

                Common.logger("bot").info("监控吉祥幸福已下载表")
                Bot.robot_download_sheet("bot", "jxxf", duration)

                Common.logger("bot").info("监控众妙音信已下载表")
                Bot.robot_download_sheet("bot", "zmyx", duration)

                Common.logger("bot").info("监控本山祝福已下载表")
                Bot.robot_download_sheet("bot", "bszf", duration)

                Common.logger("bot").info("监控快手已下载表")
                Bot.robot_download_sheet("bot", "kuaishou_recommend", duration)
                Bot.robot_download_sheet("bot", "kuaishou_follow", duration)

                Common.logger("bot").info("监控微视已下载表")
                Bot.robot_download_sheet("bot", "weishi", duration)

                Common.logger("bot").info("监控视频号已下载表")
                Bot.robot_download_sheet("bot", "shipinhao", duration)

                # Common.logger("bot").info("监控公众号已下载表")
                # Bot.robot_download_sheet("bot", "gzh", duration)

                # Common.logger("bot").info("监控音乐相册已下载表")
                # Bot.robot_download_sheet("bot", "music_album", duration)

                # Common.logger("bot").info("监控微群视频已下载表")
                # Bot.robot_download_sheet("bot", "weiqun", duration)

                Common.del_logs("bot")
                Common.logger("bot").info("休眠{}小时", 24-datetime.datetime.now().hour)
                time.sleep(3600 * (24-datetime.datetime.now().hour))
            else:
                pass


if __name__ == "__main__":
    # Bot.robot_download_sheet("bot", "kuaishou_recommend", 1)
    # Bot.robot_download_sheet("bot", "zmyx", 1)
    Bot.main()

    pass