piaoquan_crawler/changsha_bot.py

"""
独立执行，每个人执行一次，单日总量到达 300 即执行一次
"""
import json
import datetime
import time

import requests
import multiprocessing

from common.db import RedisClient


def protect_spider_timeout(function):
    """
    守护进程，在程序启动后的某一个时段内守护爬虫进程
    :param function: 被守护的函数
    :param hour: 守护时长 / hour
    """
    process = multiprocessing.Process(target=function)
    process.start()
    while True:
        if not process.is_alive():
            process.terminate()
            time.sleep(60)
            process = multiprocessing.Process(target=function)
            process.start()
        time.sleep(60)


def bot(name):
    """
    报警机器人
    """
    id_dict = {
        "余海涛": "ou_b87d153e200a04de3d82b1b9276e8f90",
        "范军": "ou_fce9cfef186e260e70554b47fee70a34",
        "罗情": "ou_88139cd84c2d105c2e1d699c14ec3375",
        "鲁涛": "ou_7986cccb78e6c981db8d0eef93443d05",
        "王雪珂": "ou_2233fb8e1302314bae166fcfa144151f",
        "邓锋": "ou_379d37645f929e1e6553a75aecda42a2"
    }
    url = "https://open.feishu.cn/open-apis/bot/v2/hook/df47bb77-ecaa-4628-b076-aae776415ae8"
    headers = {"Content-Type": "application/json"}
    payload = {
        "msg_type": "interactive",
        "card": {
            "elements": [
                {
                    "tag": "div",
                    "text": {
                        "content": "抓取数量触发限量通知, <at id={}></at>, <at id={}></at>, <at id={}></at>\n".format(
                            id_dict[name], id_dict["邓锋"], id_dict["王雪珂"]
                        ),
                        "tag": "lark_md",
                    },
                },
                {
                    "tag": "div",
                    "text": {
                        "content": "当天已经入库 300 条视频",
                        "tag": "lark_md",
                    },
                },
            ],
            "header": {"title": {"content": "【 通知 】", "tag": "plain_text"}},
        },
    }
    requests.post(url, headers=headers, data=json.dumps(payload))


def monitor():
    """
    监测 redis 中数据
    """
    keys = {"352": "余海涛", "353": "罗情", "53": "范军", "51": "鲁涛"}
    now = datetime.datetime.now().time()
    start_alert_time = datetime.time(10)
    end_alert_time = datetime.time(20, 30)
    while True:
        if start_alert_time <= now <= end_alert_time:
            try:
                R = RedisClient()
                if R.connect():
                    for key in keys:
                        count = R.select(key)
                        if count:
                            OO = int(count.decode("utf-8"))
                            name = keys[key]
                            redis_date_key = key + "-" + datetime.date.today().strftime("%Y%m%d")
                            if R.select(redis_date_key):
                                # 说明已经存储进去了， 不需要再报警了
                                continue
                            else:
                                if OO > 300:
                                    R.insert(redis_date_key, "already bot", 86400)
                                    # print("超过了， 报警", name)
                                    bot(name)
                                else:
                                    continue
            except Exception as e:
                print(e)
                pass
            # 查询一次之后等待 60 s
        time.sleep(60)


if __name__ == '__main__':
    protect_spider_timeout(monitor)
    # monitor()
    # R = RedisClient()
    # R.connect()
    # # w = R.select("53-20240327")
    # # print(w)
    # R.delete("51-20240327")
    # R.delete("53-20240327")
    # R.delete("353-20240327")
    # R.delete("352-20240327")