model_script/monitor/automation_provide_job_monitor.py

import datetime

from aliyun.log import LogClient
from aliyun.log.auth import AUTH_VERSION_4

from util import feishu_inform_util

endpoint = "cn-hangzhou.log.aliyuncs.com"
access_key = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P"
access_key_id = "LTAIWYUujJAm7CbH"
project = "crawler-scheduler"
log_store = "aigc-provider"
query_sql = "* | select crawlerMode, result, if(reason='null', '成功', reason) as reason, count(distinct videoId) as videoIdCnt, count(distinct crawlerPlanId) as crawlerPlanIdCnt  from log where reason not in ('该账号已经存在爬取计划，跳过执行', '该视频近期已经处理过', '该Topic已经创建过爬取计划', '该关键词已经创建过爬取计划') group by crawlerMode, result, reason order by crawlerMode, result desc, reason"

client = LogClient(endpoint=endpoint, accessKey=access_key, accessKeyId=access_key_id, auth_version=AUTH_VERSION_4, region='cn-hangzhou')
webhook = 'https://open.feishu.cn/open-apis/bot/v2/hook/9f5c5cce-5eb2-4731-b368-33926f5549f9'


def send_feishu_card_msg(title, content):
    card_json = {
        "schema": "2.0",
        "header": {
            "title": {
                "tag": "plain_text",
                "content": title
            },
            "template": "blue"
        },
        "body": {
            "elements": [
                {
                    "tag": "markdown",
                    "content": content,
                    "text_align": "left",
                    "text_size": "normal",
                    "element_id": "taskExecuteCnt"
                }
            ]
        }
    }
    feishu_inform_util.send_card_msg_to_feishu(webhook, card_json)


def main():
    # 获取当前日期
    today = datetime.datetime.now()

    # 当天开始时间（00:00:00）
    start_of_day = datetime.datetime.combine(today.date(), datetime.time.min)
    # 当天结束时间（23:59:59.999999）
    end_of_day = datetime.datetime.combine(today.date(), datetime.time.max)

    # 转换为时间戳（秒级）
    start_timestamp = int(start_of_day.timestamp())
    end_timestamp = int(end_of_day.timestamp())

    resp = client.get_log(project=project, logstore=log_store, from_time=start_timestamp, to_time=end_timestamp, query=query_sql)
    log_data = resp.get_body().get('data')

    crawler_mode_set = set()
    for datum in log_data:
        crawler_mode_set.add(datum.get('crawlerMode'))

    for crawler_mode in crawler_mode_set:
        title = f"{crawler_mode} 执行情况监控"
        content = "| reason | videoIdCnt | crawlerPlanIdCnt |\n"
        content += "| --- | --- | --- |\n"
        for datum in resp.get_body().get('data'):
            if crawler_mode != datum.get('crawlerMode'):
                continue
            reason = datum.get('reason')
            video_id_cnt = datum.get('videoIdCnt')
            crawler_plan_id_cnt = datum.get('crawlerPlanIdCnt')
            content += f"| {reason} | {video_id_cnt} | {crawler_plan_id_cnt} |\n"

        send_feishu_card_msg(title, content)


if __name__ == "__main__":
    main()