rov-offline/redis_data_monitor.py

import datetime
from my_config import set_config
from my_utils import RedisHelper, send_msg_to_feishu
from log import Log

config_, _ = set_config()
log_ = Log()
redis_helper = RedisHelper()

region_code = config_.REGION_CODE


def rov_data_monitor(now_date, now_h):
    """rov模型预测列表"""
    if now_h > 6:
        rov_key_name = f"{config_.RECALL_KEY_NAME_PREFIX}{now_date}"
        if not redis_helper.key_exists(key_name=rov_key_name):
            msg_text = f"\n- 所属项目: rov-server/rov-offline" \
                       f"\n- 告警名称: 离线更新数据未按时更新告警" \
                       f"\n- 所属环境: {config_.ENV_TEXT}" \
                       f"\n- now_date: {now_date}" \
                       f"\n- now_h: {now_h}" \
                       f"\n- 告警描述: rov模型预测列表数据未按时更新"
            log_.info(f"msg_text = {msg_text}")
            send_msg_to_feishu(
                webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
                key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
                msg_text=msg_text
            )


def region_data_monitor(now_date, now_h, rule_params, key_prefix_dict):
    """地域分组数据"""
    region_code_list = [code for region, code in region_code.items()] +\
                       [code for city, code in config_.CITY_CODE.items()]
    # print(region_code_list)
    for param in rule_params.get('params_list'):
        data_key = param.get('data')
        rule_key = param.get('rule')
        for key_con, key_prefix in key_prefix_dict.items():
            # if key_con == '不区分地域相对24h筛选后剩余去重后数据' and rule_key != 'rule4':
            #     continue
            no_update_region_list = []
            for region in region_code_list:
                region_key_name = f"{key_prefix}{region}:{data_key}:{rule_key}:{now_date}:{now_h}"
                if not redis_helper.key_exists(key_name=region_key_name):
                    no_update_region_list.append(region)
            if len(no_update_region_list) == len(region_code_list):
                msg_text = f"\n- 所属项目: rov-server/rov-offline" \
                           f"\n- 告警名称: 离线更新数据未按时更新告警" \
                           f"\n- 所属环境: {config_.ENV_TEXT}" \
                           f"\n- now_date: {now_date}" \
                           f"\n- now_h: {now_h}" \
                           f"\n- 告警描述: {key_con}未按时更新, data_key={data_key}, rule_key={rule_key}"
                log_.info(f"msg_text = {msg_text}")
                send_msg_to_feishu(
                    webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
                    key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
                    msg_text=msg_text
                )


def special_videos_monitor(now_date, now_h):
    """特殊mid(屏蔽名单)指定视频列表"""
    if now_h > 4:
        rov_key_name = f"{config_.KEY_NAME_PREFIX_SPECIAL_VIDEOS}{now_date}"
        if not redis_helper.key_exists(key_name=rov_key_name):
            msg_text = f"\n- 所属项目: rov-server/rov-offline" \
                       f"\n- 告警名称: 离线更新数据未按时更新告警" \
                       f"\n- 所属环境: {config_.ENV_TEXT}" \
                       f"\n- now_date: {now_date}" \
                       f"\n- now_h: {now_h}" \
                       f"\n- 告警描述: 特殊mid(屏蔽名单)指定视频列表数据未按时更新"
            log_.info(f"msg_text = {msg_text}")
            send_msg_to_feishu(
                webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
                key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
                msg_text=msg_text
            )


def whole_movies_monitor(now_date, now_h):
    """完整影视资源列表"""
    rov_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_WHOLE_MOVIES}{now_date}.{now_h}"
    if not redis_helper.key_exists(key_name=rov_key_name):
        msg_text = f"\n- 所属项目: rov-server/rov-offline" \
                   f"\n- 告警名称: 离线更新数据未按时更新告警" \
                   f"\n- 所属环境: {config_.ENV_TEXT}" \
                   f"\n- now_date: {now_date}" \
                   f"\n- now_h: {now_h}" \
                   f"\n- 告警描述: 完整影视资源列表数据未按时更新"
        log_.info(f"msg_text = {msg_text}")
        send_msg_to_feishu(
            webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
            key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
            msg_text=msg_text
        )


def get_redis_data_keys(now_date, now_h):
    # 检查所需列表是否已更新好
    redis_data_keys = []
    # rov模型预测列表
    if now_h > 6:
        rov_key_name = f"{config_.RECALL_KEY_NAME_PREFIX}{now_date}"
        redis_data_keys.append(rov_key_name)
    # 地域分组小时级列表
    rule_params = config_.RULE_PARAMS_REGION_APP_TYPE
    key_prefix_list = [
        config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H,
        config_.RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H,
        config_.RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H,
        config_.RECALL_KEY_NAME_PREFIX_DUP_REGION_H
    ]
    region_code_list = [code for region, code in region_code.items()]
    for app_type, params in rule_params.items():
        for data_key, data_param in params['data_params'].items():
            for rule_key, rule_param in params['rule_params'].items():
                for region in region_code_list:
                    if region == '-1':
                        region_key_name = [f"{key_prefix_list[-1]}{region}.{app_type}.{data_key}.{rule_key}.{now_date}.{now_h}"]
                    else:
                        region_key_name = [f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}.{now_date}.{now_h}"
                                           for key_prefix in key_prefix_list]
                    redis_data_keys.extend(region_key_name)

    return redis_data_keys


def monitor(now_date, now_h):
    # rov_data_monitor(now_date=now_date, now_h=now_h)
    key_prefix_dict_24h = {
        '地域分组小时级数据': config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H,
        '地域分组相对24h去重后数据': config_.RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H,
        '不区分地域相对24h去重后数据': config_.RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H,
        '不区分地域相对24h筛选后剩余去重后数据': config_.RECALL_KEY_NAME_PREFIX_DUP3_REGION_24H_H,
        # 'rov模型预测列表去重后数据': config_.RECALL_KEY_NAME_PREFIX_DUP_REGION_H,
    }
    region_data_monitor(now_date=now_date, now_h=now_h, rule_params=config_.RULE_PARAMS_REGION_APP_TYPE,
                        key_prefix_dict=key_prefix_dict_24h)
    # key_prefix_dict_48h = {
    #     '地域分组小时级数据': config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H,
    #     '地域分组相对24h去重后数据': config_.RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H,
    #     '不区分地域相对48h去重后数据': config_.RECALL_KEY_NAME_PREFIX_DUP2_REGION_48H_H,
    #     '不区分地域相对48h筛选后剩余去重后数据': config_.RECALL_KEY_NAME_PREFIX_DUP3_REGION_48H_H,
    #     'rov模型预测列表去重后数据': config_.RECALL_KEY_NAME_PREFIX_DUP_REGION_H,
    # }
    # region_data_monitor(now_date=now_date, now_h=now_h, rule_params=config_.RULE_PARAMS_REGION_APP_TYPE_48H,
    #                     key_prefix_dict=key_prefix_dict_48h)
    special_videos_monitor(now_date=now_date, now_h=now_h)
    # whole_movies_monitor(now_date=now_date, now_h=now_h)


if __name__ == '__main__':
    now_date = datetime.datetime.today().strftime('%Y%m%d')
    now_h = datetime.datetime.now().hour
    log_.info(f"now_date = {now_date}, now_h = {now_h}")
    monitor(now_date=now_date, now_h=now_h)
    log_.info("end!")