rov-offline/region_rule_rank_h.py

# -*- coding: utf-8 -*-
# @ModuleName: region_rule_rank_h
# @Author: Liqian
# @Time: 2022/5/5 15:54
# @Software: PyCharm

import multiprocessing
import os
import sys
import traceback

import gevent
import datetime
import pandas as pd
import math
from functools import reduce
from odps import ODPS
from threading import Timer, Thread
from utils import MysqlHelper, RedisHelper, get_data_from_odps, filter_video_status, filter_shield_video, \
    check_table_partition_exits, filter_video_status_app, send_msg_to_feishu, filter_political_videos
from config import set_config
from log import Log
from check_video_limit_distribute import update_limit_video_score

# os.environ['NUMEXPR_MAX_THREADS'] = '16'

config_, _ = set_config()
log_ = Log()

region_code = config_.REGION_CODE

features = [
    'apptype',
    'code',
    'videoid',
    'lastonehour_preview',  # 过去1小时预曝光人数
    'lastonehour_view',  # 过去1小时曝光人数
    'lastonehour_play',  # 过去1小时播放人数
    'lastonehour_share',  # 过去1小时分享人数
    'lastonehour_return',  # 过去1小时分享，过去1小时回流人数
    'lastonehour_preview_total',  # 过去1小时预曝光次数
    'lastonehour_view_total',  # 过去1小时曝光次数
    'lastonehour_play_total',  # 过去1小时播放次数
    'lastonehour_share_total',  # 过去1小时分享次数
    'platform_return',
    'lastonehour_show',  # 不区分地域
    'lastonehour_show_region',  # 地域分组
]


def get_region_code(region):
    """获取省份对应的code"""
    mysql_helper = MysqlHelper(mysql_info=config_.MYSQL_INFO)
    sql = f"SELECT ad_code FROM region_adcode WHERE parent_id = 0 AND region LIKE '{region}%';"
    ad_code = mysql_helper.get_data(sql=sql)
    return ad_code[0][0]


def h_data_check(project, table, now_date):
    """检查数据是否准备好"""
    odps = ODPS(
        access_id=config_.ODPS_CONFIG['ACCESSID'],
        secret_access_key=config_.ODPS_CONFIG['ACCESSKEY'],
        project=project,
        endpoint=config_.ODPS_CONFIG['ENDPOINT'],
        connect_timeout=3000,
        read_timeout=500000,
        pool_maxsize=1000,
        pool_connections=1000
    )

    try:
        dt = datetime.datetime.strftime(now_date, '%Y%m%d%H')
        check_res = check_table_partition_exits(date=dt, project=project, table=table)
        if check_res:
            sql = f'select * from {project}.{table} where dt = {dt}'
            with odps.execute_sql(sql=sql).open_reader() as reader:
                data_count = reader.count
        else:
            data_count = 0
    except Exception as e:
        data_count = 0
    return data_count


def get_rov_redis_key(now_date):
    """获取rov模型结果存放key"""
    redis_helper = RedisHelper()
    now_dt = datetime.datetime.strftime(now_date, '%Y%m%d')
    key_name = f'{config_.RECALL_KEY_NAME_PREFIX}{now_dt}'
    if not redis_helper.key_exists(key_name=key_name):
        pre_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
        key_name = f'{config_.RECALL_KEY_NAME_PREFIX}{pre_dt}'
    return key_name


def get_day_30day_videos(now_date, data_key, rule_key):
    """获取天级更新相对30天的视频id"""
    redis_helper = RedisHelper()
    day_30day_recall_key_prefix = config_.RECALL_KEY_NAME_PREFIX_30DAY
    now_dt = datetime.datetime.strftime(now_date, '%Y%m%d')
    day_30day_recall_key_name = f"{day_30day_recall_key_prefix}{data_key}:{rule_key}:{now_dt}"
    if not redis_helper.key_exists(key_name=day_30day_recall_key_name):
        redis_dt = datetime.datetime.strftime((now_date - datetime.timedelta(days=1)), '%Y%m%d')
        day_30day_recall_key_name = f"{day_30day_recall_key_prefix}{data_key}:{rule_key}:{redis_dt}"
    data = redis_helper.get_all_data_from_zset(key_name=day_30day_recall_key_name, with_scores=True)
    if data is None:
        return None
    video_ids = [int(video_id) for video_id, _ in data]
    return video_ids


def get_feature_data(project, table, now_date):
    """获取特征数据"""
    dt = datetime.datetime.strftime(now_date, '%Y%m%d%H')
    # dt = '2022041310'
    records = get_data_from_odps(date=dt, project=project, table=table)
    feature_data = []
    for record in records:
        item = {}
        for feature_name in features:
            item[feature_name] = record[feature_name]
        feature_data.append(item)
    feature_df = pd.DataFrame(feature_data)
    return feature_df


def cal_score(df, param):
    """
    计算score
    :param df: 特征数据
    :param param: 规则参数
    :return:
    """
    # score计算公式: sharerate*backrate*logback*ctr
    # sharerate = lastonehour_share/(lastonehour_play+1000)
    # backrate = lastonehour_return/(lastonehour_share+10)
    # ctr = lastonehour_play/(lastonehour_preview+1000), 对ctr限最大值：K2 = 0.6 if ctr > 0.6 else ctr
    # score = sharerate * backrate * LOG(lastonehour_return+1) * K2

    df = df.fillna(0)
    df['share_rate'] = df['lastonehour_share'] / (df['lastonehour_play'] + 1000)
    df['back_rate'] = df['lastonehour_return'] / (df['lastonehour_share'] + 10)
    df['log_back'] = (df['lastonehour_return'] + 1).apply(math.log)
    if param.get('view_type', None) == 'video-show':
        df['ctr'] = df['lastonehour_play'] / (df['lastonehour_show'] + 1000)
    elif param.get('view_type', None) == 'video-show-region':
        df['ctr'] = df['lastonehour_play'] / (df['lastonehour_show_region'] + 1000)
    else:
        df['ctr'] = df['lastonehour_play'] / (df['lastonehour_preview'] + 1000)
    df['K2'] = df['ctr'].apply(lambda x: 0.6 if x > 0.6 else x)
    df['platform_return_rate'] = df['platform_return'] / df['lastonehour_return']

    df['score1'] = df['share_rate'] * df['back_rate'] * df['log_back'] * df['K2']

    click_score_rate = param.get('click_score_rate', None)
    back_score_rate = param.get('click_score_rate', None)
    if click_score_rate is not None:
        df['score'] = (1 - click_score_rate) * df['score1'] + click_score_rate * df['K2']
    elif back_score_rate is not None:
        df['score'] = (1 - back_score_rate) * df['score1'] + back_score_rate * df['back_rate']
    else:
        df['score'] = df['score1']

    df = df.sort_values(by=['score'], ascending=False)
    return df


def add_videos(initial_df, now_date, rule_key, region, data_key):
    """
    地域小时级数据列表中增加前6h优质视频
    :param initial_df: 地域小时级筛选结果
    :param now_date:
    :param data_key:
    :param region:
    :param rule_key:
    :return: df
    """
    redis_helper = RedisHelper()
    pre_h_data = []
    for pre_h in range(1, 7):
        pre_date = now_date - datetime.timedelta(hours=pre_h)


        pre_h_recall_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H}{region}:{data_key}:{rule_key}:" \
                                f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{pre_h}"
        initial_data = redis_helper.get_all_data_from_zset(key_name=pre_h_recall_key_name, with_scores=True)
        if initial_data is None:
            continue
        pre_h_data.extend(initial_data)
    pre_h_df = pd.DataFrame(data=pre_h_data, columns=['videoid', 'score'])
    score_list = initial_df['score'].to_list()
    if len(score_list) > 0:
        min_score = min(score_list)
    else:
        min_score = 0
    pre_h_df = pre_h_df[pre_h_df['score'] > min_score]
    df = pd.concat([initial_df, pre_h_df], ignore_index=True)
    # videoid去重，保留分值高
    df['videoid'] = df['videoid'].astype(int)
    df = df.sort_values(by=['score'], ascending=False)
    df = df.drop_duplicates(subset=['videoid'], keep="first")
    return df


def video_rank(df, now_date, now_h, rule_key, param, region, data_key, rule_rank_h_flag, add_videos_in_20h=False):
    """
    获取符合进入召回源条件的视频，与每日更新的rov模型结果视频列表进行合并
    :param df:
    :param now_date:
    :param now_h:
    :param rule_key: 小时级数据进入条件
    :param param: 小时级数据进入条件参数
    :param region: 所属地域
    :return:
    """
    redis_helper = RedisHelper()

    # 获取符合进入召回源条件的视频，进入条件：小时级回流>=20 && score>=0.005
    return_count = param.get('return_count', 1)
    score_value = param.get('score_rule', 0)
    platform_return_rate = param.get('platform_return_rate', 0)
    h_recall_df = df[(df['lastonehour_return'] >= return_count) & (df['score'] >= score_value)
                     & (df['platform_return_rate'] >= platform_return_rate)]

    # videoid重复时，保留分值高
    h_recall_df = h_recall_df.sort_values(by=['score'], ascending=False)
    h_recall_df = h_recall_df.drop_duplicates(subset=['videoid'], keep='first')
    h_recall_df['videoid'] = h_recall_df['videoid'].astype(int)

    # 20点增加打捞的优质视频
    if now_h == 20 and add_videos_in_20h is True:
        # print(len(h_recall_df))
        h_recall_df = add_videos(initial_df=h_recall_df, now_date=now_date, rule_key=rule_key,
                                 region=region, data_key=data_key)
        # print(len(h_recall_df))


    h_recall_videos = h_recall_df['videoid'].to_list()
    # log_.info(f'h_recall videos count = {len(h_recall_videos)}')

    # 视频状态过滤
    if data_key in ['data7', ]:
        filtered_videos = filter_video_status_app(h_recall_videos)
    else:
        filtered_videos = filter_video_status(h_recall_videos)
    # log_.info('filtered_videos count = {}'.format(len(filtered_videos)))

    # 屏蔽视频过滤
    shield_key_name_list = config_.SHIELD_CONFIG.get(region, None)
    if shield_key_name_list is not None:
        filtered_videos = filter_shield_video(video_ids=filtered_videos, shield_key_name_list=shield_key_name_list)
        # log_.info(f"shield filtered_videos count = {len(filtered_videos)}")

    # 涉政视频过滤
    political_filter = param.get('political_filter', None)
    if political_filter is True:
        log_.info(f"political filter videos count = {len(filtered_videos)}")
        filtered_videos = filter_political_videos(video_ids=filtered_videos)
        log_.info(f"political filtered videos count = {len(filtered_videos)}")

    # 写入对应的redis
    h_video_ids = []
    by_30day_rule_key = param.get('30day_rule_key', None)
    if by_30day_rule_key is not None:
        # 与相对30天列表去重
        h_video_ids = get_day_30day_videos(now_date=now_date, data_key=data_key, rule_key=by_30day_rule_key)
        log_.info(f"h_video_ids count = {len(h_video_ids)}")
        if h_video_ids is not None:
            filtered_videos = [video_id for video_id in filtered_videos if int(video_id) not in h_video_ids]
            log_.info(f"filtered_videos count = {len(filtered_videos)}")

    h_recall_result = {}
    for video_id in filtered_videos:
        score = h_recall_df[h_recall_df['videoid'] == video_id]['score']
        # print(score)
        h_recall_result[int(video_id)] = float(score)
        h_video_ids.append(int(video_id))
    h_recall_key_name = \
        f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H}{region}:{data_key}:{rule_key}:" \
        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
    if len(h_recall_result) > 0:
        log_.info(f"h_recall_result count = {len(h_recall_result)}")
        redis_helper.add_data_with_zset(key_name=h_recall_key_name, data=h_recall_result, expire_time=2 * 24 * 3600)
        # 限流视频score调整
        update_limit_video_score(initial_videos=h_recall_result, key_name=h_recall_key_name)
        # 清空线上过滤应用列表
        # redis_helper.del_keys(key_name=f"{config_.REGION_H_VIDEO_FILER}{region}.{app_type}.{data_key}.{rule_key}")

    region_24h_rule_key = param.get('region_24h_rule_key', 'rule1')
    by_24h_rule_key = param.get('24h_rule_key', None)
    by_48h_rule_key = param.get('48h_rule_key', None)
    # 与其他召回视频池去重，存入对应的redis
    dup_to_redis(h_video_ids=h_video_ids, now_date=now_date, now_h=now_h, rule_key=rule_key,
                 region_24h_rule_key=region_24h_rule_key, by_24h_rule_key=by_24h_rule_key, by_48h_rule_key=by_48h_rule_key,
                 region=region, data_key=data_key, rule_rank_h_flag=rule_rank_h_flag, political_filter=political_filter)


def dup_data(h_video_ids, initial_key_name, dup_key_name, region, political_filter):
    redis_helper = RedisHelper()
    if redis_helper.key_exists(key_name=initial_key_name):
        initial_data = redis_helper.get_all_data_from_zset(key_name=initial_key_name, with_scores=True)
        # 屏蔽视频过滤
        initial_video_ids = [int(video_id) for video_id, _ in initial_data]
        shield_key_name_list = config_.SHIELD_CONFIG.get(region, None)
        if shield_key_name_list is not None:
            initial_video_ids = filter_shield_video(video_ids=initial_video_ids,
                                                    shield_key_name_list=shield_key_name_list)
        # 涉政视频过滤
        if political_filter is True:
            initial_video_ids = filter_political_videos(video_ids=initial_video_ids)

        dup_data = {}
        for video_id, score in initial_data:
            if int(video_id) not in h_video_ids and int(video_id) in initial_video_ids:
                dup_data[int(video_id)] = score
                h_video_ids.append(int(video_id))

        if len(dup_data) > 0:
            redis_helper.add_data_with_zset(key_name=dup_key_name, data=dup_data, expire_time=2 * 24 * 3600)
            # 限流视频score调整
            update_limit_video_score(initial_videos=dup_data, key_name=dup_key_name)
    return h_video_ids


def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, by_24h_rule_key, by_48h_rule_key,
                 region, data_key, rule_rank_h_flag, political_filter):
    """将地域分组小时级数据与其他召回视频池去重，存入对应的redis"""
    # ##### 去重更新地域分组小时级24h列表，并另存为redis中
    region_24h_key_name = \
        f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_24H}{region}:{data_key}:{region_24h_rule_key}:" \
        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
    region_24h_dup_key_name = \
        f"{config_.RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H}{region}:{data_key}:{rule_key}:" \
        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
    h_video_ids = dup_data(h_video_ids=h_video_ids, initial_key_name=region_24h_key_name,
                           dup_key_name=region_24h_dup_key_name, region=region, political_filter=political_filter)

    if rule_rank_h_flag == '48h':

        # ##### 去重小程序相对48h更新结果，并另存为redis中
        h_48h_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_48H}{data_key}:{by_48h_rule_key}:" \
                         f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
        h_48h_dup_key_name = \
            f"{config_.RECALL_KEY_NAME_PREFIX_DUP2_REGION_48H_H}{region}:{data_key}:{rule_key}:" \
            f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
        h_video_ids = dup_data(h_video_ids=h_video_ids, initial_key_name=h_48h_key_name,
                               dup_key_name=h_48h_dup_key_name, region=region, political_filter=political_filter)

        # ##### 去重小程序相对48h 筛选后剩余数据 更新结果，并另存为redis中
        if by_48h_rule_key == 'rule1':
            other_h_48h_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_48H_OTHER}{data_key}:" \
                                   f"{by_48h_rule_key}:{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
            other_h_48h_dup_key_name = \
                f"{config_.RECALL_KEY_NAME_PREFIX_DUP3_REGION_48H_H}{region}:{data_key}:{rule_key}:" \
                f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
            h_video_ids = dup_data(h_video_ids=h_video_ids, initial_key_name=other_h_48h_key_name,
                                   dup_key_name=other_h_48h_dup_key_name, region=region,
                                   political_filter=political_filter)

    else:
        # ##### 去重小程序相对24h更新结果，并另存为redis中
        h_24h_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}{data_key}:{by_24h_rule_key}:" \
                         f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
        h_24h_dup_key_name = \
            f"{config_.RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H}{region}:{data_key}:{rule_key}:" \
            f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
        h_video_ids = dup_data(h_video_ids=h_video_ids, initial_key_name=h_24h_key_name,
                               dup_key_name=h_24h_dup_key_name, region=region, political_filter=political_filter)

        # ##### 去重小程序相对24h 筛选后剩余数据 更新结果，并另存为redis中
        # if by_24h_rule_key in ['rule3', 'rule4']:
        other_h_24h_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H_OTHER}{data_key}:" \
                               f"{by_24h_rule_key}:{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
        other_h_24h_dup_key_name = \
            f"{config_.RECALL_KEY_NAME_PREFIX_DUP3_REGION_24H_H}{region}:{data_key}:{rule_key}:" \
            f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
        h_video_ids = dup_data(h_video_ids=h_video_ids, initial_key_name=other_h_24h_key_name,
                               dup_key_name=other_h_24h_dup_key_name, region=region, political_filter=political_filter)

    # ##### 去重小程序模型更新结果，并另存为redis中
    # model_key_name = get_rov_redis_key(now_date=now_date)
    # model_data_dup_key_name = \
    #     f"{config_.RECALL_KEY_NAME_PREFIX_DUP_REGION_H}{region}:{data_key}:{rule_key}:" \
    #     f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
    # h_video_ids = dup_data(h_video_ids=h_video_ids, initial_key_name=model_key_name,
    #                        dup_key_name=model_data_dup_key_name, region=region)


def merge_df(df_left, df_right):
    """
    df按照videoid, code 合并，对应特征求和
    :param df_left:
    :param df_right:
    :return:
    """
    df_merged = pd.merge(df_left, df_right, on=['videoid', 'code'], how='outer', suffixes=['_x', '_y'])
    df_merged.fillna(0, inplace=True)
    feature_list = ['videoid', 'code']
    for feature in features:
        if feature in ['apptype', 'videoid', 'code']:
            continue
        df_merged[feature] = df_merged[f'{feature}_x'] + df_merged[f'{feature}_y']
        feature_list.append(feature)
    return df_merged[feature_list]


def merge_df_with_score(df_left, df_right):
    """
    df 按照[videoid, code]合并，平台回流人数、回流人数、分数 分别求和
    :param df_left:
    :param df_right:
    :return:
    """
    df_merged = pd.merge(df_left, df_right, on=['videoid', 'code'], how='outer', suffixes=['_x', '_y'])
    df_merged.fillna(0, inplace=True)
    feature_list = ['videoid', 'code', 'lastonehour_return', 'platform_return', 'score']
    for feature in feature_list[2:]:
        df_merged[feature] = df_merged[f'{feature}_x'] + df_merged[f'{feature}_y']
    return df_merged[feature_list]


def process_with_region(region, df_merged, data_key, rule_key, rule_param, now_date, now_h,
                        rule_rank_h_flag, add_videos_in_20h):
    log_.info(f"region = {region} start...")
    # 计算score
    region_df = df_merged[df_merged['code'] == region]
    log_.info(f'region = {region}, region_df count = {len(region_df)}')
    score_df = cal_score(df=region_df, param=rule_param)
    video_rank(df=score_df, now_date=now_date, now_h=now_h, rule_key=rule_key, param=rule_param,
               region=region, data_key=data_key, rule_rank_h_flag=rule_rank_h_flag,
               add_videos_in_20h=add_videos_in_20h)
    log_.info(f"region = {region} end!")


def process_with_region2(region, df_merged, data_key, rule_key, rule_param, now_date, now_h,
                         rule_rank_h_flag, add_videos_in_20h):
    log_.info(f"region = {region} start...")
    region_score_df = df_merged[df_merged['code'] == region]
    log_.info(f'region = {region}, region_score_df count = {len(region_score_df)}')
    video_rank(df=region_score_df, now_date=now_date, now_h=now_h, region=region,
               rule_key=rule_key, param=rule_param, data_key=data_key, rule_rank_h_flag=rule_rank_h_flag,
               add_videos_in_20h=add_videos_in_20h)
    log_.info(f"region = {region} end!")


def process_with_app_type(app_type, params, region_code_list, feature_df, now_date, now_h, rule_rank_h_flag):
    log_.info(f"app_type = {app_type} start...")
    data_params_item = params.get('data_params')
    rule_params_item = params.get('rule_params')
    task_list = []
    for param in params.get('params_list'):
        data_key = param.get('data')
        data_param = data_params_item.get(data_key)
        log_.info(f"data_key = {data_key}, data_param = {data_param}")
        df_list = [feature_df[feature_df['apptype'] == apptype] for apptype in data_param]
        df_merged = reduce(merge_df, df_list)

        rule_key = param.get('rule')
        rule_param = rule_params_item.get(rule_key)
        log_.info(f"rule_key = {rule_key}, rule_param = {rule_param}")
        task_list.extend(
            [
                gevent.spawn(process_with_region, region, df_merged, app_type, data_key, rule_key, rule_param,
                             now_date, now_h, rule_rank_h_flag)
                for region in region_code_list
            ]
        )
    gevent.joinall(task_list)
    log_.info(f"app_type = {app_type} end!")


    # log_.info(f"app_type = {app_type}")
    # task_list = []
    # for data_key, data_param in params['data_params'].items():
    #     log_.info(f"data_key = {data_key}, data_param = {data_param}")
    #     df_list = [feature_df[feature_df['apptype'] == apptype] for apptype in data_param]
    #     df_merged = reduce(merge_df, df_list)
    #     for rule_key, rule_param in params['rule_params'].items():
    #         log_.info(f"rule_key = {rule_key}, rule_param = {rule_param}")
    #         task_list.extend(
    #             [
    #                 gevent.spawn(process_with_region, region, df_merged, app_type, data_key, rule_key, rule_param,
    #                              now_date, now_h)
    #                 for region in region_code_list
    #             ]
    #         )
    # gevent.joinall(task_list)


def copy_data_for_city(region, city_code, data_key, rule_key, now_date, now_h):
    """copy 对应数据到城市对应redis，并做相应屏蔽视频过滤"""
    log_.info(f"city_code = {city_code} start ...")
    redis_helper = RedisHelper()
    key_prefix_list = [
        config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H,  # 地域小时级
        config_.RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H,  # 地域相对24h
        config_.RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H,  # 不区分地域相对24h
        config_.RECALL_KEY_NAME_PREFIX_DUP3_REGION_24H_H,  # 不区分地域相对24h筛选后
        config_.RECALL_KEY_NAME_PREFIX_DUP_REGION_H,  # rov大列表
    ]
    for key_prefix in key_prefix_list:
        region_key = f"{key_prefix}{region}:{data_key}:{rule_key}:{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
        city_key = f"{key_prefix}{city_code}:{data_key}:{rule_key}:{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
        if not redis_helper.key_exists(key_name=region_key):
            continue
        region_data = redis_helper.get_all_data_from_zset(key_name=region_key, with_scores=True)
        if not region_data:
            continue
        # 屏蔽视频过滤
        region_video_ids = [int(video_id) for video_id, _ in region_data]
        shield_key_name_list = config_.SHIELD_CONFIG.get(city_code, None)
        if shield_key_name_list is not None:
            filtered_video_ids = filter_shield_video(video_ids=region_video_ids,
                                                     shield_key_name_list=shield_key_name_list)
        else:
            filtered_video_ids = region_video_ids
        city_data = {}
        for video_id, score in region_data:
            if int(video_id) in filtered_video_ids:
                city_data[int(video_id)] = score

        if len(city_data) > 0:
            redis_helper.add_data_with_zset(key_name=city_key, data=city_data, expire_time=2 * 24 * 3600)

    log_.info(f"city_code = {city_code} end!")


def process_with_param(param, data_params_item, rule_params_item, region_code_list, feature_df, now_date, now_h, rule_rank_h_flag):
    log_.info(f"param = {param} start...")

    data_key = param.get('data')
    data_param = data_params_item.get(data_key)
    log_.info(f"data_key = {data_key}, data_param = {data_param}")
    rule_key = param.get('rule')
    rule_param = rule_params_item.get(rule_key)
    log_.info(f"rule_key = {rule_key}, rule_param = {rule_param}")
    merge_func = rule_param.get('merge_func', None)
    # 是否在20点的数据中增加打捞的优质视频
    add_videos_in_20h = rule_param.get('add_videos_in_20h', False)

    if merge_func == 2:
        score_df_list = []
        for apptype, weight in data_param.items():
            df = feature_df[feature_df['apptype'] == apptype]
            # 计算score
            score_df = cal_score(df=df, param=rule_param)
            score_df['score'] = score_df['score'] * weight
            score_df_list.append(score_df)
        # 分数合并
        df_merged = reduce(merge_df_with_score, score_df_list)
        # 更新平台回流比
        df_merged['platform_return_rate'] = df_merged['platform_return'] / df_merged['lastonehour_return']
        task_list = [
            gevent.spawn(process_with_region2,
                         region, df_merged, data_key, rule_key, rule_param, now_date, now_h, rule_rank_h_flag,
                         add_videos_in_20h)
            for region in region_code_list
        ]
    else:
        df_list = [feature_df[feature_df['apptype'] == apptype] for apptype in data_param]
        df_merged = reduce(merge_df, df_list)
        task_list = [
            gevent.spawn(process_with_region,
                         region, df_merged, data_key, rule_key, rule_param, now_date, now_h, rule_rank_h_flag,
                         add_videos_in_20h)
            for region in region_code_list
        ]

    gevent.joinall(task_list)

    # 特殊城市视频数据准备
    for region, city_list in config_.REGION_CITY_MAPPING.items():
        t = [
            gevent.spawn(
                copy_data_for_city,
                region, city_code, data_key, rule_key, now_date, now_h
            )
            for city_code in city_list
        ]
        gevent.joinall(t)

    log_.info(f"param = {param} end!")


def rank_by_h(project, table, now_date, now_h, rule_params, region_code_list, rule_rank_h_flag):
    # 获取特征数据
    feature_df = get_feature_data(project=project, table=table, now_date=now_date)
    feature_df['apptype'] = feature_df['apptype'].astype(int)
    data_params_item = rule_params.get('data_params')
    rule_params_item = rule_params.get('rule_params')
    params_list = rule_params.get('params_list')
    pool = multiprocessing.Pool(processes=len(params_list))
    for param in params_list:
        pool.apply_async(
            func=process_with_param,
            args=(param, data_params_item, rule_params_item, region_code_list, feature_df, now_date, now_h, rule_rank_h_flag)
        )
    pool.close()
    pool.join()




    # pool = multiprocessing.Pool(processes=len(config_.APP_TYPE))
    # for app_type, params in rule_params.items():
    #     pool.apply_async(func=process_with_app_type,
    #                      args=(app_type, params, region_code_list, feature_df, now_date, now_h, rule_rank_h_flag))
    # pool.close()
    # pool.join()

    """
    for app_type, params in rule_params.items():
        log_.info(f"app_type = {app_type} start...")
        data_params_item = params.get('data_params')
        rule_params_item = params.get('rule_params')
        for param in params.get('params_list'):
            log_.info(f"param = {param} start...")
            data_key = param.get('data')
            data_param = data_params_item.get(data_key)
            log_.info(f"data_key = {data_key}, data_param = {data_param}")
            df_list = [feature_df[feature_df['apptype'] == apptype] for apptype in data_param]
            df_merged = reduce(merge_df, df_list)
            rule_key = param.get('rule')
            rule_param = rule_params_item.get(rule_key)
            log_.info(f"rule_key = {rule_key}, rule_param = {rule_param}")

            task_list = []
            for region in region_code_list:
                t = Thread(target=process_with_region,
                           args=(region, df_merged, app_type, data_key, rule_key, rule_param, now_date, now_h)
                           )
                t.start()
                task_list.append(t)
            for t in task_list:
                t.join()
            log_.info(f"param = {param} end!")
        log_.info(f"app_type = {app_type} end!")
    """

    # for app_type, params in rule_params.items():
    #     log_.info(f"app_type = {app_type}")
    #     for data_key, data_param in params['data_params'].items():
    #         log_.info(f"data_key = {data_key}, data_param = {data_param}")
    #         df_list = [feature_df[feature_df['apptype'] == apptype] for apptype in data_param]
    #         df_merged = reduce(merge_df, df_list)
    #         for rule_key, rule_param in params['rule_params'].items():
    #             log_.info(f"rule_key = {rule_key}, rule_param = {rule_param}")
    #             task_list = [
    #                 gevent.spawn(process_with_region, region, df_merged, app_type, data_key, rule_key, rule_param, now_date, now_h)
    #                 for region in region_code_list
    #             ]
    #             gevent.joinall(task_list)

    # rank
    # for key, value in rule_params.items():
    #     log_.info(f"rule = {key}, param = {value}")
    #     for region in region_code_list:
    #         log_.info(f"region = {region}")
    #         # 计算score
    #         region_df = feature_df[feature_df['code'] == region]
    #         log_.info(f'region_df count = {len(region_df)}')
    #         score_df = cal_score(df=region_df, param=value)
    #         video_rank(df=score_df, now_date=now_date, now_h=now_h, rule_key=key, param=value, region=region)
    #         # to-csv
    #         score_filename = f"score_{region}_{key}_{datetime.datetime.strftime(now_date, '%Y%m%d%H')}.csv"
    #         score_df.to_csv(f'./data/{score_filename}')
    #         # to-logs
    #         log_.info({"date": datetime.datetime.strftime(now_date, '%Y%m%d%H'),
    #                    "region_code": region,
    #                    "redis_key_prefix": config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H,
    #                    "rule_key": key,
    #                    # "score_df": score_df[['videoid', 'score']]
    #                    }
    #                   )


def h_rank_bottom(now_date, now_h, rule_params, region_code_list, rule_rank_h_flag):
    """未按时更新数据，用上一小时结果作为当前小时的数据"""
    # 获取rov模型结果
    redis_helper = RedisHelper()
    if now_h == 0:
        redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
        redis_h = 23
    else:
        redis_dt = datetime.datetime.strftime(now_date, '%Y%m%d')
        redis_h = now_h - 1

    # 以上一小时的地域分组数据作为当前小时的数据
    key_prefix = config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H
    rule_params_item = rule_params.get('rule_params')
    for param in rule_params.get('params_list'):
        data_key = param.get('data')
        rule_key = param.get('rule')
        rule_param = rule_params_item.get(rule_key)
        log_.info(f"data_key = {data_key}, rule_key = {rule_key}, rule_param = {rule_param}")
        region_24h_rule_key = rule_param.get('region_24h_rule_key', 'rule1')
        by_24h_rule_key = rule_param.get('24h_rule_key', None)
        by_48h_rule_key = rule_param.get('48h_rule_key', None)
        # 涉政视频过滤
        political_filter = param.get('political_filter', None)
        for region in region_code_list:
            log_.info(f"region = {region}")
            key_name = f"{key_prefix}{region}:{data_key}:{rule_key}:{redis_dt}:{redis_h}"
            initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
            if initial_data is None:
                initial_data = []
            final_data = dict()
            h_video_ids = []
            for video_id, score in initial_data:
                final_data[video_id] = score
                h_video_ids.append(int(video_id))
            # 存入对应的redis
            final_key_name = \
                f"{key_prefix}{region}:{data_key}:{rule_key}:{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
            if len(final_data) > 0:
                redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=2 * 24 * 3600)
            # 与其他召回视频池去重，存入对应的redis
            dup_to_redis(h_video_ids=h_video_ids, now_date=now_date, now_h=now_h, rule_key=rule_key,
                         region_24h_rule_key=region_24h_rule_key, region=region,
                         data_key=data_key, by_24h_rule_key=by_24h_rule_key,
                         by_48h_rule_key=by_48h_rule_key, rule_rank_h_flag=rule_rank_h_flag,
                         political_filter=political_filter)
        # 特殊城市视频数据准备
        for region, city_list in config_.REGION_CITY_MAPPING.items():
            t = [
                gevent.spawn(
                    copy_data_for_city,
                    region, city_code, data_key, rule_key, now_date, now_h
                )
                for city_code in city_list
            ]
            gevent.joinall(t)


def h_timer_check():
    try:
        rule_rank_h_flag = sys.argv[1]
        if rule_rank_h_flag == '48h':
            rule_params = config_.RULE_PARAMS_REGION_APP_TYPE_48H
        else:
            rule_params = config_.RULE_PARAMS_REGION_APP_TYPE
        project = config_.PROJECT_REGION_APP_TYPE
        table = config_.TABLE_REGION_APP_TYPE
        region_code_list = [code for region, code in region_code.items()]
        now_date = datetime.datetime.today()
        log_.info(f"now_date: {datetime.datetime.strftime(now_date, '%Y%m%d%H')}, rule_rank_h_flag: {rule_rank_h_flag}")
        now_h = datetime.datetime.now().hour
        now_min = datetime.datetime.now().minute
        if now_h == 0:
            h_rank_bottom(now_date=now_date, now_h=now_h, rule_params=rule_params, region_code_list=region_code_list,
                          rule_rank_h_flag=rule_rank_h_flag)
            return
        # 查看当前小时更新的数据是否已准备好
        h_data_count = h_data_check(project=project, table=table, now_date=now_date)
        if h_data_count > 0:
            log_.info(f'region_h_data_count = {h_data_count}')
            # 数据准备好，进行更新
            rank_by_h(now_date=now_date, now_h=now_h, rule_params=rule_params,
                      project=project, table=table, region_code_list=region_code_list, rule_rank_h_flag=rule_rank_h_flag)
            log_.info(f"region_h_data end!")
        elif now_min > 50:
            log_.info('h_recall data is None, use bottom data!')
            h_rank_bottom(now_date=now_date, now_h=now_h, rule_params=rule_params, region_code_list=region_code_list,
                          rule_rank_h_flag=rule_rank_h_flag)
            log_.info(f"region_h_data end!")
        else:
            # 数据没准备好，1分钟后重新检查
            Timer(60, h_timer_check).start()
        send_msg_to_feishu(
            webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
            key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
            msg_text=f"rov-offline{config_.ENV_TEXT} - 推荐视频数据更新完成\n"
                     f"now_date: {datetime.datetime.strftime(now_date, '%Y%m%d')}\n"
                     f"now_h: {now_h}\n"
                     f"finished time: {datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d %H:%M:%S')}"
        )
    except Exception as e:
        log_.error(f"地域分组小时级数据更新失败, exception: {e}, traceback: {traceback.format_exc()}")
        send_msg_to_feishu(
            webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'),
            key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'),
            msg_text=f"rov-offline{config_.ENV_TEXT} - 地域分组小时级数据更新失败\n"
                     f"exception: {e}\n"
                     f"traceback: {traceback.format_exc()}"
        )


if __name__ == '__main__':
    log_.info(f"region_h_data start...")
    h_timer_check()