rov-server/ad_recommend.py

import traceback
import datetime
from utils import RedisHelper
from ad_xgboost_predict import xgboost_predict
from config import set_config
from log import Log
log_ = Log()
config_ = set_config()
redis_helper = RedisHelper()


def get_params(ab_exp_info, ab_test_code):
    """
    根据实验分组给定对应的参数
    :param ab_exp_info: AB实验组参数
    :param ab_test_code: 用户对应的ab组
    :return:
    """
    abtest_id, abtest_config_tag = None, None
    ad_abtest_id_list = [key.split('-')[0] for key in config_.AD_ABTEST_CONFIG]
    # 获取广告实验配置
    config_value_dict = {}
    if ab_exp_info:
        ab_exp_list = ab_exp_info.get('ab_test002', None)
        if ab_exp_list:
            for ab_item in ab_exp_list:
                ab_exp_code = ab_item.get('abExpCode', None)
                if not ab_exp_code:
                    continue
                if ab_exp_code in ad_abtest_id_list:
                    config_value = ab_item.get('configValue', None)
                    if config_value:
                        config_value_dict[str(ab_exp_code)] = eval(str(config_value))

    if len(config_value_dict) > 0:
        for ab_exp_code, config_value in config_value_dict.items():
            for tag, value in config_value.items():
                if ab_test_code in value:
                    abtest_id = ab_exp_code
                    abtest_config_tag = tag
                    break

    return abtest_id, abtest_config_tag


def get_threshold(abtest_id, abtest_config_tag, ab_test_code, mid_group, care_model_status, abtest_param):
    """获取对应的阈值"""
    # 判断是否是关怀模式实验
    care_model_status_param = abtest_param.get('care_model_status_param', None)
    care_model_ab_mid_group = abtest_param.get('care_model_ab_mid_group', [])
    if care_model_status_param is None:
        # 无关怀模式实验
        threshold_key_name_prefix = config_.KEY_NAME_PREFIX_AD_THRESHOLD
    else:
        # 关怀模式实验
        if care_model_status is None or len(care_model_ab_mid_group) == 0 or care_model_status == 'null':
            # 参数缺失，走默认
            threshold_key_name_prefix = config_.KEY_NAME_PREFIX_AD_THRESHOLD
        elif int(care_model_status) == int(care_model_status_param) and mid_group in care_model_ab_mid_group:
            # 实验匹配，获取对应的阈值
            threshold_key_name_prefix = config_.KEY_NAME_PREFIX_AD_THRESHOLD_CARE_MODEL
        else:
            threshold_key_name_prefix = config_.KEY_NAME_PREFIX_AD_THRESHOLD

    threshold_key_name = f"{threshold_key_name_prefix}{abtest_id}:{abtest_config_tag}:{ab_test_code}:{mid_group}"
    threshold = redis_helper.get_data_from_redis(key_name=threshold_key_name)
    if threshold is None:
        threshold = 0
    else:
        threshold = float(threshold)
    return threshold


def predict_mid_video_res(now_date, mid, video_id, abtest_param, abtest_id, abtest_config_tag, ab_test_code, care_model_status):
    now_dt = datetime.datetime.strftime(now_date, '%Y%m%d')
    user_data_key = abtest_param['user'].get('data')
    user_rule_key = abtest_param['user'].get('rule')
    video_data_key = abtest_param['video'].get('data')
    group_class_key = abtest_param.get('group_class_key')
    no_ad_mid_group_list = abtest_param.get('no_ad_mid_group_list', [])

    # 判断mid所属分组
    mid_group_key_name = f"{config_.KEY_NAME_PREFIX_MID_GROUP}{group_class_key}:{mid}"
    mid_group = redis_helper.get_data_from_redis(key_name=mid_group_key_name)
    if mid_group is None:
        mid_group = 'mean_group'

    # 判断用户是否在免广告用户组列表中
    if mid_group in no_ad_mid_group_list:
        # 在免广告用户组列表中，则不出广告
        ad_predict = 1
        result = {
            'mid_group': mid_group,
            'ad_predict': ad_predict
        }
    else:
        # 获取用户组分享率
        group_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_GROUP}{user_data_key}:{user_rule_key}:{now_dt}"
        if not redis_helper.key_exists(group_share_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            group_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_GROUP}{user_data_key}:{user_rule_key}:{redis_dt}"
        group_share_rate = redis_helper.get_score_with_value(key_name=group_share_rate_key, value=mid_group)
        # 获取视频分享率
        video_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{video_data_key}:{now_dt}"
        if not redis_helper.key_exists(video_share_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            video_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{video_data_key}:{redis_dt}"
        video_share_rate = redis_helper.get_score_with_value(key_name=video_share_rate_key, value=int(video_id))
        if video_share_rate is None:
            video_share_rate = redis_helper.get_score_with_value(key_name=video_share_rate_key, value=-1)

        # 计算 mid-video 分享率
        if group_share_rate is None or video_share_rate is None:
            return None
        mid_video_predict_res = float(group_share_rate) * float(video_share_rate)

        # 获取对应的阈值
        threshold = get_threshold(
            abtest_id=abtest_id,
            abtest_config_tag=abtest_config_tag,
            ab_test_code=ab_test_code,
            mid_group=mid_group,
            care_model_status=care_model_status,
            abtest_param=abtest_param
        )
        # 阈值判断
        if mid_video_predict_res > threshold:
            # 大于阈值，出广告
            ad_predict = 2
        else:
            # 否则，不出广告
            ad_predict = 1
        result = {
            'mid_group': mid_group,
            'group_share_rate': group_share_rate,
            'video_share_rate': video_share_rate,
            'mid_video_predict_res': mid_video_predict_res,
            'threshold': threshold,
            'ad_predict': ad_predict}
    return result


def predict_mid_video_res_with_add(now_date, mid, video_id, abtest_param, abtest_id, abtest_config_tag, ab_test_code, care_model_status):
    now_dt = datetime.datetime.strftime(now_date, '%Y%m%d')

    # 判断mid所属分组
    group_class_key = abtest_param.get('group_class_key')
    mid_group_key_name = f"{config_.KEY_NAME_PREFIX_MID_GROUP}{group_class_key}:{mid}"
    mid_group = redis_helper.get_data_from_redis(key_name=mid_group_key_name)
    if mid_group is None:
        mid_group = 'mean_group'

    # 判断用户是否在免广告用户组列表中
    no_ad_mid_group_list = abtest_param.get('no_ad_mid_group_list', [])
    if mid_group in no_ad_mid_group_list:
        # 在免广告用户组列表中，则不出广告
        ad_predict = 1
        result = {
            'mid_group': mid_group,
            'ad_predict': ad_predict
        }
    else:
        # 获取用户组出广告后分享的概率
        share_user_data_key = abtest_param['share']['user'].get('data')
        share_user_rule_key = abtest_param['share']['user'].get('rule')
        group_share_rate_key = \
            f"{config_.KEY_NAME_PREFIX_AD_GROUP}{share_user_data_key}:{share_user_rule_key}:{now_dt}"
        if not redis_helper.key_exists(group_share_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            group_share_rate_key = \
                f"{config_.KEY_NAME_PREFIX_AD_GROUP}{share_user_data_key}:{share_user_rule_key}:{redis_dt}"
        group_share_rate = redis_helper.get_score_with_value(key_name=group_share_rate_key, value=mid_group)

        # 获取视频出广告后分享的概率
        share_video_data_key = abtest_param['share']['video'].get('data')
        video_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{share_video_data_key}:{now_dt}"
        if not redis_helper.key_exists(video_share_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            video_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{share_video_data_key}:{redis_dt}"
        video_share_rate = redis_helper.get_score_with_value(key_name=video_share_rate_key, value=int(video_id))
        if video_share_rate is None:
            video_share_rate = redis_helper.get_score_with_value(key_name=video_share_rate_key, value=-1)

        # 获取用户组出广告后不直接跳出的概率
        out_user_data_key = abtest_param['out']['user'].get('data')
        out_user_rule_key = abtest_param['out']['user'].get('rule')
        group_out_rate_key = \
            f"{config_.KEY_NAME_PREFIX_AD_GROUP}{out_user_data_key}:{out_user_rule_key}:{now_dt}"
        if not redis_helper.key_exists(group_out_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            group_out_rate_key = \
                f"{config_.KEY_NAME_PREFIX_AD_GROUP}{out_user_data_key}:{out_user_rule_key}:{redis_dt}"
        group_out_rate = redis_helper.get_score_with_value(key_name=group_out_rate_key, value=mid_group)

        # 获取视频出广告后不直接跳出的概率
        out_video_data_key = abtest_param['out']['video'].get('data')
        video_out_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{out_video_data_key}:{now_dt}"
        if not redis_helper.key_exists(video_out_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            video_out_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{out_video_data_key}:{redis_dt}"
        video_out_rate = redis_helper.get_score_with_value(key_name=video_out_rate_key, value=int(video_id))
        if video_out_rate is None:
            video_out_rate = redis_helper.get_score_with_value(key_name=video_out_rate_key, value=-1)

        # 计算 mid-video 预测值
        if group_share_rate is None or video_share_rate is None or group_out_rate is None or video_out_rate is None:
            return None
        # 加权融合
        share_weight = abtest_param['mix_param']['share_weight']
        out_weight = abtest_param['mix_param']['out_weight']
        group_rate = share_weight * float(group_share_rate) + out_weight * float(group_out_rate)
        video_rate = share_weight * float(video_share_rate) + out_weight * float(video_out_rate)
        mid_video_predict_res = group_rate * video_rate

        # 获取对应的阈值
        threshold = get_threshold(
            abtest_id=abtest_id,
            abtest_config_tag=abtest_config_tag,
            ab_test_code=ab_test_code,
            mid_group=mid_group,
            care_model_status=care_model_status,
            abtest_param=abtest_param
        )
        # 阈值判断
        if mid_video_predict_res > threshold:
            # 大于阈值，出广告
            ad_predict = 2
        else:
            # 否则，不出广告
            ad_predict = 1
        result = {
            'mid_group': mid_group,
            'group_share_rate': group_share_rate,
            'video_share_rate': video_share_rate,
            'group_out_rate': group_out_rate,
            'video_out_rate': video_out_rate,
            'group_rate': group_rate,
            'video_rate': video_rate,
            'mid_video_predict_res': mid_video_predict_res,
            'threshold': threshold,
            'ad_predict': ad_predict}
    return result


def predict_mid_video_res_with_multiply(now_date, mid, video_id, abtest_param, abtest_id, abtest_config_tag, ab_test_code, care_model_status):
    now_dt = datetime.datetime.strftime(now_date, '%Y%m%d')

    # 判断mid所属分组
    group_class_key = abtest_param.get('group_class_key')
    mid_group_key_name = f"{config_.KEY_NAME_PREFIX_MID_GROUP}{group_class_key}:{mid}"
    mid_group = redis_helper.get_data_from_redis(key_name=mid_group_key_name)
    if mid_group is None:
        mid_group = 'mean_group'

    # 判断用户是否在免广告用户组列表中
    no_ad_mid_group_list = abtest_param.get('no_ad_mid_group_list', [])
    if mid_group in no_ad_mid_group_list:
        # 在免广告用户组列表中，则不出广告
        ad_predict = 1
        result = {
            'mid_group': mid_group,
            'ad_predict': ad_predict
        }
    else:
        # 获取用户组出广告后分享的概率
        share_user_data_key = abtest_param['share']['user'].get('data')
        share_user_rule_key = abtest_param['share']['user'].get('rule')
        group_share_rate_key = \
            f"{config_.KEY_NAME_PREFIX_AD_GROUP}{share_user_data_key}:{share_user_rule_key}:{now_dt}"
        if not redis_helper.key_exists(group_share_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            group_share_rate_key = \
                f"{config_.KEY_NAME_PREFIX_AD_GROUP}{share_user_data_key}:{share_user_rule_key}:{redis_dt}"
        group_share_rate = redis_helper.get_score_with_value(key_name=group_share_rate_key, value=mid_group)

        # 获取视频出广告后分享的概率
        share_video_data_key = abtest_param['share']['video'].get('data')
        video_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{share_video_data_key}:{now_dt}"
        if not redis_helper.key_exists(video_share_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            video_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{share_video_data_key}:{redis_dt}"
        video_share_rate = redis_helper.get_score_with_value(key_name=video_share_rate_key, value=int(video_id))
        if video_share_rate is None:
            video_share_rate = redis_helper.get_score_with_value(key_name=video_share_rate_key, value=-1)

        # 获取用户组出广告后不直接跳出的概率
        out_user_data_key = abtest_param['out']['user'].get('data')
        out_user_rule_key = abtest_param['out']['user'].get('rule')
        group_out_rate_key = \
            f"{config_.KEY_NAME_PREFIX_AD_GROUP}{out_user_data_key}:{out_user_rule_key}:{now_dt}"
        if not redis_helper.key_exists(group_out_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            group_out_rate_key = \
                f"{config_.KEY_NAME_PREFIX_AD_GROUP}{out_user_data_key}:{out_user_rule_key}:{redis_dt}"
        group_out_rate = redis_helper.get_score_with_value(key_name=group_out_rate_key, value=mid_group)

        # 获取视频出广告后不直接跳出的概率
        out_video_data_key = abtest_param['out']['video'].get('data')
        video_out_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{out_video_data_key}:{now_dt}"
        if not redis_helper.key_exists(video_out_rate_key):
            redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
            video_out_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{out_video_data_key}:{redis_dt}"
        video_out_rate = redis_helper.get_score_with_value(key_name=video_out_rate_key, value=int(video_id))
        if video_out_rate is None:
            video_out_rate = redis_helper.get_score_with_value(key_name=video_out_rate_key, value=-1)

        # 计算 mid-video 预测值
        if group_share_rate is None or video_share_rate is None or group_out_rate is None or video_out_rate is None:
            return None
        # 乘积融合
        group_rate = float(group_share_rate) * float(group_out_rate)
        video_rate = float(video_share_rate) * float(video_out_rate)
        mid_video_predict_res = group_rate * video_rate

        # 获取对应的阈值
        threshold = get_threshold(
            abtest_id=abtest_id,
            abtest_config_tag=abtest_config_tag,
            ab_test_code=ab_test_code,
            mid_group=mid_group,
            care_model_status=care_model_status,
            abtest_param=abtest_param
        )
        # 阈值判断
        if mid_video_predict_res > threshold:
            # 大于阈值，出广告
            ad_predict = 2
        else:
            # 否则，不出广告
            ad_predict = 1
        result = {
            'mid_group': mid_group,
            'group_share_rate': group_share_rate,
            'video_share_rate': video_share_rate,
            'group_out_rate': group_out_rate,
            'video_out_rate': video_out_rate,
            'group_rate': group_rate,
            'video_rate': video_rate,
            'mid_video_predict_res': mid_video_predict_res,
            'threshold': threshold,
            'ad_predict': ad_predict}
    return result


def ad_recommend_predict(model, app_type, mid, video_id, ab_exp_info, ab_test_code, care_model_status):
    """
    广告推荐预测
    :param app_type: app_type
    :param mid: mid
    :param video_id: video_id
    :param ab_exp_info: AB实验组参数
    :param ab_test_code: 用户对应的ab组
    :param care_model_status: 用户关怀模式状态 1-未开启，2-开启
    :return: ad_predict, type-int, 1-不发放广告,2-发放广告
    """
    try:
        now_date = datetime.datetime.today()
        # now_dt = datetime.datetime.strftime(now_date, '%Y%m%d')
        # 获取实验参数
        abtest_id, abtest_config_tag = get_params(ab_exp_info=ab_exp_info, ab_test_code=ab_test_code)
        if abtest_id is None or abtest_config_tag is None:
            return None
        abtest_param = config_.AD_ABTEST_CONFIG.get(f'{abtest_id}-{abtest_config_tag}')
        if abtest_param is None:
            return None
        predict_model = abtest_param.get('predict_model', None)
        threshold_mix_func = abtest_param.get('threshold_mix_func', None)
        if predict_model == 'xgb':
            result = xgboost_predict(
                model=model,
                app_type=app_type,
                mid=mid,
                video_id=video_id,
                abtest_id=abtest_id,
                ab_test_code=ab_test_code
            )
        elif threshold_mix_func == 'add':
            result = predict_mid_video_res_with_add(
                now_date=now_date,
                mid=mid,
                video_id=video_id,
                abtest_param=abtest_param,
                abtest_id=abtest_id,
                abtest_config_tag=abtest_config_tag,
                ab_test_code=ab_test_code,
                care_model_status=care_model_status
            )
        elif threshold_mix_func == 'multiply':
            result = predict_mid_video_res_with_multiply(
                now_date=now_date,
                mid=mid,
                video_id=video_id,
                abtest_param=abtest_param,
                abtest_id=abtest_id,
                abtest_config_tag=abtest_config_tag,
                ab_test_code=ab_test_code,
                care_model_status=care_model_status
            )
        else:
            result = predict_mid_video_res(
                now_date=now_date,
                mid=mid,
                video_id=video_id,
                abtest_param=abtest_param,
                abtest_id=abtest_id,
                abtest_config_tag=abtest_config_tag,
                ab_test_code=ab_test_code,
                care_model_status=care_model_status
            )

        # user_data_key = abtest_param['user'].get('data')
        # user_rule_key = abtest_param['user'].get('rule')
        # video_data_key = abtest_param['video'].get('data')
        # group_class_key = abtest_param.get('group_class_key')
        # no_ad_mid_group_list = abtest_param.get('no_ad_mid_group_list', [])
        #
        # # 判断mid所属分组
        # mid_group_key_name = f"{config_.KEY_NAME_PREFIX_MID_GROUP}{group_class_key}:{mid}"
        # mid_group = redis_helper.get_data_from_redis(key_name=mid_group_key_name)
        # if mid_group is None:
        #     mid_group = 'mean_group'
        #
        # # 判断用户是否在免广告用户组列表中
        # if mid_group in no_ad_mid_group_list:
        #     # 在免广告用户组列表中，则不出广告
        #     ad_predict = 1
        #     result = {
        #         'mid_group': mid_group,
        #         'ad_predict': ad_predict
        #     }
        # else:
        #     # 获取用户组分享率
        #     group_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_GROUP}{user_data_key}:{user_rule_key}:{now_dt}"
        #     if not redis_helper.key_exists(group_share_rate_key):
        #         redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
        #         group_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_GROUP}{user_data_key}:{user_rule_key}:{redis_dt}"
        #     group_share_rate = redis_helper.get_score_with_value(key_name=group_share_rate_key, value=mid_group)
        #     # 获取视频分享率
        #     video_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{video_data_key}:{now_dt}"
        #     if not redis_helper.key_exists(video_share_rate_key):
        #         redis_dt = datetime.datetime.strftime(now_date - datetime.timedelta(days=1), '%Y%m%d')
        #         video_share_rate_key = f"{config_.KEY_NAME_PREFIX_AD_VIDEO}{video_data_key}:{redis_dt}"
        #     video_share_rate = redis_helper.get_score_with_value(key_name=video_share_rate_key, value=int(video_id))
        #     if video_share_rate is None:
        #         video_share_rate = redis_helper.get_score_with_value(key_name=video_share_rate_key, value=-1)
        #
        #     # 计算 mid-video 分享率
        #     if group_share_rate is None or video_share_rate is None:
        #         return None
        #     mid_video_share_rate = float(group_share_rate) * float(video_share_rate)
        #
        #     # 获取对应的阈值
        #     threshold = get_threshold(
        #         abtest_id=abtest_id,
        #         abtest_config_tag=abtest_config_tag,
        #         ab_test_code=ab_test_code,
        #         mid_group=mid_group,
        #         care_model_status=care_model_status,
        #         abtest_param=abtest_param
        #     )
        #     # 阈值判断
        #     if mid_video_share_rate > threshold:
        #         # 大于阈值，出广告
        #         ad_predict = 2
        #     else:
        #         # 否则，不出广告
        #         ad_predict = 1
        #     result = {
        #         'mid_group': mid_group,
        #         'group_share_rate': group_share_rate,
        #         'video_share_rate': video_share_rate,
        #         'mid_video_share_rate': mid_video_share_rate,
        #         'threshold': threshold,
        #         'ad_predict': ad_predict}

        return result

    except Exception as e:
        log_.error(traceback.format_exc())
        return None