import datetime import traceback from threading import Timer from utils import RedisHelper, data_check, get_feature_data, send_msg_to_feishu from config import set_config from log import Log config_, _ = set_config() log_ = Log() redis_helper = RedisHelper() features = [ 'apptype', 'group', 'sharerate_all', 'sharerate_ad' ] def predict_user_group_share_rate(user_group_initial_df, dt, data_key, data_param): """预估用户组对应的有广告时分享率""" # 获取对应的用户组特征 user_group_df = user_group_initial_df.copy() user_group_df['apptype'] = user_group_df['apptype'].astype(int) user_group_df = user_group_df[user_group_df['apptype'] == data_param] if data_key == 'data1:1': user_group_df = user_group_df[~user_group_df['group'].isin(config_.NO_AD_MID_GROUP_LIST)] user_group_df['sharerate_all'].fillna(0, inplace=True) user_group_df['sharerate_ad'].fillna(0, inplace=True) user_group_df['sharerate_all'] = user_group_df['sharerate_all'].astype(float) user_group_df['sharerate_ad'] = user_group_df['sharerate_ad'].astype(float) # 获取有广告时所有用户组近30天的分享率 ad_all_group_share_rate = user_group_df[user_group_df['group'] == 'allmids']['sharerate_ad'].values[0] user_group_df = user_group_df[user_group_df['group'] != 'allmids'] # 计算用户组有广告时分享率 user_group_df['group_ad_share_rate'] = \ user_group_df['sharerate_ad'] * float(ad_all_group_share_rate) / user_group_df['sharerate_all'] user_group_df['group_ad_share_rate'].fillna(0, inplace=True) # 结果写入redis key_name = f"{config_.KEY_NAME_PREFIX_AD_GROUP}{data_key}:{dt}" redis_data = {} for index, item in user_group_df.iterrows(): redis_data[item['group']] = item['group_ad_share_rate'] group_ad_share_rate_mean = user_group_df['group_ad_share_rate'].mean() redis_data['mean_group'] = group_ad_share_rate_mean if len(redis_data) > 0: redis_helper = RedisHelper() redis_helper.add_data_with_zset(key_name=key_name, data=redis_data, expire_time=2 * 24 * 3600) return user_group_df def update_users_data(project, table, dt, update_params): """预估用户组有广告时分享率""" # 获取用户组特征 user_group_initial_df = get_feature_data(project=project, table=table, features=features, dt=dt) for data_key, data_param in update_params.items(): log_.info(f"data_key = {data_key} update start...") predict_user_group_share_rate(user_group_initial_df=user_group_initial_df, dt=dt, data_key=data_key, data_param=data_param) log_.info(f"data_key = {data_key} update end!") def timer_check(): try: update_params = config_.AD_USER_DATA_PARAMS project = config_.ad_model_data['users_share_rate'].get('project') table = config_.ad_model_data['users_share_rate'].get('table') now_date = datetime.datetime.today() dt = datetime.datetime.strftime(now_date, '%Y%m%d') log_.info(f"now_date: {dt}") now_min = datetime.datetime.now().minute # 查看当前更新的数据是否已准备好 data_count = data_check(project=project, table=table, dt=dt) if data_count > 0: log_.info(f"ad user group data count = {data_count}") # 数据准备好,进行更新 update_users_data(project=project, table=table, dt=dt, update_params=update_params) log_.info(f"ad user group data update end!") # elif now_min > 45: # log_.info('ad user group data is None!') # send_msg_to_feishu( # webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'), # key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'), # msg_text=f"rov-offline{config_.ENV_TEXT} - 用户组分享率数据未准备好!\n" # f"traceback: {traceback.format_exc()}" # ) else: # 数据没准备好,1分钟后重新检查 Timer(60, timer_check).start() except Exception as e: log_.error(f"用户组分享率预测数据更新失败, exception: {e}, traceback: {traceback.format_exc()}") send_msg_to_feishu( webhook=config_.FEISHU_ROBOT['server_robot'].get('webhook'), key_word=config_.FEISHU_ROBOT['server_robot'].get('key_word'), msg_text=f"rov-offline{config_.ENV_TEXT} - 用户组分享率预测数据更新失败\n" f"exception: {e}\n" f"traceback: {traceback.format_exc()}" ) if __name__ == '__main__': timer_check()