import json
import time
import multiprocessing
import traceback
import hashlib
from datetime import datetime
import config
from log import Log
from config import set_config
from video_recall import PoolRecall
from video_rank import video_rank, bottom_strategy, video_rank_by_w_h_rate, video_rank_with_old_video, bottom_strategy2
from db_helper import RedisHelper
import gevent
from utils import FilterVideos
import ast
log_ = Log()
config_ = set_config()
def relevant_video_top_recommend(request_id, app_type, mid, uid, head_vid, videos, size):
"""
相关推荐强插 运营给定置顶相关性视频
:param request_id: request_id
:param app_type: 产品标识 type-int
:param mid: mid
:param uid: uid
:param head_vid: 相关推荐头部视频id type-int
:param videos: 当前相关推荐结果 type-list
:param size: 返回视频个数 type-int
:return: rank_result
"""
# 获取头部视频对应的相关性视频
key_name = '{}{}'.format(config_.RELEVANT_VIDEOS_WITH_OP_KEY_NAME, head_vid)
redis_helper = RedisHelper()
relevant_videos = redis_helper.get_data_from_redis(key_name=key_name)
if relevant_videos is None:
# 该视频没有指定的相关性视频
return videos
relevant_videos = json.loads(relevant_videos)
# 按照指定顺序排序
relevant_videos_sorted = sorted(relevant_videos, key=lambda x: x['order'], reverse=False)
# 过滤
relevant_video_ids = [int(item['recommend_vid']) for item in relevant_videos_sorted]
filter_helper = FilterVideos(request_id=request_id, app_type=app_type, video_ids=relevant_video_ids, mid=mid, uid=uid)
filtered_ids = filter_helper.filter_videos()
if filtered_ids is None:
return videos
# 获取生效中的视频
now = int(time.time())
relevant_videos_in_effect = [
{'videoId': int(item['recommend_vid']), 'pushFrom': config_.PUSH_FROM['relevant_video_op'],
'abCode': config_.AB_CODE['relevant_video_op']}
for item in relevant_videos_sorted
if item['start_time'] < now < item['finish_time'] and int(item['recommend_vid']) in filtered_ids
]
if len(relevant_videos_in_effect) == 0:
return videos
# 与现有排序结果 进行合并重排
# 获取现有排序结果中流量池视频 及其位置
relevant_ids = [item['videoId'] for item in relevant_videos_in_effect]
flow_pool_videos = []
other_videos = []
for i, item in enumerate(videos):
if item.get('pushFrom', None) == config_.PUSH_FROM['flow_recall'] and item.get('videoId') not in relevant_ids:
flow_pool_videos.append((i, item))
elif item.get('videoId') not in relevant_ids:
other_videos.append(item)
else:
continue
# 重排，保持流量池视频位置不变
rank_result = relevant_videos_in_effect + other_videos
for i, item in flow_pool_videos:
rank_result.insert(i, item)
return rank_result[:size]
def video_position_recommend(request_id, mid, uid, app_type, videos):
# videos = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
# algo_type=algo_type, client_info=client_info)
redis_helper = RedisHelper()
pos1_vids = redis_helper.get_data_from_redis(config.BaseConfig.RECALL_POSITION1_KEY_NAME)
pos2_vids = redis_helper.get_data_from_redis(config.BaseConfig.RECALL_POSITION2_KEY_NAME)
if pos1_vids is not None:
pos1_vids = ast.literal_eval(pos1_vids)
if pos2_vids is not None:
pos2_vids = ast.literal_eval(pos2_vids)
pos1_vids = [] if pos1_vids is None else pos1_vids
pos2_vids = [] if pos2_vids is None else pos2_vids
pos1_vids = [int(i) for i in pos1_vids]
pos2_vids = [int(i) for i in pos2_vids]
filter_1 = FilterVideos(request_id=request_id, app_type=app_type, video_ids=pos1_vids, mid=mid, uid=uid)
filter_2 = FilterVideos(request_id=request_id, app_type=app_type, video_ids=pos2_vids, mid=mid, uid=uid)
t = [gevent.spawn(filter_1.filter_videos), gevent.spawn(filter_2.filter_videos)]
gevent.joinall(t)
filted_list = [i.get() for i in t]
pos1_vids = filted_list[0]
pos2_vids = filted_list[1]
videos = positon_duplicate(pos1_vids, pos2_vids, videos)
if pos1_vids is not None and len(pos1_vids) >0 :
videos.insert(0, {'videoId': int(pos1_vids[0]), 'rovScore': 100,
'pushFrom': config_.PUSH_FROM['position_insert'], 'abCode': config_.AB_CODE['position_insert']})
if pos2_vids is not None and len(pos2_vids) >0 :
videos.insert(1, {'videoId': int(pos2_vids[0]), 'rovScore': 100,
'pushFrom': config_.PUSH_FROM['position_insert'], 'abCode': config_.AB_CODE['position_insert']})
return videos[:10]
def positon_duplicate(pos1_vids, pos2_vids, videos):
s = set()
if pos1_vids is not None and len(pos1_vids) >0:
s.add(int(pos1_vids[0]))
if pos2_vids is not None and len(pos2_vids) >0:
s.add(int(pos2_vids[0]))
l = []
for item in videos:
if item['videoId'] in s:
continue
else:
l.append(item)
return l
def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, algo_type, client_info,
expire_time=24*3600, ab_code=config_.AB_CODE['initial'], rule_key='', data_key='',
no_op_flag=False, old_video_index=-1, video_id=None, params=None):
"""
首页线上推荐逻辑
:param request_id: request_id
:param mid: mid type-string
:param uid: uid type-string
:param size: 请求视频数量 type-int
:param top_K: 保证topK为召回池视频 type-int
:param flow_pool_P: size-top_K视频为流量池视频的概率 type-float
:param app_type: 产品标识 type-int
:param algo_type: 算法类型 type-string
:param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
:param expire_time: 末位视频记录redis过期时间
:param ab_code: AB实验code
:param video_id: 相关推荐头部视频id
:param params:
:return:
"""
result = {}
# ####### 多进程召回
start_recall = time.time()
# log_.info('====== recall')
'''
cores = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=cores)
pool_recall = PoolRecall(app_type=app_type, mid=mid, uid=uid, ab_code=ab_code)
_, last_rov_recall_key, _ = pool_recall.get_video_last_idx()
pool_list = [
# rov召回池
pool.apply_async(pool_recall.rov_pool_recall, (size,)),
# 流量池
pool.apply_async(pool_recall.flow_pool_recall, (size,))
]
recall_result_list = [p.get() for p in pool_list]
pool.close()
pool.join()
'''
recall_result_list = []
pool_recall = PoolRecall(request_id=request_id,
app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
client_info=client_info, rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
params=params)
_, last_rov_recall_key, _ = pool_recall.get_video_last_idx()
# # 小时级实验
# if ab_code in [code for _, code in config_.AB_CODE['rank_by_h'].items()]:
# t = [gevent.spawn(pool_recall.rule_recall_by_h, size, expire_time),
# gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
# gevent.spawn(pool_recall.flow_pool_recall, size)]
# # 小时级实验
# elif ab_code in [code for _, code in config_.AB_CODE['rank_by_24h'].items()]:
# t = [gevent.spawn(pool_recall.rov_pool_recall_by_h, size, expire_time),
# gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
# gevent.spawn(pool_recall.flow_pool_recall, size)]
# 地域分组实验
if ab_code in [code for _, code in config_.AB_CODE['region_rank_by_h'].items()]:
if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time)]
else:
t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time),
gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
gevent.spawn(pool_recall.flow_pool_recall, size)]
# 最惊奇相关推荐实验
# elif ab_code == config_.AB_CODE['top_video_relevant_appType_19']:
# t = [gevent.spawn(pool_recall.relevant_recall_19, video_id, size, expire_time),
# gevent.spawn(pool_recall.flow_pool_recall_18_19, size)]
# 最惊奇完整影视实验
# elif ab_code == config_.AB_CODE['whole_movies']:
# t = [gevent.spawn(pool_recall.rov_pool_recall_19, size, expire_time)]
# 最惊奇/老好看实验
# elif app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
# t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
# gevent.spawn(pool_recall.flow_pool_recall_18_19, size)]
# # 天级实验
# elif ab_code in [code for _, code in config_.AB_CODE['rank_by_day'].items()]:
# t = [gevent.spawn(pool_recall.rov_pool_recall_by_day, size, expire_time),
# gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
# gevent.spawn(pool_recall.flow_pool_recall, size)]
# 老视频实验
# elif ab_code in [config_.AB_CODE['old_video']]:
# t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
# gevent.spawn(pool_recall.flow_pool_recall, size),
# gevent.spawn(pool_recall.old_videos_recall, size)]
else:
if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time)]
else:
t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
gevent.spawn(pool_recall.flow_pool_recall, size)]
gevent.joinall(t)
recall_result_list = [i.get() for i in t]
# end_recall = time.time()
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'recall',
# 'recall_result': recall_result_list,
# 'executeTime': (time.time() - start_recall) * 1000
# })
result['recallResult'] = recall_result_list
result['recallTime'] = (time.time() - start_recall) * 1000
# ####### 排序
start_rank = time.time()
# log_.info('====== rank')
if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
if ab_code in [
config_.AB_CODE['rov_rank_appType_18_19'],
config_.AB_CODE['rov_rank_appType_19'],
config_.AB_CODE['top_video_relevant_appType_19']
]:
data = {
'rov_pool_recall': recall_result_list[0],
'flow_pool_recall': recall_result_list[1]
}
else:
data = {
'rov_pool_recall': recall_result_list[0],
'flow_pool_recall': []
}
else:
if recall_result_list[1]:
redis_helper = RedisHelper()
quick_flow_pool_P = redis_helper.get_data_from_redis(
key_name=f"{config_.QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX}{config_.QUICK_FLOW_POOL_ID}"
)
if quick_flow_pool_P:
flow_pool_P = quick_flow_pool_P
data = {
'rov_pool_recall': recall_result_list[0],
'flow_pool_recall': recall_result_list[1]
}
else:
data = {
'rov_pool_recall': recall_result_list[0],
'flow_pool_recall': recall_result_list[2]
}
rank_result = video_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
# 老视频实验
# if ab_code in [config_.AB_CODE['old_video']]:
# rank_result = video_rank_with_old_video(rank_result=rank_result, old_video_recall=recall_result_list[2],
# size=size, top_K=top_K, old_video_index=old_video_index)
# end_rank = time.time()
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'rank',
# 'rank_result': rank_result,
# 'executeTime': (time.time() - start_rank) * 1000
# })
result['rankResult'] = rank_result
result['rankTime'] = (time.time() - start_rank) * 1000
if not rank_result:
# 兜底策略
# log_.info('====== bottom strategy')
start_bottom = time.time()
if ab_code == config_.AB_CODE['region_rank_by_h'].get('abtest_130'):
rank_result = bottom_strategy2(
size=size, app_type=app_type, mid=mid, uid=uid, ab_code=ab_code, client_info=client_info, params=params
)
else:
rank_result = bottom_strategy(
request_id=request_id, size=size, app_type=app_type, ab_code=ab_code, params=params
)
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'bottom',
# 'bottom_result': rank_result,
# 'executeTime': (time.time() - start_bottom) * 1000
# })
result['bottomResult'] = rank_result
result['bottomTime'] = (time.time() - start_bottom) * 1000
result['rankResult'] = rank_result
return result, last_rov_recall_key
# return rank_result, last_rov_recall_key
def ab_test_op(rank_result, ab_code_list, app_type, mid, uid, **kwargs):
"""
对排序后的结果 按照AB实验进行对应的分组操作
:param rank_result: 排序后的结果
:param ab_code_list: 此次请求参与的 ab实验组
:param app_type: 产品标识
:param mid: mid
:param uid: uid
:param kwargs: 其他参数
:return:
"""
# ####### 视频宽高比AB实验
# 对内容精选进行 视频宽高比分发实验
# if config_.AB_CODE['w_h_rate'] in ab_code_list and app_type in config_.AB_TEST.get('w_h_rate', []):
# rank_result = video_rank_by_w_h_rate(videos=rank_result)
# log_.info('app_type: {}, mid: {}, uid: {}, rank_by_w_h_rate_result: {}'.format(
# app_type, mid, uid, rank_result))
# 按position位置排序
if config_.AB_CODE['position_insert'] in ab_code_list and app_type in config_.AB_TEST.get('position_insert', []):
rank_result = video_position_recommend(mid, uid, app_type, rank_result)
print('===========================')
print(rank_result)
log_.info('app_type: {}, mid: {}, uid: {}, rank_by_position_insert_result: {}'.format(
app_type, mid, uid, rank_result))
# 相关推荐强插
# if config_.AB_CODE['relevant_video_op'] in ab_code_list \
# and app_type in config_.AB_TEST.get('relevant_video_op', []):
# head_vid = kwargs['head_vid']
# size = kwargs['size']
# rank_result = relevant_video_top_recommend(
# app_type=app_type, mid=mid, uid=uid, head_vid=head_vid, videos=rank_result, size=size
# )
# log_.info('app_type: {}, mid: {}, uid: {}, head_vid: {}, rank_by_relevant_video_op_result: {}'.format(
# app_type, mid, uid, head_vid, rank_result))
return rank_result
def update_redis_data(result, app_type, mid, last_rov_recall_key, top_K, expire_time=24*3600):
"""
根据最终的排序结果更新相关redis数据
:param result: 排序结果
:param app_type: 产品标识
:param mid: mid
:param last_rov_recall_key: 用户上一次在rov召回池对应的位置 redis key
:param top_K: 保证topK为召回池视频 type-int
:param expire_time: 末位视频记录redis过期时间
:return: None
"""
# ####### redis数据刷新
try:
redis_helper = RedisHelper()
# log_.info('====== update redis')
if mid and mid != 'null':
# mid为空时，不做预曝光和定位数据更新
# 预曝光数据同步刷新到Redis, 过期时间为0.5h
preview_key_name = f"{config_.PREVIEW_KEY_PREFIX}{app_type}:{mid}"
preview_video_ids = [int(item['videoId']) for item in result]
if preview_video_ids:
# log_.error('key_name = {} \n values = {}'.format(preview_key_name, tuple(preview_video_ids)))
redis_helper.add_data_with_set(key_name=preview_key_name, values=tuple(preview_video_ids), expire_time=30 * 60)
# log_.info('preview redis update success!')
# 将此次获取的ROV召回池top_K末位视频id同步刷新到Redis中，方便下次快速定位到召回位置，过期时间为1天
rov_recall_video = [item['videoId'] for item in result[:top_K]
if item['pushFrom'] == config_.PUSH_FROM['rov_recall']]
if len(rov_recall_video) > 0:
if app_type == config_.APP_TYPE['APP']:
key_name = config_.UPDATE_ROV_KEY_NAME_APP
else:
key_name = config_.UPDATE_ROV_KEY_NAME
if not redis_helper.get_score_with_value(key_name=key_name, value=rov_recall_video[-1]):
redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=rov_recall_video[-1],
expire_time=expire_time)
# log_.info('last video redis update success!')
# 将此次获取的 地域分组小时级数据列表 中的视频id同步刷新到redis中，方便下次快速定位到召回位置
rov_recall_h_video = [item['videoId'] for item in result[:top_K]
if item['pushFrom'] == config_.PUSH_FROM['rov_recall_region_h']]
if len(rov_recall_h_video) > 0:
last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_H_PREFIX}{app_type}:{mid}'
redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_h_video[-1],
expire_time=expire_time)
# 将此次获取的 地域分组相对24h数据列表 中的视频id同步刷新到redis中，方便下次快速定位到召回位置
rov_recall_24h_dup1_video = [item['videoId'] for item in result[:top_K]
if item['pushFrom'] == config_.PUSH_FROM['rov_recall_region_24h']]
if len(rov_recall_24h_dup1_video) > 0:
last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP1_24H_PREFIX}{app_type}:{mid}'
redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup1_video[-1],
expire_time=expire_time)
# 将此次获取的 相对24h筛选数据列表 中的视频id同步刷新到redis中，方便下次快速定位到召回位置
rov_recall_24h_dup2_video = [item['videoId'] for item in result[:top_K]
if item['pushFrom'] == config_.PUSH_FROM['rov_recall_24h']]
if len(rov_recall_24h_dup2_video) > 0:
last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP2_24H_PREFIX}{app_type}:{mid}'
redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup2_video[-1],
expire_time=expire_time)
# 将此次获取的 相对24h筛选后剩余数据列表 中的视频id同步刷新到redis中，方便下次快速定位到召回位置
rov_recall_24h_dup3_video = [item['videoId'] for item in result[:top_K]
if item['pushFrom'] == config_.PUSH_FROM['rov_recall_24h_dup']]
if len(rov_recall_24h_dup3_video) > 0:
last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP3_24H_PREFIX}{app_type}:{mid}'
redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup3_video[-1],
expire_time=expire_time)
# 将此次获取的 相对48h筛选数据列表 中的视频id同步刷新到redis中，方便下次快速定位到召回位置
rov_recall_48h_dup2_video = [item['videoId'] for item in result[:top_K]
if item['pushFrom'] == config_.PUSH_FROM['rov_recall_48h']]
if len(rov_recall_48h_dup2_video) > 0:
last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP2_48H_PREFIX}{app_type}:{mid}'
redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_48h_dup2_video[-1],
expire_time=expire_time)
# 将此次获取的 相对48h筛选后剩余数据列表 中的视频id同步刷新到redis中，方便下次快速定位到召回位置
rov_recall_48h_dup3_video = [item['videoId'] for item in result[:top_K]
if item['pushFrom'] == config_.PUSH_FROM['rov_recall_48h_dup']]
if len(rov_recall_48h_dup3_video) > 0:
last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP3_48H_PREFIX}{app_type}:{mid}'
redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_48h_dup3_video[-1],
expire_time=expire_time)
# 将此次分发的流量池视频，对 本地分发数-1 进行记录
if app_type not in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
flow_recall_video = [item for item in result if item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
if flow_recall_video:
update_local_distribute_count(flow_recall_video)
# log_.info('update local distribute count success!')
# 限流视频分发数记录
if app_type == config_.APP_TYPE['APP']:
# APP 不计入
return
limit_video_id_list = redis_helper.get_data_from_set(
key_name=f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_SET}{datetime.today().strftime('%Y%m%d')}"
)
if limit_video_id_list is not None:
limit_video_id_list = [int(item) for item in limit_video_id_list]
for item in result:
video_id = item['videoId']
if video_id in limit_video_id_list:
key_name = f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_DISTRIBUTE_COUNT}{video_id}"
redis_helper.setnx_key(key_name=key_name, value=0, expire_time=24*2600)
redis_helper.incr_key(key_name=key_name, amount=1, expire_time=24*3600)
except Exception as e:
log_.error("update redis data fail!")
log_.error(traceback.format_exc())
def update_local_distribute_count(videos):
"""
更新本地分发数
:param videos: 视频列表 type-list [{'videoId':'', 'flowPool':'', 'distributeCount': '',
'rovScore': '', 'pushFrom': 'flow_pool', 'abCode': self.ab_code}, ....]
:return:
"""
try:
redis_helper = RedisHelper()
for item in videos:
key_name = f"{config_.LOCAL_DISTRIBUTE_COUNT_PREFIX}{item['videoId']}:{item['flowPool']}"
# 本地记录的分发数 - 1
redis_helper.decr_key(key_name=key_name, amount=1, expire_time=15 * 60)
# if redis_helper.key_exists(key_name=key_name):
# # 该视频本地有记录，本地记录的分发数 - 1
# redis_helper.decr_key(key_name=key_name, amount=1, expire_time=5 * 60)
# else:
# # 该视频本地无记录，接口获取的分发数 - 1
# redis_helper.incr_key(key_name=key_name, amount=int(item['distributeCount']) - 1, expire_time=5 * 60)
except Exception as e:
log_.error('update_local_distribute_count error...')
log_.error(traceback.format_exc())
def get_recommend_params(recommend_type, ab_exp_info, ab_info_data, page_type=0):
"""
根据实验分组给定对应的推荐参数
:param recommend_type: 首页推荐和相关推荐区分参数（0-首页推荐，1-相关推荐）
:param ab_exp_info: AB实验组参数
:param ab_info_data: app实验组参数
:param page_type: 页面区分参数，默认：0（首页）
:return:
"""
top_K = config_.K
flow_pool_P = config_.P
# 不获取人工干预数据标记
no_op_flag = False
old_video_index = -1
# if not ab_exp_info:
# ab_code = config_.AB_CODE['initial']
# expire_time = 24 * 3600
# rule_key = config_.RULE_KEY_REGION['initial'].get('rule_key')
# data_key = config_.RULE_KEY_REGION['initial'].get('data_key')
# # old_video_index = -1
# else:
# 默认使用 095 实验的配置
ab_code = config_.AB_EXP_CODE['095'].get('ab_code')
expire_time = 3600
rule_key = config_.AB_EXP_CODE['095'].get('rule_key')
data_key = config_.AB_EXP_CODE['095'].get('data_key')
no_op_flag = True
# 获取实验配置
if ab_exp_info:
ab_exp_code_list = []
config_value_dict = {}
for _, item in ab_exp_info.items():
if not item:
continue
for ab_item in item:
ab_exp_code = ab_item.get('abExpCode', None)
if not ab_exp_code:
continue
ab_exp_code_list.append(str(ab_exp_code))
config_value_dict[str(ab_exp_code)] = ab_item.get('configValue', None)
for code, param in config_.AB_EXP_CODE.items():
if code in ab_exp_code_list:
ab_code = param.get('ab_code')
expire_time = 3600
rule_key = param.get('rule_key')
data_key = param.get('data_key')
no_op_flag = True
break
"""
# 推荐条数 10->4 实验
# if config_.AB_EXP_CODE['rec_size_home'] in ab_exp_code_list:
# config_value = config_value_dict.get(config_.AB_EXP_CODE['rec_size_home'], None)
# if config_value:
# config_value = eval(str(config_value))
# else:
# config_value = {}
# log_.info(f'config_value: {config_value}, type: {type(config_value)}')
# size = int(config_value.get('size', 4))
# top_K = int(config_value.get('K', 3))
# flow_pool_P = float(config_value.get('P', 0.3))
# else:
# size = size
# top_K = config_.K
# flow_pool_P = config_.P
# 算法实验相对对照组
# if config_.AB_EXP_CODE['ab_initial'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['ab_initial']
# expire_time = 24 * 3600
# rule_key = config_.RULE_KEY['initial']
# no_op_flag = True
# 小时级更新-规则1 实验
# elif config_.AB_EXP_CODE['rule_rank1'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank1')
# expire_time = 3600
# rule_key = config_.RULE_KEY['rule_rank1']
# no_op_flag = True
# elif config_.AB_EXP_CODE['rule_rank2'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank2')
# expire_time = 3600
# rule_key = config_.RULE_KEY['rule_rank2']
# elif config_.AB_EXP_CODE['rule_rank3'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank3')
# expire_time = 3600
# rule_key = config_.RULE_KEY['rule_rank3']
# no_op_flag = True
# elif config_.AB_EXP_CODE['rule_rank4'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank4')
# expire_time = 3600
# rule_key = config_.RULE_KEY['rule_rank4']
# elif config_.AB_EXP_CODE['rule_rank5'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank5')
# expire_time = 3600
# rule_key = config_.RULE_KEY['rule_rank5']
# elif config_.AB_EXP_CODE['day_rule_rank1'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_day'].get('day_rule_rank1')
# expire_time = 24 * 3600
# rule_key = config_.RULE_KEY_DAY['day_rule_rank1']
# no_op_flag = True
# if config_.AB_EXP_CODE['rule_rank6'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank6')
# expire_time = 3600
# rule_key = config_.RULE_KEY['rule_rank6']
# no_op_flag = True
# elif config_.AB_EXP_CODE['day_rule_rank2'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_day'].get('day_rule_rank2')
# expire_time = 24 * 3600
# rule_key = config_.RULE_KEY_DAY['day_rule_rank2']
# no_op_flag = True
# elif config_.AB_EXP_CODE['region_rule_rank1'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank1')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank1']
# no_op_flag = True
# elif config_.AB_EXP_CODE['24h_rule_rank1'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_24h'].get('24h_rule_rank1')
# expire_time = 3600
# rule_key = config_.RULE_KEY_24H['24h_rule_rank1']
# no_op_flag = True
# elif config_.AB_EXP_CODE['24h_rule_rank2'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rank_by_24h'].get('24h_rule_rank2')
# expire_time = 3600
# rule_key = config_.RULE_KEY_24H['24h_rule_rank2']
# no_op_flag = True
# elif config_.AB_EXP_CODE['region_rule_rank2'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank2')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank2']
# no_op_flag = True
# if config_.AB_EXP_CODE['region_rule_rank3'] in ab_exp_code_list or\
# config_.AB_EXP_CODE['region_rule_rank3_appType_19'] in ab_exp_code_list or\
# config_.AB_EXP_CODE['region_rule_rank3_appType_4'] in ab_exp_code_list or\
# config_.AB_EXP_CODE['region_rule_rank3_appType_6'] in ab_exp_code_list or\
# config_.AB_EXP_CODE['region_rule_rank3_appType_18'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank3')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank3'].get('rule_key')
# data_key = config_.RULE_KEY_REGION['region_rule_rank3'].get('data_key')
# no_op_flag = True
# if config_.AB_EXP_CODE['region_rule_rank4'] in ab_exp_code_list or\
if config_.AB_EXP_CODE['region_rule_rank4_appType_19'] in ab_exp_code_list or \
config_.AB_EXP_CODE['region_rule_rank4_appType_4'] in ab_exp_code_list or\
config_.AB_EXP_CODE['region_rule_rank4_appType_6'] in ab_exp_code_list or\
config_.AB_EXP_CODE['region_rule_rank4_appType_18'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank4'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank4'].get('data_key')
no_op_flag = True
# elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data1'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data1'].get('rule_key')
# data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data1'].get('data_key')
# no_op_flag = True
# elif config_.AB_EXP_CODE['region_rule_rank3_appType_5_data2'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank3_appType_5_data2')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank3_appType_5_data2'].get('rule_key')
# data_key = config_.RULE_KEY_REGION['region_rule_rank3_appType_5_data2'].get('data_key')
# no_op_flag = True
elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data3'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_5_data3')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data3'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data3'].get('data_key')
no_op_flag = True
elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data4'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_5_data4')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data4'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data4'].get('data_key')
no_op_flag = True
elif config_.AB_EXP_CODE['region_rule_rank4_appType_0_data2'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_0_data2')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_0_data2'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_0_data2'].get('data_key')
no_op_flag = True
# elif config_.AB_EXP_CODE['region_rule_rank4_appType_19_data2'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_19_data2')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data2'].get('rule_key')
# data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data2'].get('data_key')
# no_op_flag = True
# elif config_.AB_EXP_CODE['region_rule_rank4_appType_19_data3'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_19_data3')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data3'].get('rule_key')
# data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data3'].get('data_key')
# no_op_flag = True
elif config_.AB_EXP_CODE['region_rule_rank5_appType_0_data1'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank5_appType_0_data1')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank5_appType_0_data1'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank5_appType_0_data1'].get('data_key')
no_op_flag = True
elif config_.AB_EXP_CODE['region_rule_rank4_appType_4_data2'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_4_data2')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data2'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data2'].get('data_key')
no_op_flag = True
elif config_.AB_EXP_CODE['region_rule_rank4_appType_4_data3'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_4_data3')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data3'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data3'].get('data_key')
no_op_flag = True
elif config_.AB_EXP_CODE['region_rule_rank4_appType_6_data2'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_6_data2')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data2'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data2'].get('data_key')
no_op_flag = True
elif config_.AB_EXP_CODE['region_rule_rank4_appType_6_data3'] in ab_exp_code_list:
ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_6_data3')
expire_time = 3600
rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data3'].get('rule_key')
data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data3'].get('data_key')
no_op_flag = True
# elif config_.AB_EXP_CODE['region_rule_rank4_appType_18_data2'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_18_data2')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_18_data2'].get('rule_key')
# data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_18_data2'].get('data_key')
# no_op_flag = True
# elif config_.AB_EXP_CODE['region_rule_rank6_appType_0_data1'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank6_appType_0_data1')
# expire_time = 3600
# rule_key = config_.RULE_KEY_REGION['region_rule_rank6_appType_0_data1'].get('rule_key')
# data_key = config_.RULE_KEY_REGION['region_rule_rank6_appType_0_data1'].get('data_key')
# no_op_flag = True
else:
ab_code = config_.AB_CODE['initial']
expire_time = 24 * 3600
rule_key = config_.RULE_KEY_REGION['initial'].get('rule_key')
data_key = config_.RULE_KEY_REGION['initial'].get('data_key')
# # 老好看视频 / 票圈最惊奇 首页/相关推荐逻辑更新实验
# if config_.AB_EXP_CODE['rov_rank_appType_18_19'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rov_rank_appType_18_19']
# expire_time = 3600
# flow_pool_P = config_.P_18_19
# no_op_flag = True
#
# elif config_.AB_EXP_CODE['rov_rank_appType_19'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['rov_rank_appType_19']
# expire_time = 3600
# top_K = 0
# flow_pool_P = config_.P_18_19
# no_op_flag = True
#
# elif config_.AB_EXP_CODE['top_video_relevant_appType_19'] in ab_exp_code_list and page_type == 2:
# ab_code = config_.AB_CODE['top_video_relevant_appType_19']
# expire_time = 3600
# top_K = 1
# flow_pool_P = config_.P_18_19
# no_op_flag = True
#
# # 票圈最惊奇完整影视资源实验
# elif config_.AB_EXP_CODE['whole_movies'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['whole_movies']
# expire_time = 24 * 3600
# no_op_flag = True
# 老视频实验
# if config_.AB_EXP_CODE['old_video'] in ab_exp_code_list:
# ab_code = config_.AB_CODE['old_video']
# no_op_flag = True
# old_video_index = 2
# else:
# old_video_index = -1
"""
# APP实验组
# if ab_info_data:
# ab_info_app = {}
# for page_code, item in json.loads(ab_info_data).items():
# if not item:
# continue
# ab_info_code = item.get('eventId', None)
# if ab_info_code:
# ab_info_app[page_code] = ab_info_code
# print(f"======{ab_info_app}")
#
# # 首页推荐
# if recommend_type == 0:
# if config_.APP_AB_CODE['10003'].get('region_rule_rank6_appType_13_data2') == ab_info_app.get('10003', None):
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank6_appType_13_data2')
# expire_time = 3600
# rule_key = 'rule6'
# data_key = 'data7'
# no_op_flag = True
# else:
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
# expire_time = 3600
# rule_key = 'rule3'
# data_key = 'data1'
# no_op_flag = True
# # 相关推荐
# elif recommend_type == 1:
# if config_.APP_AB_CODE['10037'] == ab_info_app.get('10037', None):
# ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
# expire_time = 3600
# rule_key = 'rule3'
# data_key = 'data1'
# no_op_flag = True
return top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, no_op_flag, old_video_index
def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
client_info, ab_exp_info, params, ab_info_data, version_audit_status):
"""
首页线上推荐逻辑
:param request_id: request_id
:param mid: mid type-string
:param uid: uid type-string
:param size: 请求视频数量 type-int
:param app_type: 产品标识 type-int
:param algo_type: 算法类型 type-string
:param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
:param ab_exp_info: ab实验分组参数 [{"expItemId":1, "configValue":{"size":4, "K":3, ...}}, ...]
:param params:
:param ab_info_data: app实验分组参数
:param version_audit_status: 小程序版本审核参数：1-审核中，2-审核通过
:return:
"""
# 对 vlog 切换10%的流量做实验
# 对mid进行哈希
# hash_mid = hashlib.md5(mid.encode('utf-8')).hexdigest()
# if app_type in config_.AB_TEST['rank_by_h'] and hash_mid[-1:] in ['8', '0', 'a', 'b']:
# # 简单召回 - 排序 - 兜底
# rank_result, last_rov_recall_key = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
# algo_type=algo_type, client_info=client_info,
# expire_time=3600,
# ab_code=config_.AB_CODE['rank_by_h'])
# # ab-test
# result = ab_test_op(rank_result=rank_result,
# ab_code_list=[config_.AB_CODE['position_insert']],
# app_type=app_type, mid=mid, uid=uid)
# # redis数据刷新
# update_redis_data(result=result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
# expire_time=3600)
# if app_type == config_.APP_TYPE['APP']:
# # 票圈视频APP
# top_K = config_.K
# flow_pool_P = config_.P
# # 简单召回 - 排序 - 兜底
# rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
# mid=mid, uid=uid, app_type=app_type,
# size=size, top_K=top_K, flow_pool_P=flow_pool_P,
# algo_type=algo_type, client_info=client_info,
# expire_time=12 * 3600, params=params)
# # ab-test
# # result = ab_test_op(rank_result=rank_result,
# # ab_code_list=[config_.AB_CODE['position_insert']],
# # app_type=app_type, mid=mid, uid=uid)
# # redis数据刷新
# update_redis_data(result=rank_result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
# top_K=top_K, expire_time=12 * 3600)
#
# else:
recommend_result = {}
param_st = time.time()
# 特殊mid 和 小程序审核版本推荐处理
if mid in get_special_mid_list() or version_audit_status == 1:
rank_result = special_mid_recommend(request_id=request_id, mid=mid, uid=uid, app_type=app_type, size=size)
recommend_result['videos'] = rank_result
return recommend_result
# 普通mid推荐处理
top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, no_op_flag, old_video_index = \
get_recommend_params(recommend_type=0, ab_exp_info=ab_exp_info, ab_info_data=ab_info_data)
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'app_type': app_type,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'get_recommend_params',
# 'executeTime': (time.time() - param_st) * 1000
# })
recommend_result['getRecommendParamsTime'] = (time.time() - param_st) * 1000
# 简单召回 - 排序 - 兜底
get_result_st = time.time()
result, last_rov_recall_key = video_recommend(request_id=request_id,
# rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
mid=mid, uid=uid, app_type=app_type,
size=size, top_K=top_K, flow_pool_P=flow_pool_P,
algo_type=algo_type, client_info=client_info,
ab_code=ab_code, expire_time=expire_time,
rule_key=rule_key, data_key=data_key,
no_op_flag=no_op_flag, old_video_index=old_video_index,
params=params)
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'app_type': app_type,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'get_recommend_result',
# 'executeTime': (time.time() - get_result_st) * 1000
# })
recommend_result['recommendOperation'] = result
rank_result = result.get('rankResult')
recommend_result['videos'] = rank_result
recommend_result['getRecommendResultTime'] = (time.time() - get_result_st) * 1000
# ab-test
# result = ab_test_op(rank_result=rank_result,
# ab_code_list=[config_.AB_CODE['position_insert']],
# app_type=app_type, mid=mid, uid=uid)
# redis数据刷新
update_redis_st = time.time()
update_redis_data(result=rank_result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
top_K=top_K)
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'app_type': app_type,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'update_redis_data',
# 'executeTime': (time.time() - update_redis_st) * 1000
# })
recommend_result['updateRedisDataTime'] = (time.time() - update_redis_st) * 1000
return recommend_result
# return rank_result
def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_exp_info, client_info,
page_type, params, ab_info_data, version_audit_status):
"""
相关推荐逻辑
:param request_id: request_id
:param video_id: 相关推荐的头部视频id
:param mid: mid type-string
:param uid: uid type-string
:param size: 请求视频数量 type-int
:param app_type: 产品标识 type-int
:param ab_exp_info: ab实验分组参数 [{"expItemId":1, "configValue":{"size":4, "K":3, ...}}, ...]
:param client_info: 地域参数
:param page_type: 页面区分参数 1:详情页；2:分享页
:param params:
:param ab_info_data: app实验分组参数
:param version_audit_status: 小程序版本审核参数：1-审核中，2-审核通过
:return: videos type-list
"""
recommend_result = {}
param_st = time.time()
# 特殊mid 和 小程序审核版本推荐处理
if mid in get_special_mid_list() or version_audit_status == 1:
rank_result = special_mid_recommend(request_id=request_id, mid=mid, uid=uid, app_type=app_type, size=size)
recommend_result['videos'] = rank_result
return recommend_result
# return rank_result
# 普通mid推荐处理
top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, no_op_flag, old_video_index = \
get_recommend_params(recommend_type=1, ab_exp_info=ab_exp_info, ab_info_data=ab_info_data, page_type=page_type)
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'app_type': app_type,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'get_recommend_params',
# 'executeTime': (time.time() - param_st) * 1000
# })
recommend_result['getRecommendParamsTime'] = (time.time() - param_st) * 1000
# 简单召回 - 排序 - 兜底
get_result_st = time.time()
result, last_rov_recall_key = video_recommend(request_id=request_id,
# rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
mid=mid, uid=uid, app_type=app_type,
size=size, top_K=top_K, flow_pool_P=flow_pool_P,
algo_type='', client_info=client_info,
ab_code=ab_code, expire_time=expire_time,
rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
old_video_index=old_video_index, video_id=video_id,
params=params)
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'app_type': app_type,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'get_recommend_result',
# 'executeTime': (time.time() - get_result_st) * 1000
# })
recommend_result['recommendOperation'] = result
rank_result = result.get('rankResult')
recommend_result['videos'] = rank_result
recommend_result['getRecommendResultTime'] = (time.time() - get_result_st) * 1000
# ab-test
# result = ab_test_op(rank_result=rank_result,
# ab_code_list=[config_.AB_CODE['position_insert'], config_.AB_CODE['relevant_video_op']],
# app_type=app_type, mid=mid, uid=uid, head_vid=video_id, size=size)
# redis数据刷新
update_redis_st = time.time()
update_redis_data(result=rank_result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
top_K=top_K)
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': request_id,
# 'app_type': app_type,
# 'mid': mid,
# 'uid': uid,
# 'operation': 'update_redis_data',
# 'executeTime': (time.time() - update_redis_st) * 1000
# })
recommend_result['updateRedisDataTime'] = (time.time() - update_redis_st) * 1000
return recommend_result
# return rank_result
def special_mid_recommend(request_id, mid, uid, app_type, size,
ab_code=config_.AB_CODE['special_mid'],
push_from=config_.PUSH_FROM['special_mid'],
expire_time=24*3600):
redis_helper = RedisHelper()
# 特殊mid推荐指定视频列表
pool_recall = PoolRecall(request_id=request_id, app_type=app_type,
mid=mid, uid=uid, ab_code=ab_code)
# 获取相关redis key
special_key_name, redis_date = pool_recall.get_pool_redis_key(pool_type='special')
# 用户上一次在rov召回池对应的位置
last_special_recall_key = f'{config_.LAST_VIDEO_FROM_SPECIAL_POOL_PREFIX}{app_type}:{mid}:{redis_date}'
value = redis_helper.get_data_from_redis(last_special_recall_key)
if value:
idx = redis_helper.get_index_with_data(special_key_name, value)
if not idx:
idx = 0
else:
idx += 1
else:
idx = 0
recall_result = []
# 每次获取的视频数
get_size = size * 5
# 记录获取频次
freq = 0
while len(recall_result) < size:
freq += 1
if freq > config_.MAX_FREQ_FROM_ROV_POOL:
break
# 获取数据
data = redis_helper.get_data_zset_with_index(key_name=special_key_name,
start=idx, end=idx + get_size - 1,
with_scores=True)
if not data:
break
# 获取视频id,并转换类型为int,并存储为key-value{videoId: score}
# 添加视频源参数 pushFrom, abCode
temp_result = [{'videoId': int(value[0]), 'rovScore': value[1],
'pushFrom': push_from, 'abCode': ab_code}
for value in data]
recall_result.extend(temp_result)
idx += get_size
# 将此次获取的末位视频id同步刷新到Redis中，方便下次快速定位到召回位置，过期时间为1天
if mid and recall_result:
# mid为空时，不做记录
redis_helper.set_data_to_redis(key_name=last_special_recall_key,
value=recall_result[:size][-1]['videoId'],
expire_time=expire_time)
return recall_result[:size]
def get_special_mid_list():
redis_helper = RedisHelper()
special_mid_list = redis_helper.get_data_from_set(key_name=config_.KEY_NAME_SPECIAL_MID)
if special_mid_list:
return special_mid_list
else:
return []
if __name__ == '__main__':
videos = [
{"videoId": 10136461, "rovScore": 99.971, "pushFrom": "recall_pool", "abCode": 10000},
{"videoId": 10239014, "rovScore": 99.97, "pushFrom": "recall_pool", "abCode": 10000},
{"videoId": 9851154, "rovScore": 99.969, "pushFrom": "recall_pool", "abCode": 10000},
{"videoId": 10104347, "rovScore": 99.968, "pushFrom": "recall_pool", "abCode": 10000},
{"videoId": 10141507, "rovScore": 99.967, "pushFrom": "recall_pool", "abCode": 10000},
{"videoId": 10292817, "flowPool": "2#6#2#1641780979606", "rovScore": 53.926690610816486,
"pushFrom": "flow_pool", "abCode": 10000},
{"videoId": 10224932, "flowPool": "2#5#1#1641800279644", "rovScore": 53.47890460059617, "pushFrom": "flow_pool",
"abCode": 10000},
{"videoId": 9943255, "rovScore": 99.966, "pushFrom": "recall_pool", "abCode": 10000},
{"videoId": 10282970, "flowPool": "2#5#1#1641784814103", "rovScore": 52.682815076325575,
"pushFrom": "flow_pool", "abCode": 10000},
{"videoId": 10282205, "rovScore": 99.965, "pushFrom": "recall_pool", "abCode": 10000}
]
res = relevant_video_top_recommend(app_type=4, mid='', uid=1111, head_vid=123, videos=videos, size=10)
print(res)