recommend.py 91 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888
  1. import json
  2. import time
  3. import multiprocessing
  4. import traceback
  5. import hashlib
  6. from datetime import datetime, timedelta
  7. import config
  8. from log import Log
  9. from config import set_config
  10. from video_recall import PoolRecall
  11. from video_rank import video_new_rank2, video_sanke_rank,video_new_rank,video_rank,refactor_video_rank, bottom_strategy, video_rank_by_w_h_rate, video_rank_with_old_video, bottom_strategy2
  12. from db_helper import RedisHelper
  13. import gevent
  14. from utils import FilterVideos, get_user_has30day_return
  15. import ast
  16. log_ = Log()
  17. config_ = set_config()
  18. def relevant_video_top_recommend(request_id, app_type, mid, uid, head_vid, videos, size):
  19. """
  20. 相关推荐强插 运营给定置顶相关性视频
  21. :param request_id: request_id
  22. :param app_type: 产品标识 type-int
  23. :param mid: mid
  24. :param uid: uid
  25. :param head_vid: 相关推荐头部视频id type-int
  26. :param videos: 当前相关推荐结果 type-list
  27. :param size: 返回视频个数 type-int
  28. :return: rank_result
  29. """
  30. # 获取头部视频对应的相关性视频
  31. key_name = '{}{}'.format(config_.RELEVANT_VIDEOS_WITH_OP_KEY_NAME, head_vid)
  32. redis_helper = RedisHelper()
  33. relevant_videos = redis_helper.get_data_from_redis(key_name=key_name)
  34. if relevant_videos is None:
  35. # 该视频没有指定的相关性视频
  36. return videos
  37. relevant_videos = json.loads(relevant_videos)
  38. # 按照指定顺序排序
  39. relevant_videos_sorted = sorted(relevant_videos, key=lambda x: x['order'], reverse=False)
  40. # 过滤
  41. relevant_video_ids = [int(item['recommend_vid']) for item in relevant_videos_sorted]
  42. filter_helper = FilterVideos(request_id=request_id, app_type=app_type, video_ids=relevant_video_ids, mid=mid, uid=uid)
  43. filtered_ids = filter_helper.filter_videos()
  44. if filtered_ids is None:
  45. return videos
  46. # 获取生效中的视频
  47. now = int(time.time())
  48. relevant_videos_in_effect = [
  49. {'videoId': int(item['recommend_vid']), 'pushFrom': config_.PUSH_FROM['relevant_video_op'],
  50. 'abCode': config_.AB_CODE['relevant_video_op']}
  51. for item in relevant_videos_sorted
  52. if item['start_time'] < now < item['finish_time'] and int(item['recommend_vid']) in filtered_ids
  53. ]
  54. if len(relevant_videos_in_effect) == 0:
  55. return videos
  56. # 与现有排序结果 进行合并重排
  57. # 获取现有排序结果中流量池视频 及其位置
  58. relevant_ids = [item['videoId'] for item in relevant_videos_in_effect]
  59. flow_pool_videos = []
  60. other_videos = []
  61. for i, item in enumerate(videos):
  62. if item.get('pushFrom', None) == config_.PUSH_FROM['flow_recall'] and item.get('videoId') not in relevant_ids:
  63. flow_pool_videos.append((i, item))
  64. elif item.get('videoId') not in relevant_ids:
  65. other_videos.append(item)
  66. else:
  67. continue
  68. # 重排,保持流量池视频位置不变
  69. rank_result = relevant_videos_in_effect + other_videos
  70. for i, item in flow_pool_videos:
  71. rank_result.insert(i, item)
  72. return rank_result[:size]
  73. def video_position_recommend(request_id, mid, uid, app_type, videos):
  74. # videos = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
  75. # algo_type=algo_type, client_info=client_info)
  76. redis_helper = RedisHelper()
  77. pos1_vids = redis_helper.get_data_from_redis(config.BaseConfig.RECALL_POSITION1_KEY_NAME)
  78. pos2_vids = redis_helper.get_data_from_redis(config.BaseConfig.RECALL_POSITION2_KEY_NAME)
  79. if pos1_vids is not None:
  80. pos1_vids = ast.literal_eval(pos1_vids)
  81. if pos2_vids is not None:
  82. pos2_vids = ast.literal_eval(pos2_vids)
  83. pos1_vids = [] if pos1_vids is None else pos1_vids
  84. pos2_vids = [] if pos2_vids is None else pos2_vids
  85. pos1_vids = [int(i) for i in pos1_vids]
  86. pos2_vids = [int(i) for i in pos2_vids]
  87. filter_1 = FilterVideos(request_id=request_id, app_type=app_type, video_ids=pos1_vids, mid=mid, uid=uid)
  88. filter_2 = FilterVideos(request_id=request_id, app_type=app_type, video_ids=pos2_vids, mid=mid, uid=uid)
  89. t = [gevent.spawn(filter_1.filter_videos), gevent.spawn(filter_2.filter_videos)]
  90. gevent.joinall(t)
  91. filted_list = [i.get() for i in t]
  92. pos1_vids = filted_list[0]
  93. pos2_vids = filted_list[1]
  94. videos = positon_duplicate(pos1_vids, pos2_vids, videos)
  95. if pos1_vids is not None and len(pos1_vids) >0 :
  96. videos.insert(0, {'videoId': int(pos1_vids[0]), 'rovScore': 100,
  97. 'pushFrom': config_.PUSH_FROM['position_insert'], 'abCode': config_.AB_CODE['position_insert']})
  98. if pos2_vids is not None and len(pos2_vids) >0 :
  99. videos.insert(1, {'videoId': int(pos2_vids[0]), 'rovScore': 100,
  100. 'pushFrom': config_.PUSH_FROM['position_insert'], 'abCode': config_.AB_CODE['position_insert']})
  101. return videos[:10]
  102. def positon_duplicate(pos1_vids, pos2_vids, videos):
  103. s = set()
  104. if pos1_vids is not None and len(pos1_vids) >0:
  105. s.add(int(pos1_vids[0]))
  106. if pos2_vids is not None and len(pos2_vids) >0:
  107. s.add(int(pos2_vids[0]))
  108. l = []
  109. for item in videos:
  110. if item['videoId'] in s:
  111. continue
  112. else:
  113. l.append(item)
  114. return l
  115. def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, algo_type, client_info,
  116. expire_time=24*3600, ab_code=config_.AB_CODE['initial'], rule_key='', data_key='',
  117. no_op_flag=False, old_video_index=-1, video_id=None, params=None, rule_key_30day=None,
  118. shield_config=None):
  119. """
  120. 首页线上推荐逻辑
  121. :param request_id: request_id
  122. :param mid: mid type-string
  123. :param uid: uid type-string
  124. :param size: 请求视频数量 type-int
  125. :param top_K: 保证topK为召回池视频 type-int
  126. :param flow_pool_P: size-top_K视频为流量池视频的概率 type-float
  127. :param app_type: 产品标识 type-int
  128. :param algo_type: 算法类型 type-string
  129. :param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
  130. :param expire_time: 末位视频记录redis过期时间
  131. :param ab_code: AB实验code
  132. :param video_id: 相关推荐头部视频id
  133. :param params:
  134. :return:
  135. """
  136. result = {}
  137. # ####### 多进程召回
  138. start_recall = time.time()
  139. # log_.info('====== recall')
  140. '''
  141. cores = multiprocessing.cpu_count()
  142. pool = multiprocessing.Pool(processes=cores)
  143. pool_recall = PoolRecall(app_type=app_type, mid=mid, uid=uid, ab_code=ab_code)
  144. _, last_rov_recall_key, _ = pool_recall.get_video_last_idx()
  145. pool_list = [
  146. # rov召回池
  147. pool.apply_async(pool_recall.rov_pool_recall, (size,)),
  148. # 流量池
  149. pool.apply_async(pool_recall.flow_pool_recall, (size,))
  150. ]
  151. recall_result_list = [p.get() for p in pool_list]
  152. pool.close()
  153. pool.join()
  154. '''
  155. recall_result_list = []
  156. pool_recall = PoolRecall(request_id=request_id,
  157. app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
  158. client_info=client_info, rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  159. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config, video_id= video_id)
  160. # _, last_rov_recall_key, _ = pool_recall.get_video_last_idx()
  161. # # 小时级实验
  162. # if ab_code in [code for _, code in config_.AB_CODE['rank_by_h'].items()]:
  163. # t = [gevent.spawn(pool_recall.rule_recall_by_h, size, expire_time),
  164. # gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  165. # gevent.spawn(pool_recall.flow_pool_recall, size)]
  166. # # 小时级实验
  167. # elif ab_code in [code for _, code in config_.AB_CODE['rank_by_24h'].items()]:
  168. # t = [gevent.spawn(pool_recall.rov_pool_recall_by_h, size, expire_time),
  169. # gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  170. # gevent.spawn(pool_recall.flow_pool_recall, size)]
  171. # 地域分组实验
  172. # if ab_code in [code for _, code in config_.AB_CODE['region_rank_by_h'].items()]:
  173. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  174. t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time)]
  175. if ab_code==60058:
  176. t.append(gevent.spawn(pool_recall.get_hot_item_reall))
  177. elif ab_code==60059:
  178. t.append(gevent.spawn(pool_recall.get_word2vec_item_reall))
  179. elif ab_code==60061:
  180. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter))
  181. else:
  182. t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time),
  183. gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  184. gevent.spawn(pool_recall.flow_pool_recall, size)]
  185. if ab_code==60058:
  186. t.append(gevent.spawn(pool_recall.get_hot_item_reall))
  187. elif ab_code == 60059:
  188. t.append(gevent.spawn(pool_recall.get_word2vec_item_reall))
  189. elif ab_code == 60061:
  190. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter))
  191. # 最惊奇相关推荐实验
  192. # elif ab_code == config_.AB_CODE['top_video_relevant_appType_19']:
  193. # t = [gevent.spawn(pool_recall.relevant_recall_19, video_id, size, expire_time),
  194. # gevent.spawn(pool_recall.flow_pool_recall_18_19, size)]
  195. # 最惊奇完整影视实验
  196. # elif ab_code == config_.AB_CODE['whole_movies']:
  197. # t = [gevent.spawn(pool_recall.rov_pool_recall_19, size, expire_time)]
  198. # 最惊奇/老好看实验
  199. # elif app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  200. # t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
  201. # gevent.spawn(pool_recall.flow_pool_recall_18_19, size)]
  202. # # 天级实验
  203. # elif ab_code in [code for _, code in config_.AB_CODE['rank_by_day'].items()]:
  204. # t = [gevent.spawn(pool_recall.rov_pool_recall_by_day, size, expire_time),
  205. # gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  206. # gevent.spawn(pool_recall.flow_pool_recall, size)]
  207. # 老视频实验
  208. # elif ab_code in [config_.AB_CODE['old_video']]:
  209. # t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
  210. # gevent.spawn(pool_recall.flow_pool_recall, size),
  211. # gevent.spawn(pool_recall.old_videos_recall, size)]
  212. # else:
  213. # if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  214. # t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time)]
  215. # else:
  216. # t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
  217. # gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  218. # gevent.spawn(pool_recall.flow_pool_recall, size)]
  219. gevent.joinall(t)
  220. recall_result_list = [i.get() for i in t]
  221. # end_recall = time.time()
  222. # log_.info({
  223. # 'logTimestamp': int(time.time() * 1000),
  224. # 'request_id': request_id,
  225. # 'mid': mid,
  226. # 'uid': uid,
  227. # 'operation': 'recall',
  228. # 'recall_result': recall_result_list,
  229. # 'executeTime': (time.time() - start_recall) * 1000
  230. # })
  231. result['recallResult'] = recall_result_list
  232. result['recallTime'] = (time.time() - start_recall) * 1000
  233. # ####### 排序
  234. start_rank = time.time()
  235. # log_.info('====== rank')
  236. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  237. if ab_code in [
  238. config_.AB_CODE['rov_rank_appType_18_19'],
  239. config_.AB_CODE['rov_rank_appType_19'],
  240. config_.AB_CODE['top_video_relevant_appType_19']
  241. ]:
  242. data = {
  243. 'rov_pool_recall': recall_result_list[0],
  244. 'flow_pool_recall': recall_result_list[1]
  245. }
  246. else:
  247. data = {
  248. 'rov_pool_recall': recall_result_list[0],
  249. 'flow_pool_recall': []
  250. }
  251. else:
  252. if recall_result_list[1]:
  253. redis_helper = RedisHelper()
  254. quick_flow_pool_P = redis_helper.get_data_from_redis(
  255. key_name=f"{config_.QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX}{config_.QUICK_FLOW_POOL_ID}"
  256. )
  257. if quick_flow_pool_P:
  258. flow_pool_P = quick_flow_pool_P
  259. data = {
  260. 'rov_pool_recall': recall_result_list[0],
  261. 'flow_pool_recall': recall_result_list[1]
  262. }
  263. else:
  264. data = {
  265. 'rov_pool_recall': recall_result_list[0],
  266. 'flow_pool_recall': recall_result_list[2]
  267. }
  268. if ab_code == 60058:
  269. if len(recall_result_list)>=4:
  270. data['hot_recall'] = recall_result_list[3]
  271. else:
  272. data['hot_recall'] = []
  273. elif ab_code == 60059:
  274. if len(recall_result_list)>=4:
  275. data['w2v_recall'] = recall_result_list[3]
  276. else:
  277. data['w2v_recall'] = []
  278. elif ab_code == 60061:
  279. if len(recall_result_list)>=4:
  280. data['sim_recall'] = recall_result_list[3]
  281. else:
  282. data['sim_recall'] = []
  283. #if ab_code=="ab_new_test":
  284. # rank_result = video_new_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
  285. #else:
  286. #print("data['hot_recall']", data['hot_recall'])
  287. if ab_code == 60058 or ab_code == 60059 or ab_code == 60060 or ab_code == 60061 :
  288. test_config = None
  289. flow_config = None
  290. if ab_code == 60058:
  291. test_config = pool_recall.get_hotrecall_config()
  292. flow_config = pool_recall.get_hotrecall_flow_config()
  293. elif ab_code == 60059:
  294. test_config = pool_recall.get_w2v_config()
  295. flow_config = pool_recall.get_w2v_flow_config()
  296. elif ab_code == 60060:
  297. test_config = pool_recall.get_test_config()
  298. flow_config = pool_recall.get_flow_config()
  299. elif ab_code == 60061:
  300. test_config = pool_recall.get_simrecall_config()
  301. flow_config = pool_recall.get_simrecall_flow_config()
  302. #print("test config:",test_config)
  303. rank_result, flow_result = video_sanke_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P), ab_Code=ab_code, exp_config=test_config, flowConfig=flow_config)
  304. result['flow_num'] = flow_result
  305. result['rank_num'] = len(rank_result)
  306. else:
  307. rank_result = video_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
  308. # 老视频实验
  309. # if ab_code in [config_.AB_CODE['old_video']]:
  310. # rank_result = video_rank_with_old_video(rank_result=rank_result, old_video_recall=recall_result_list[2],
  311. # size=size, top_K=top_K, old_video_index=old_video_index)
  312. # end_rank = time.time()
  313. # log_.info({
  314. # 'logTimestamp': int(time.time() * 1000),
  315. # 'request_id': request_id,
  316. # 'mid': mid,
  317. # 'uid': uid,
  318. # 'operation': 'rank',
  319. # 'rank_result': rank_result,
  320. # 'executeTime': (time.time() - start_rank) * 1000
  321. # })
  322. result['rankResult'] = rank_result
  323. result['rankTime'] = (time.time() - start_rank) * 1000
  324. # if not rank_result:
  325. # # 兜底策略
  326. # # log_.info('====== bottom strategy')
  327. # start_bottom = time.time()
  328. # rank_result = bottom_strategy2(
  329. # size=size, app_type=app_type, mid=mid, uid=uid, ab_code=ab_code, client_info=client_info, params=params
  330. # )
  331. #
  332. # # if ab_code == config_.AB_CODE['region_rank_by_h'].get('abtest_130'):
  333. # # rank_result = bottom_strategy2(
  334. # # size=size, app_type=app_type, mid=mid, uid=uid, ab_code=ab_code, client_info=client_info, params=params
  335. # # )
  336. # # else:
  337. # # rank_result = bottom_strategy(
  338. # # request_id=request_id, size=size, app_type=app_type, ab_code=ab_code, params=params
  339. # # )
  340. #
  341. # # log_.info({
  342. # # 'logTimestamp': int(time.time() * 1000),
  343. # # 'request_id': request_id,
  344. # # 'mid': mid,
  345. # # 'uid': uid,
  346. # # 'operation': 'bottom',
  347. # # 'bottom_result': rank_result,
  348. # # 'executeTime': (time.time() - start_bottom) * 1000
  349. # # })
  350. # result['bottomResult'] = rank_result
  351. # result['bottomTime'] = (time.time() - start_bottom) * 1000
  352. #
  353. # result['rankResult'] = rank_result
  354. return result
  355. # return rank_result, last_rov_recall_key
  356. def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, algo_type, client_info,
  357. expire_time=24*3600, ab_code=config_.AB_CODE['initial'], rule_key='', data_key='',
  358. no_op_flag=False, old_video_index=-1, video_id=None, params=None, rule_key_30day=None,
  359. shield_config=None):
  360. """
  361. 首页线上推荐逻辑
  362. :param request_id: request_id
  363. :param mid: mid type-string
  364. :param uid: uid type-string
  365. :param size: 请求视频数量 type-int
  366. :param top_K: 保证topK为召回池视频 type-int
  367. :param flow_pool_P: size-top_K视频为流量池视频的概率 type-float
  368. :param app_type: 产品标识 type-int
  369. :param algo_type: 算法类型 type-string
  370. :param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
  371. :param expire_time: 末位视频记录redis过期时间
  372. :param ab_code: AB实验code
  373. :param video_id: 相关推荐头部视频id
  374. :param params:
  375. :return:
  376. """
  377. result = {}
  378. # ####### 多进程召回
  379. start_recall = time.time()
  380. # log_.info('====== recall')
  381. #print("abcode",ab_code)
  382. recall_result_list = []
  383. pool_recall = PoolRecall(request_id=request_id,
  384. app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
  385. client_info=client_info, rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  386. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config, video_id= video_id)
  387. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  388. t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time)]
  389. if ab_code ==60054:
  390. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter))
  391. if ab_code == 60055:
  392. t.append(gevent.spawn(pool_recall.get_3days_hot_item_reall))
  393. if ab_code == 60056:
  394. t.append(gevent.spawn(pool_recall.get_hot_item_reall))
  395. else:
  396. t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time),
  397. gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  398. gevent.spawn(pool_recall.flow_pool_recall, size)]
  399. if ab_code == 60054:
  400. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter))
  401. if ab_code == 60055:
  402. t.append(gevent.spawn(pool_recall.get_3days_hot_item_reall))
  403. if ab_code == 60056:
  404. t.append(gevent.spawn(pool_recall.get_hot_item_reall))
  405. gevent.joinall(t)
  406. recall_result_list = [i.get() for i in t]
  407. #print("recall:",recall_result_list)
  408. if len(recall_result_list)<0:
  409. result['recallResult']= []
  410. result['rankResult'] = []
  411. return result
  412. #1. merge simrecall
  413. if ab_code == 60054:
  414. rov_pool_recall = []
  415. if len(recall_result_list) >= 2:
  416. region_recall = recall_result_list[0]
  417. sim_recall = []
  418. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  419. sim_recall = recall_result_list[1]
  420. else:
  421. if len(recall_result_list)>=4:
  422. sim_recall = recall_result_list[3]
  423. now_video_ids = set('')
  424. if len(region_recall) > 0:
  425. for video in region_recall:
  426. video_id = video.get('videoId')
  427. if video_id not in now_video_ids:
  428. rov_pool_recall.append(video)
  429. now_video_ids.add(video_id)
  430. if len(sim_recall) > 0:
  431. for video in sim_recall:
  432. video_id = video.get('videoId')
  433. # print("sim video_id:", video_id)
  434. if video_id not in now_video_ids:
  435. rov_pool_recall.append(video)
  436. now_video_ids.add(video_id)
  437. if len(rov_pool_recall) > 0:
  438. recall_result_list[0] = rov_pool_recall
  439. # merge hot 3day recall
  440. if ab_code == 60055:
  441. rov_pool_recall = []
  442. if len(recall_result_list)>=2:
  443. region_recall = recall_result_list[0]
  444. hot_3_day_recall = []
  445. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  446. hot_3_day_recall = recall_result_list[1]
  447. else:
  448. if len(recall_result_list)>=4:
  449. hot_3_day_recall = recall_result_list[3]
  450. #print("sim:",sim_recall)
  451. now_video_ids = set('')
  452. if len(region_recall)>0:
  453. for video in region_recall:
  454. video_id = video.get('videoId')
  455. if video_id not in now_video_ids:
  456. rov_pool_recall.append(video)
  457. now_video_ids.add(video_id)
  458. if len(hot_3_day_recall)>0:
  459. for video in hot_3_day_recall:
  460. video_id = video.get('videoId')
  461. #print("sim video_id:", video_id)
  462. if video_id not in now_video_ids:
  463. rov_pool_recall.append(video)
  464. now_video_ids.add(video_id)
  465. if len(rov_pool_recall)>0:
  466. recall_result_list[0] = rov_pool_recall
  467. # merge old day recall
  468. if ab_code == 60056:
  469. rov_pool_recall = []
  470. if len(recall_result_list)>=2:
  471. region_recall = recall_result_list[0]
  472. hot_7_day_recall = []
  473. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  474. hot_7_day_recall = recall_result_list[1]
  475. else:
  476. if len(recall_result_list) >= 3:
  477. hot_7_day_recall = recall_result_list[3]
  478. now_video_ids = set('')
  479. if len(region_recall)>0:
  480. for video in region_recall:
  481. video_id = video.get('videoId')
  482. if video_id not in now_video_ids:
  483. rov_pool_recall.append(video)
  484. now_video_ids.add(video_id)
  485. if len(hot_7_day_recall)>0:
  486. for video in hot_7_day_recall:
  487. video_id = video.get('videoId')
  488. #print("sim video_id:", video_id)
  489. if video_id not in now_video_ids:
  490. rov_pool_recall.append(video)
  491. now_video_ids.add(video_id)
  492. if len(rov_pool_recall)>0:
  493. recall_result_list[0] = rov_pool_recall
  494. result['recallResult'] = recall_result_list
  495. result['recallTime'] = (time.time() - start_recall) * 1000
  496. #print("recall:", recall_result_list)
  497. # ####### 排序
  498. start_rank = time.time()
  499. # log_.info('====== rank')
  500. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  501. if ab_code in [
  502. config_.AB_CODE['rov_rank_appType_18_19'],
  503. config_.AB_CODE['rov_rank_appType_19'],
  504. config_.AB_CODE['top_video_relevant_appType_19']
  505. ]:
  506. data = {
  507. 'rov_pool_recall': recall_result_list[0],
  508. 'flow_pool_recall': recall_result_list[1]
  509. }
  510. else:
  511. data = {
  512. 'rov_pool_recall': recall_result_list[0],
  513. 'flow_pool_recall': []
  514. }
  515. else:
  516. if recall_result_list[1]:
  517. redis_helper = RedisHelper()
  518. quick_flow_pool_P = redis_helper.get_data_from_redis(
  519. key_name=f"{config_.QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX}{config_.QUICK_FLOW_POOL_ID}"
  520. )
  521. if quick_flow_pool_P:
  522. flow_pool_P = quick_flow_pool_P
  523. data = {
  524. 'rov_pool_recall': recall_result_list[0],
  525. 'flow_pool_recall': recall_result_list[1]
  526. }
  527. else:
  528. data = {
  529. 'rov_pool_recall': recall_result_list[0],
  530. 'flow_pool_recall': recall_result_list[2]
  531. }
  532. #if ab_code=="ab_new_test":
  533. #print("before data:", data)
  534. flow_config = None
  535. if ab_code == 60052:
  536. flow_config = pool_recall.get_flow_exp_7_config()
  537. elif ab_code == 60053:
  538. flow_config = pool_recall.get_flow_exp_8_config()
  539. elif ab_code == 60057:
  540. flow_config = pool_recall.get_flow_exp_6_config()
  541. rank_result, flow_num = video_new_rank2(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P), ab_code=ab_code, flowConfig=flow_config)
  542. print("result:", len(rank_result))
  543. result['rankResult'] = rank_result
  544. result['rank_num'] = len(rank_result)
  545. result['flow_num'] = flow_num
  546. result['rankTime'] = (time.time() - start_rank) * 1000
  547. return result
  548. # return rank_result, last_rov_recall_key
  549. def new_video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, algo_type, client_info,
  550. expire_time=24*3600, ab_code=config_.AB_CODE['initial'], rule_key='', data_key='',
  551. no_op_flag=False, old_video_index=-1, video_id=None, params=None, rule_key_30day=None,
  552. shield_config=None):
  553. """
  554. 首页线上推荐逻辑
  555. :param request_id: request_id
  556. :param mid: mid type-string
  557. :param uid: uid type-string
  558. :param size: 请求视频数量 type-int
  559. :param top_K: 保证topK为召回池视频 type-int
  560. :param flow_pool_P: size-top_K视频为流量池视频的概率 type-float
  561. :param app_type: 产品标识 type-int
  562. :param algo_type: 算法类型 type-string
  563. :param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
  564. :param expire_time: 末位视频记录redis过期时间
  565. :param ab_code: AB实验code
  566. :param video_id: 相关推荐头部视频id
  567. :param params:
  568. :return:
  569. """
  570. #1. recall
  571. result = {}
  572. result['rankResult'] = []
  573. # ####### 多进程召回
  574. start_recall = time.time()
  575. # 1. 根据城市或者省份获取region_code
  576. city_code_list = [code for _, code in config_.CITY_CODE.items()]
  577. # 获取provinceCode
  578. province_code = client_info.get('provinceCode', '-1')
  579. # 获取cityCode
  580. city_code = client_info.get('cityCode', '-1')
  581. if city_code in city_code_list:
  582. # 分城市数据存在时,获取城市分组数据
  583. region_code = city_code
  584. else:
  585. region_code = province_code
  586. if region_code == '':
  587. region_code = '-1'
  588. #print("region_code:", region_code)
  589. #size =1000
  590. pool_recall = PoolRecall(request_id=request_id,
  591. app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
  592. client_info=client_info, rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  593. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config, video_id= video_id)
  594. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  595. t = [gevent.spawn(pool_recall.get_region_hour_recall, size, region_code),
  596. gevent.spawn(pool_recall.get_region_day_recall, size, region_code),
  597. gevent.spawn(pool_recall.get_selected_recall, size, region_code),
  598. gevent.spawn(pool_recall.get_no_selected_recall, size, region_code)
  599. ]
  600. if ab_code == 60049:
  601. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall))
  602. else:
  603. t = [
  604. gevent.spawn(pool_recall.get_region_hour_recall, size, region_code),
  605. gevent.spawn(pool_recall.get_region_day_recall, size, region_code),
  606. gevent.spawn(pool_recall.get_selected_recall, size, region_code),
  607. gevent.spawn(pool_recall.get_no_selected_recall, size, region_code),
  608. gevent.spawn(pool_recall.new_flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  609. gevent.spawn(pool_recall.new_flow_pool_recall, size)]
  610. if ab_code ==60049:
  611. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall))
  612. gevent.joinall(t)
  613. # all recall_result
  614. all_recall_result_list = [i.get() for i in t]
  615. all_recall_result = []
  616. #print(all_recall_result_list)
  617. result['recallTime'] = (time.time() - start_recall) * 1000
  618. #print("recall time:", result['recallTime'])
  619. if not all_recall_result_list or len(all_recall_result_list)==0:
  620. return result
  621. for recall_item in all_recall_result_list:
  622. if not recall_item or len(recall_item)==0:
  623. continue
  624. for per_item in recall_item:
  625. all_recall_result.append(per_item)
  626. #print("all_recall_result:", all_recall_result)
  627. #2. duplicate
  628. dup_time = time.time()
  629. recall_dict = {}
  630. fast_flow_set = set('')
  631. flow_flow_set = set('')
  632. region_h_recall = []
  633. region_day_recall = []
  634. select_day_recall = []
  635. no_selected_recall = []
  636. sim_hot_recall = []
  637. flow_recall = []
  638. flowFlag_dict = {}
  639. for per_item in all_recall_result:
  640. #print(per_item)
  641. try:
  642. vId = int(per_item.get("videoId",0))
  643. if vId==0:
  644. continue
  645. recall_name = per_item.get("pushFrom",'')
  646. flow_pool = per_item.get("flowPool", '')
  647. if flow_pool != '':
  648. flow_pool_id = int(flow_pool.split('#')[0])
  649. if flow_pool_id == config_.QUICK_FLOW_POOL_ID:
  650. fast_flow_set.add(vId)
  651. else:
  652. flow_flow_set.add(vId)
  653. flowFlag_dict[vId] = flow_pool
  654. #duplicate divide into
  655. if vId not in recall_dict:
  656. if recall_name == config_.PUSH_FROM['rov_recall_region_h']:
  657. region_h_recall.append(per_item)
  658. elif recall_name == config_.PUSH_FROM['rov_recall_region_24h']:
  659. region_day_recall.append(per_item)
  660. elif recall_name == config_.PUSH_FROM['rov_recall_24h']:
  661. select_day_recall.append(per_item)
  662. elif recall_name == config_.PUSH_FROM['rov_recall_24h_dup']:
  663. no_selected_recall.append(per_item)
  664. elif recall_name == config_.PUSH_FROM['sim_hot_vid_recall']:
  665. sim_hot_recall.append(per_item)
  666. elif recall_name == config_.PUSH_FROM['flow_recall']:
  667. flow_recall.append(per_item)
  668. if vId not in recall_dict:
  669. recall_dict[vId] = recall_name
  670. else:
  671. recall_name = recall_dict[vId] + "," + recall_name
  672. recall_dict[vId] = recall_name
  673. except:
  674. continue
  675. #print("recall_dict:", recall_dict)
  676. #print("recall time:", (time.time()-dup_time)*1000)
  677. #3. filter video, 先过预曝光
  678. filter_time = time.time()
  679. filter_ = FilterVideos(request_id=request_id,
  680. app_type=app_type, mid=mid, uid=uid, video_ids=list(recall_dict.keys()))
  681. #print("filer:", list(recall_dict.keys()))
  682. #a).expose filter
  683. #all_recall_list = list(recall_dict.keys())
  684. all_recall_list = filter_.filter_videos_new(region_code=region_code, shield_config=shield_config, flow_set=flowFlag_dict.keys())
  685. #print("filer after:", all_recall_list)
  686. #print("filter_ time:", (time.time() - filter_time) * 1000)
  687. #4. sort: old sort: flow 按概率出
  688. start_rank = time.time()
  689. #quick_flow_pool_P get from redis
  690. redis_helper = RedisHelper()
  691. quick_flow_pool_P = redis_helper.get_data_from_redis(
  692. key_name=f"{config_.QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX}{config_.QUICK_FLOW_POOL_ID}"
  693. )
  694. if quick_flow_pool_P:
  695. flow_pool_P = quick_flow_pool_P
  696. rank_result= []
  697. if ab_code==60048 or ab_code==60049:
  698. rank_ids, add_flow_set = video_new_rank(videoIds=all_recall_list,fast_flow_set=fast_flow_set, flow_set=flow_flow_set,size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
  699. #print("rank_ids:", rank_ids)
  700. for rank_item in rank_ids:
  701. rank_id = rank_item[0]
  702. rank_score = rank_item[1]
  703. pushFrom = recall_dict.get(rank_id, '')
  704. #print(pushFrom, rank_id)
  705. flowPoolFlag = ''
  706. if rank_id in add_flow_set:
  707. flowPoolFlag = flowFlag_dict.get(rank_id,'')
  708. rank_result.append({'videoId': rank_id, 'flowPool': flowPoolFlag,
  709. 'rovScore': rank_score, 'pushFrom': pushFrom,
  710. 'abCode': ab_code})
  711. #
  712. #print("rank_result:", rank_result)
  713. else:
  714. all_dup_recall_result = region_h_recall+region_day_recall+select_day_recall+no_selected_recall+flow_recall
  715. rank_result = refactor_video_rank(rov_recall_rank=all_dup_recall_result,fast_flow_set=fast_flow_set, flow_set=flow_flow_set, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
  716. result['rankResult'] = rank_result
  717. result['rankTime'] = (time.time() - start_rank) * 1000
  718. #print("rank time:", result['rankTime'])
  719. return result
  720. # return rank_result, last_rov_recall_key
  721. def ab_test_op(rank_result, ab_code_list, app_type, mid, uid, **kwargs):
  722. """
  723. 对排序后的结果 按照AB实验进行对应的分组操作
  724. :param rank_result: 排序后的结果
  725. :param ab_code_list: 此次请求参与的 ab实验组
  726. :param app_type: 产品标识
  727. :param mid: mid
  728. :param uid: uid
  729. :param kwargs: 其他参数
  730. :return:
  731. """
  732. # ####### 视频宽高比AB实验
  733. # 对内容精选进行 视频宽高比分发实验
  734. # if config_.AB_CODE['w_h_rate'] in ab_code_list and app_type in config_.AB_TEST.get('w_h_rate', []):
  735. # rank_result = video_rank_by_w_h_rate(videos=rank_result)
  736. # log_.info('app_type: {}, mid: {}, uid: {}, rank_by_w_h_rate_result: {}'.format(
  737. # app_type, mid, uid, rank_result))
  738. # 按position位置排序
  739. if config_.AB_CODE['position_insert'] in ab_code_list and app_type in config_.AB_TEST.get('position_insert', []):
  740. rank_result = video_position_recommend(mid, uid, app_type, rank_result)
  741. print('===========================')
  742. print(rank_result)
  743. log_.info('app_type: {}, mid: {}, uid: {}, rank_by_position_insert_result: {}'.format(
  744. app_type, mid, uid, rank_result))
  745. # 相关推荐强插
  746. # if config_.AB_CODE['relevant_video_op'] in ab_code_list \
  747. # and app_type in config_.AB_TEST.get('relevant_video_op', []):
  748. # head_vid = kwargs['head_vid']
  749. # size = kwargs['size']
  750. # rank_result = relevant_video_top_recommend(
  751. # app_type=app_type, mid=mid, uid=uid, head_vid=head_vid, videos=rank_result, size=size
  752. # )
  753. # log_.info('app_type: {}, mid: {}, uid: {}, head_vid: {}, rank_by_relevant_video_op_result: {}'.format(
  754. # app_type, mid, uid, head_vid, rank_result))
  755. return rank_result
  756. def update_redis_data(result, app_type, mid, top_K, expire_time=24*3600):
  757. """
  758. 根据最终的排序结果更新相关redis数据
  759. :param result: 排序结果
  760. :param app_type: 产品标识
  761. :param mid: mid
  762. :param top_K: 保证topK为召回池视频 type-int
  763. :param expire_time: 末位视频记录redis过期时间
  764. :return: None
  765. """
  766. # ####### redis数据刷新
  767. try:
  768. redis_helper = RedisHelper()
  769. # log_.info('====== update redis')
  770. if mid and mid != 'null':
  771. # mid为空时,不做预曝光和定位数据更新
  772. # 预曝光数据同步刷新到Redis, 过期时间为0.5h
  773. preview_key_name = f"{config_.PREVIEW_KEY_PREFIX}{app_type}:{mid}"
  774. preview_video_ids = [int(item['videoId']) for item in result]
  775. if preview_video_ids:
  776. # log_.error('key_name = {} \n values = {}'.format(preview_key_name, tuple(preview_video_ids)))
  777. redis_helper.add_data_with_set(key_name=preview_key_name, values=tuple(preview_video_ids), expire_time=30 * 60)
  778. # log_.info('preview redis update success!')
  779. # # 将此次获取的ROV召回池top_K末位视频id同步刷新到Redis中,方便下次快速定位到召回位置,过期时间为1天
  780. # rov_recall_video = [item['videoId'] for item in result[:top_K]
  781. # if item['pushFrom'] == config_.PUSH_FROM['rov_recall']]
  782. # if len(rov_recall_video) > 0:
  783. # if app_type == config_.APP_TYPE['APP']:
  784. # key_name = config_.UPDATE_ROV_KEY_NAME_APP
  785. # else:
  786. # key_name = config_.UPDATE_ROV_KEY_NAME
  787. # if not redis_helper.get_score_with_value(key_name=key_name, value=rov_recall_video[-1]):
  788. # redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=rov_recall_video[-1],
  789. # expire_time=expire_time)
  790. # log_.info('last video redis update success!')
  791. # 将此次获取的 地域分组小时级数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  792. rov_recall_h_video = [item['videoId'] for item in result[:top_K]
  793. if item['pushFrom'] == config_.PUSH_FROM['rov_recall_region_h']]
  794. if len(rov_recall_h_video) > 0:
  795. last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_H_PREFIX}{app_type}:{mid}'
  796. redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_h_video[-1],
  797. expire_time=expire_time)
  798. # 将此次获取的 地域分组相对24h数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  799. rov_recall_24h_dup1_video = [item['videoId'] for item in result[:top_K]
  800. if item['pushFrom'] == config_.PUSH_FROM['rov_recall_region_24h']]
  801. if len(rov_recall_24h_dup1_video) > 0:
  802. last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP1_24H_PREFIX}{app_type}:{mid}'
  803. redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup1_video[-1],
  804. expire_time=expire_time)
  805. # 将此次获取的 相对24h筛选数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  806. rov_recall_24h_dup2_video = [item['videoId'] for item in result[:top_K]
  807. if item['pushFrom'] == config_.PUSH_FROM['rov_recall_24h']]
  808. if len(rov_recall_24h_dup2_video) > 0:
  809. last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP2_24H_PREFIX}{app_type}:{mid}'
  810. redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup2_video[-1],
  811. expire_time=expire_time)
  812. # 将此次获取的 相对24h筛选后剩余数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  813. rov_recall_24h_dup3_video = [item['videoId'] for item in result[:top_K]
  814. if item['pushFrom'] == config_.PUSH_FROM['rov_recall_24h_dup']]
  815. if len(rov_recall_24h_dup3_video) > 0:
  816. last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP3_24H_PREFIX}{app_type}:{mid}'
  817. redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup3_video[-1],
  818. expire_time=expire_time)
  819. # # 将此次获取的 相对48h筛选数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  820. # rov_recall_48h_dup2_video = [item['videoId'] for item in result[:top_K]
  821. # if item['pushFrom'] == config_.PUSH_FROM['rov_recall_48h']]
  822. # if len(rov_recall_48h_dup2_video) > 0:
  823. # last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP2_48H_PREFIX}{app_type}:{mid}'
  824. # redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_48h_dup2_video[-1],
  825. # expire_time=expire_time)
  826. #
  827. # # 将此次获取的 相对48h筛选后剩余数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  828. # rov_recall_48h_dup3_video = [item['videoId'] for item in result[:top_K]
  829. # if item['pushFrom'] == config_.PUSH_FROM['rov_recall_48h_dup']]
  830. # if len(rov_recall_48h_dup3_video) > 0:
  831. # last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP3_48H_PREFIX}{app_type}:{mid}'
  832. # redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_48h_dup3_video[-1],
  833. # expire_time=expire_time)
  834. # 将此次分发的流量池视频,对 本地分发数-1 进行记录
  835. if app_type not in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  836. # 获取本地分发数-1策略开关
  837. switch = redis_helper.get_data_from_redis(key_name=config_.IN_FLOW_POOL_COUNT_SWITCH_KEY_NAME)
  838. if switch is not None:
  839. if int(switch) == 1:
  840. flow_recall_video = [item for item in result if item.get('flowPool', None) is not None]
  841. else:
  842. flow_recall_video = [item for item in result if
  843. item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
  844. else:
  845. flow_recall_video = [item for item in result if item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
  846. if flow_recall_video:
  847. update_local_distribute_count(flow_recall_video)
  848. # log_.info('update local distribute count success!')
  849. # 限流视频分发数记录
  850. if app_type == config_.APP_TYPE['APP']:
  851. # APP 不计入
  852. return
  853. limit_video_id_list = redis_helper.get_data_from_set(
  854. key_name=f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_SET}{datetime.today().strftime('%Y%m%d')}"
  855. )
  856. if limit_video_id_list is not None:
  857. limit_video_id_list = [int(item) for item in limit_video_id_list]
  858. for item in result:
  859. video_id = item['videoId']
  860. if video_id in limit_video_id_list:
  861. key_name = f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_DISTRIBUTE_COUNT}{video_id}"
  862. redis_helper.setnx_key(key_name=key_name, value=0, expire_time=24*2600)
  863. redis_helper.incr_key(key_name=key_name, amount=1, expire_time=24*3600)
  864. except Exception as e:
  865. log_.error("update redis data fail!")
  866. log_.error(traceback.format_exc())
  867. def update_flow_redis_data(result, app_type, mid, top_K, expire_time=24*3600):
  868. """
  869. 根据最终的排序结果更新相关redis数据
  870. :param result: 排序结果
  871. :param app_type: 产品标识
  872. :param mid: mid
  873. :param top_K: 保证topK为召回池视频 type-int
  874. :param expire_time: 末位视频记录redis过期时间
  875. :return: None
  876. """
  877. # ####### redis数据刷新
  878. try:
  879. redis_helper = RedisHelper()
  880. # log_.info('====== update redis')
  881. if mid and mid != 'null':
  882. # mid为空时,不做预曝光和定位数据更新
  883. # 预曝光数据同步刷新到Redis, 过期时间为0.5h
  884. preview_key_name = f"{config_.PREVIEW_KEY_PREFIX}{app_type}:{mid}"
  885. preview_video_ids = [int(item['videoId']) for item in result]
  886. if preview_video_ids:
  887. # log_.error('key_name = {} \n values = {}'.format(preview_key_name, tuple(preview_video_ids)))
  888. redis_helper.add_data_with_set(key_name=preview_key_name, values=tuple(preview_video_ids), expire_time=30 * 60)
  889. # log_.info('preview redis update success!')
  890. # 将此次分发的流量池视频,对 本地分发数-1 进行记录
  891. if app_type not in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  892. # 获取本地分发数-1策略开关
  893. switch = redis_helper.get_data_from_redis(key_name=config_.IN_FLOW_POOL_COUNT_SWITCH_KEY_NAME)
  894. if switch is not None:
  895. if int(switch) == 1:
  896. flow_recall_video = [item for item in result if item.get('flowPool', None) is not None]
  897. else:
  898. flow_recall_video = [item for item in result if
  899. item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
  900. else:
  901. flow_recall_video = [item for item in result if item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
  902. if flow_recall_video:
  903. update_local_distribute_count(flow_recall_video)
  904. # log_.info('update local distribute count success!')
  905. # 限流视频分发数记录
  906. if app_type == config_.APP_TYPE['APP']:
  907. # APP 不计入
  908. return
  909. limit_video_id_list = redis_helper.get_data_from_set(
  910. key_name=f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_SET}{datetime.today().strftime('%Y%m%d')}"
  911. )
  912. if limit_video_id_list is not None:
  913. limit_video_id_list = [int(item) for item in limit_video_id_list]
  914. for item in result:
  915. video_id = item['videoId']
  916. if video_id in limit_video_id_list:
  917. key_name = f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_DISTRIBUTE_COUNT}{video_id}"
  918. redis_helper.setnx_key(key_name=key_name, value=0, expire_time=24*2600)
  919. redis_helper.incr_key(key_name=key_name, amount=1, expire_time=24*3600)
  920. except Exception as e:
  921. log_.error("update redis data fail!")
  922. log_.error(traceback.format_exc())
  923. def update_local_distribute_count(videos):
  924. """
  925. 更新本地分发数
  926. :param videos: 视频列表 type-list [{'videoId':'', 'flowPool':'', 'distributeCount': '',
  927. 'rovScore': '', 'pushFrom': 'flow_pool', 'abCode': self.ab_code}, ....]
  928. :return:
  929. """
  930. try:
  931. redis_helper = RedisHelper()
  932. for item in videos:
  933. video_id, flow_pool = item['videoId'], item['flowPool']
  934. key_name = f"{config_.LOCAL_DISTRIBUTE_COUNT_PREFIX}{video_id}:{flow_pool}"
  935. # 本地记录的分发数 - 1
  936. redis_helper.decr_key(key_name=key_name, amount=1, expire_time=15 * 60)
  937. # 对该视频做分发数检查
  938. cur_count = redis_helper.get_data_from_redis(key_name=key_name)
  939. # 无记录
  940. if cur_count is None:
  941. continue
  942. # 本地分发数 cur_count <= 0,从所有的流量召回池移除,删除本地分发记录key
  943. if int(cur_count) <= 0:
  944. add_remove_log = False
  945. redis_helper.del_keys(key_name=key_name)
  946. for app_name in config_.APP_TYPE:
  947. app_type = config_.APP_TYPE.get(app_name)
  948. flow_pool_key_list = [
  949. f"{config_.FLOWPOOL_KEY_NAME_PREFIX}{app_type}",
  950. f"{config_.QUICK_FLOWPOOL_KEY_NAME_PREFIX}{app_type}:{config_.QUICK_FLOW_POOL_ID}"
  951. ]
  952. for key in flow_pool_key_list:
  953. remove_res = redis_helper.remove_value_from_zset(key_name=key, value=f"{video_id}-{flow_pool}")
  954. if remove_res > 0:
  955. add_remove_log = True
  956. video_flow_pool_key_list = [
  957. f"{config_.QUICK_FLOWPOOL_VIDEO_INFO_KEY_NAME_PREFIX}{app_type}:{config_.QUICK_FLOW_POOL_ID}:{video_id}",
  958. f"{config_.FLOWPOOL_VIDEO_INFO_KEY_NAME_PREFIX}{app_type}:{video_id}"
  959. ]
  960. for key in video_flow_pool_key_list:
  961. redis_helper.remove_value_from_set(key_name=key, values=(flow_pool, ))
  962. if add_remove_log is True:
  963. log_.info({'tag': 'remove video_id from flow_pool', 'video_id': video_id, 'flow_pool': flow_pool})
  964. # if redis_helper.key_exists(key_name=key_name):
  965. # # 该视频本地有记录,本地记录的分发数 - 1
  966. # redis_helper.decr_key(key_name=key_name, amount=1, expire_time=5 * 60)
  967. # else:
  968. # # 该视频本地无记录,接口获取的分发数 - 1
  969. # redis_helper.incr_key(key_name=key_name, amount=int(item['distributeCount']) - 1, expire_time=5 * 60)
  970. except Exception as e:
  971. log_.error('update_local_distribute_count error...')
  972. log_.error(traceback.format_exc())
  973. def get_religion_class_with_mid(mid, religion_class_name):
  974. """
  975. 判断用户是否属于对应的宗教类型
  976. :param mid: mid type-string
  977. :param religion_class_name: 宗教类型, type-string, (catholicism-天主教, christianity-基督教)
  978. :return: religion_class_flag, type-int, (0-否,1-是), 默认: 0
  979. """
  980. religion_class_flag = 0
  981. now_date = datetime.today()
  982. redis_helper = RedisHelper()
  983. if mid:
  984. hash_mid = hashlib.md5(mid.encode('utf-8')).hexdigest()
  985. hash_tag = hash_mid[-1:]
  986. key_name_prefix = config_.KEY_NAME_PREFIX_RELIGION_USER.get(religion_class_name, None)
  987. if key_name_prefix is None:
  988. return religion_class_flag
  989. key_name = f"{key_name_prefix}{hash_tag}:{datetime.strftime(now_date, '%Y%m%d')}"
  990. if not redis_helper.key_exists(key_name=key_name):
  991. key_name = f"{key_name_prefix}{hash_tag}:{datetime.strftime(now_date - timedelta(days=1), '%Y%m%d')}"
  992. if redis_helper.data_exists_with_set(key_name=key_name, value=mid):
  993. religion_class_flag = 1
  994. return religion_class_flag
  995. def get_recommend_params(recommend_type, ab_exp_info, ab_info_data, mid, app_type, page_type=0):
  996. """
  997. 根据实验分组给定对应的推荐参数
  998. :param recommend_type: 首页推荐和相关推荐区分参数(0-首页推荐,1-相关推荐)
  999. :param ab_exp_info: AB实验组参数
  1000. :param ab_info_data: app实验组参数
  1001. :param mid: mid
  1002. :param app_type: app_type, type-int
  1003. :param page_type: 页面区分参数,默认:0(首页)
  1004. :return:
  1005. """
  1006. top_K = config_.K
  1007. flow_pool_P = config_.P
  1008. # 不获取人工干预数据标记
  1009. no_op_flag = True
  1010. expire_time = 3600
  1011. old_video_index = -1
  1012. # 获取对应的默认配置
  1013. ab_initial_config = config_.INITIAL_CONFIG.get(app_type, None)
  1014. if ab_initial_config is None:
  1015. ab_initial_config = config_.INITIAL_CONFIG.get('other')
  1016. param = config_.AB_EXP_CODE[ab_initial_config]
  1017. ab_code = param.get('ab_code')
  1018. rule_key = param.get('rule_key')
  1019. data_key = param.get('data_key')
  1020. rule_key_30day = param.get('30day_rule_key')
  1021. shield_config = config_.SHIELD_CONFIG
  1022. # 默认使用 095 实验的配置
  1023. # ab_code = config_.AB_EXP_CODE['095'].get('ab_code')
  1024. # rule_key = config_.AB_EXP_CODE['095'].get('rule_key')
  1025. # data_key = config_.AB_EXP_CODE['095'].get('data_key')
  1026. # rule_key_30day = None
  1027. # 获取用户近30天是否有回流
  1028. # user_30day_return_result = get_user_has30day_return(mid=mid)
  1029. # 获取实验配置
  1030. if ab_exp_info:
  1031. ab_exp_code_list = []
  1032. config_value_dict = {}
  1033. for _, item in ab_exp_info.items():
  1034. if not item:
  1035. continue
  1036. for ab_item in item:
  1037. ab_exp_code = ab_item.get('abExpCode', None)
  1038. if not ab_exp_code:
  1039. continue
  1040. ab_exp_code_list.append(str(ab_exp_code))
  1041. config_value_dict[str(ab_exp_code)] = ab_item.get('configValue', None)
  1042. # 流量池视频分发概率实验
  1043. if '211' in ab_exp_code_list:
  1044. flow_pool_P = 0.9
  1045. elif '221' in ab_exp_code_list:
  1046. flow_pool_P = 0.7
  1047. elif '299' in ab_exp_code_list:
  1048. flow_pool_P = 0.5
  1049. elif '300' in ab_exp_code_list:
  1050. flow_pool_P = 0.4
  1051. elif '301' in ab_exp_code_list:
  1052. flow_pool_P = 0.6
  1053. elif '339' in ab_exp_code_list:
  1054. flow_pool_P = 0
  1055. # if '136' in ab_exp_code_list:
  1056. # # 无回流 - 消费人群
  1057. # if user_30day_return_result == 0:
  1058. # param = config_.AB_EXP_CODE.get('136')
  1059. # ab_code = param.get('ab_code')
  1060. # expire_time = 3600
  1061. # rule_key = param.get('rule_key')
  1062. # data_key = param.get('data_key')
  1063. # no_op_flag = True
  1064. # elif '137' in ab_exp_code_list:
  1065. # # 有回流 - 分享人群
  1066. # if user_30day_return_result == 1:
  1067. # param = config_.AB_EXP_CODE.get('137')
  1068. # ab_code = param.get('ab_code')
  1069. # expire_time = 3600
  1070. # rule_key = param.get('rule_key')
  1071. # data_key = param.get('data_key')
  1072. # no_op_flag = True
  1073. # elif '161' in ab_exp_code_list:
  1074. # # 无回流 - 消费人群
  1075. # if user_30day_return_result == 0:
  1076. # param = config_.AB_EXP_CODE.get('136')
  1077. # ab_code = param.get('ab_code')
  1078. # expire_time = 3600
  1079. # rule_key = param.get('rule_key')
  1080. # data_key = param.get('data_key')
  1081. # no_op_flag = True
  1082. # # 有回流 - 分享人群
  1083. # else:
  1084. # param = config_.AB_EXP_CODE.get('137')
  1085. # ab_code = param.get('ab_code')
  1086. # expire_time = 3600
  1087. # rule_key = param.get('rule_key')
  1088. # data_key = param.get('data_key')
  1089. # no_op_flag = True
  1090. # elif '162' in ab_exp_code_list:
  1091. # # 有回流
  1092. # if user_30day_return_result == 1:
  1093. # param = config_.AB_EXP_CODE.get('162')
  1094. # ab_code = param.get('ab_code')
  1095. # expire_time = 3600
  1096. # rule_key = param.get('rule_key')
  1097. # data_key = param.get('data_key')
  1098. # no_op_flag = True
  1099. # 老好看视频 宗教人群实验
  1100. if '228' in ab_exp_code_list:
  1101. # 天主教
  1102. religion_param = config_.AB_EXP_CODE['228']
  1103. religion_class_name = religion_param.get('religion_class_name')
  1104. religion_class_flag = get_religion_class_with_mid(mid=mid, religion_class_name=religion_class_name)
  1105. if religion_class_flag == 1:
  1106. ab_code = religion_param.get('ab_code')
  1107. rule_key = religion_param.get('rule_key')
  1108. data_key = religion_param.get('data_key')
  1109. rule_key_30day = religion_param.get('30day_rule_key')
  1110. elif '229' in ab_exp_code_list:
  1111. # 基督教
  1112. religion_param = config_.AB_EXP_CODE['229']
  1113. religion_class_name = religion_param.get('religion_class_name')
  1114. religion_class_flag = get_religion_class_with_mid(mid=mid, religion_class_name=religion_class_name)
  1115. if religion_class_flag == 1:
  1116. ab_code = religion_param.get('ab_code')
  1117. rule_key = religion_param.get('rule_key')
  1118. data_key = religion_param.get('data_key')
  1119. rule_key_30day = religion_param.get('30day_rule_key')
  1120. else:
  1121. for code, param in config_.AB_EXP_CODE.items():
  1122. if code in ab_exp_code_list:
  1123. ab_code = param.get('ab_code')
  1124. rule_key = param.get('rule_key')
  1125. data_key = param.get('data_key')
  1126. rule_key_30day = param.get('30day_rule_key')
  1127. shield_config = param.get('shield_config', config_.SHIELD_CONFIG)
  1128. break
  1129. """
  1130. # 推荐条数 10->4 实验
  1131. # if config_.AB_EXP_CODE['rec_size_home'] in ab_exp_code_list:
  1132. # config_value = config_value_dict.get(config_.AB_EXP_CODE['rec_size_home'], None)
  1133. # if config_value:
  1134. # config_value = eval(str(config_value))
  1135. # else:
  1136. # config_value = {}
  1137. # log_.info(f'config_value: {config_value}, type: {type(config_value)}')
  1138. # size = int(config_value.get('size', 4))
  1139. # top_K = int(config_value.get('K', 3))
  1140. # flow_pool_P = float(config_value.get('P', 0.3))
  1141. # else:
  1142. # size = size
  1143. # top_K = config_.K
  1144. # flow_pool_P = config_.P
  1145. # 算法实验相对对照组
  1146. # if config_.AB_EXP_CODE['ab_initial'] in ab_exp_code_list:
  1147. # ab_code = config_.AB_CODE['ab_initial']
  1148. # expire_time = 24 * 3600
  1149. # rule_key = config_.RULE_KEY['initial']
  1150. # no_op_flag = True
  1151. # 小时级更新-规则1 实验
  1152. # elif config_.AB_EXP_CODE['rule_rank1'] in ab_exp_code_list:
  1153. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank1')
  1154. # expire_time = 3600
  1155. # rule_key = config_.RULE_KEY['rule_rank1']
  1156. # no_op_flag = True
  1157. # elif config_.AB_EXP_CODE['rule_rank2'] in ab_exp_code_list:
  1158. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank2')
  1159. # expire_time = 3600
  1160. # rule_key = config_.RULE_KEY['rule_rank2']
  1161. # elif config_.AB_EXP_CODE['rule_rank3'] in ab_exp_code_list:
  1162. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank3')
  1163. # expire_time = 3600
  1164. # rule_key = config_.RULE_KEY['rule_rank3']
  1165. # no_op_flag = True
  1166. # elif config_.AB_EXP_CODE['rule_rank4'] in ab_exp_code_list:
  1167. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank4')
  1168. # expire_time = 3600
  1169. # rule_key = config_.RULE_KEY['rule_rank4']
  1170. # elif config_.AB_EXP_CODE['rule_rank5'] in ab_exp_code_list:
  1171. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank5')
  1172. # expire_time = 3600
  1173. # rule_key = config_.RULE_KEY['rule_rank5']
  1174. # elif config_.AB_EXP_CODE['day_rule_rank1'] in ab_exp_code_list:
  1175. # ab_code = config_.AB_CODE['rank_by_day'].get('day_rule_rank1')
  1176. # expire_time = 24 * 3600
  1177. # rule_key = config_.RULE_KEY_DAY['day_rule_rank1']
  1178. # no_op_flag = True
  1179. # if config_.AB_EXP_CODE['rule_rank6'] in ab_exp_code_list:
  1180. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank6')
  1181. # expire_time = 3600
  1182. # rule_key = config_.RULE_KEY['rule_rank6']
  1183. # no_op_flag = True
  1184. # elif config_.AB_EXP_CODE['day_rule_rank2'] in ab_exp_code_list:
  1185. # ab_code = config_.AB_CODE['rank_by_day'].get('day_rule_rank2')
  1186. # expire_time = 24 * 3600
  1187. # rule_key = config_.RULE_KEY_DAY['day_rule_rank2']
  1188. # no_op_flag = True
  1189. # elif config_.AB_EXP_CODE['region_rule_rank1'] in ab_exp_code_list:
  1190. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank1')
  1191. # expire_time = 3600
  1192. # rule_key = config_.RULE_KEY_REGION['region_rule_rank1']
  1193. # no_op_flag = True
  1194. # elif config_.AB_EXP_CODE['24h_rule_rank1'] in ab_exp_code_list:
  1195. # ab_code = config_.AB_CODE['rank_by_24h'].get('24h_rule_rank1')
  1196. # expire_time = 3600
  1197. # rule_key = config_.RULE_KEY_24H['24h_rule_rank1']
  1198. # no_op_flag = True
  1199. # elif config_.AB_EXP_CODE['24h_rule_rank2'] in ab_exp_code_list:
  1200. # ab_code = config_.AB_CODE['rank_by_24h'].get('24h_rule_rank2')
  1201. # expire_time = 3600
  1202. # rule_key = config_.RULE_KEY_24H['24h_rule_rank2']
  1203. # no_op_flag = True
  1204. # elif config_.AB_EXP_CODE['region_rule_rank2'] in ab_exp_code_list:
  1205. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank2')
  1206. # expire_time = 3600
  1207. # rule_key = config_.RULE_KEY_REGION['region_rule_rank2']
  1208. # no_op_flag = True
  1209. # if config_.AB_EXP_CODE['region_rule_rank3'] in ab_exp_code_list or\
  1210. # config_.AB_EXP_CODE['region_rule_rank3_appType_19'] in ab_exp_code_list or\
  1211. # config_.AB_EXP_CODE['region_rule_rank3_appType_4'] in ab_exp_code_list or\
  1212. # config_.AB_EXP_CODE['region_rule_rank3_appType_6'] in ab_exp_code_list or\
  1213. # config_.AB_EXP_CODE['region_rule_rank3_appType_18'] in ab_exp_code_list:
  1214. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank3')
  1215. # expire_time = 3600
  1216. # rule_key = config_.RULE_KEY_REGION['region_rule_rank3'].get('rule_key')
  1217. # data_key = config_.RULE_KEY_REGION['region_rule_rank3'].get('data_key')
  1218. # no_op_flag = True
  1219. # if config_.AB_EXP_CODE['region_rule_rank4'] in ab_exp_code_list or\
  1220. if config_.AB_EXP_CODE['region_rule_rank4_appType_19'] in ab_exp_code_list or \
  1221. config_.AB_EXP_CODE['region_rule_rank4_appType_4'] in ab_exp_code_list or\
  1222. config_.AB_EXP_CODE['region_rule_rank4_appType_6'] in ab_exp_code_list or\
  1223. config_.AB_EXP_CODE['region_rule_rank4_appType_18'] in ab_exp_code_list:
  1224. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
  1225. expire_time = 3600
  1226. rule_key = config_.RULE_KEY_REGION['region_rule_rank4'].get('rule_key')
  1227. data_key = config_.RULE_KEY_REGION['region_rule_rank4'].get('data_key')
  1228. no_op_flag = True
  1229. # elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data1'] in ab_exp_code_list:
  1230. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
  1231. # expire_time = 3600
  1232. # rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data1'].get('rule_key')
  1233. # data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data1'].get('data_key')
  1234. # no_op_flag = True
  1235. # elif config_.AB_EXP_CODE['region_rule_rank3_appType_5_data2'] in ab_exp_code_list:
  1236. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank3_appType_5_data2')
  1237. # expire_time = 3600
  1238. # rule_key = config_.RULE_KEY_REGION['region_rule_rank3_appType_5_data2'].get('rule_key')
  1239. # data_key = config_.RULE_KEY_REGION['region_rule_rank3_appType_5_data2'].get('data_key')
  1240. # no_op_flag = True
  1241. elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data3'] in ab_exp_code_list:
  1242. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_5_data3')
  1243. expire_time = 3600
  1244. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data3'].get('rule_key')
  1245. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data3'].get('data_key')
  1246. no_op_flag = True
  1247. elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data4'] in ab_exp_code_list:
  1248. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_5_data4')
  1249. expire_time = 3600
  1250. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data4'].get('rule_key')
  1251. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data4'].get('data_key')
  1252. no_op_flag = True
  1253. elif config_.AB_EXP_CODE['region_rule_rank4_appType_0_data2'] in ab_exp_code_list:
  1254. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_0_data2')
  1255. expire_time = 3600
  1256. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_0_data2'].get('rule_key')
  1257. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_0_data2'].get('data_key')
  1258. no_op_flag = True
  1259. # elif config_.AB_EXP_CODE['region_rule_rank4_appType_19_data2'] in ab_exp_code_list:
  1260. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_19_data2')
  1261. # expire_time = 3600
  1262. # rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data2'].get('rule_key')
  1263. # data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data2'].get('data_key')
  1264. # no_op_flag = True
  1265. # elif config_.AB_EXP_CODE['region_rule_rank4_appType_19_data3'] in ab_exp_code_list:
  1266. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_19_data3')
  1267. # expire_time = 3600
  1268. # rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data3'].get('rule_key')
  1269. # data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data3'].get('data_key')
  1270. # no_op_flag = True
  1271. elif config_.AB_EXP_CODE['region_rule_rank5_appType_0_data1'] in ab_exp_code_list:
  1272. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank5_appType_0_data1')
  1273. expire_time = 3600
  1274. rule_key = config_.RULE_KEY_REGION['region_rule_rank5_appType_0_data1'].get('rule_key')
  1275. data_key = config_.RULE_KEY_REGION['region_rule_rank5_appType_0_data1'].get('data_key')
  1276. no_op_flag = True
  1277. elif config_.AB_EXP_CODE['region_rule_rank4_appType_4_data2'] in ab_exp_code_list:
  1278. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_4_data2')
  1279. expire_time = 3600
  1280. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data2'].get('rule_key')
  1281. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data2'].get('data_key')
  1282. no_op_flag = True
  1283. elif config_.AB_EXP_CODE['region_rule_rank4_appType_4_data3'] in ab_exp_code_list:
  1284. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_4_data3')
  1285. expire_time = 3600
  1286. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data3'].get('rule_key')
  1287. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data3'].get('data_key')
  1288. no_op_flag = True
  1289. elif config_.AB_EXP_CODE['region_rule_rank4_appType_6_data2'] in ab_exp_code_list:
  1290. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_6_data2')
  1291. expire_time = 3600
  1292. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data2'].get('rule_key')
  1293. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data2'].get('data_key')
  1294. no_op_flag = True
  1295. elif config_.AB_EXP_CODE['region_rule_rank4_appType_6_data3'] in ab_exp_code_list:
  1296. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_6_data3')
  1297. expire_time = 3600
  1298. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data3'].get('rule_key')
  1299. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data3'].get('data_key')
  1300. no_op_flag = True
  1301. # elif config_.AB_EXP_CODE['region_rule_rank4_appType_18_data2'] in ab_exp_code_list:
  1302. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_18_data2')
  1303. # expire_time = 3600
  1304. # rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_18_data2'].get('rule_key')
  1305. # data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_18_data2'].get('data_key')
  1306. # no_op_flag = True
  1307. # elif config_.AB_EXP_CODE['region_rule_rank6_appType_0_data1'] in ab_exp_code_list:
  1308. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank6_appType_0_data1')
  1309. # expire_time = 3600
  1310. # rule_key = config_.RULE_KEY_REGION['region_rule_rank6_appType_0_data1'].get('rule_key')
  1311. # data_key = config_.RULE_KEY_REGION['region_rule_rank6_appType_0_data1'].get('data_key')
  1312. # no_op_flag = True
  1313. else:
  1314. ab_code = config_.AB_CODE['initial']
  1315. expire_time = 24 * 3600
  1316. rule_key = config_.RULE_KEY_REGION['initial'].get('rule_key')
  1317. data_key = config_.RULE_KEY_REGION['initial'].get('data_key')
  1318. # # 老好看视频 / 票圈最惊奇 首页/相关推荐逻辑更新实验
  1319. # if config_.AB_EXP_CODE['rov_rank_appType_18_19'] in ab_exp_code_list:
  1320. # ab_code = config_.AB_CODE['rov_rank_appType_18_19']
  1321. # expire_time = 3600
  1322. # flow_pool_P = config_.P_18_19
  1323. # no_op_flag = True
  1324. #
  1325. # elif config_.AB_EXP_CODE['rov_rank_appType_19'] in ab_exp_code_list:
  1326. # ab_code = config_.AB_CODE['rov_rank_appType_19']
  1327. # expire_time = 3600
  1328. # top_K = 0
  1329. # flow_pool_P = config_.P_18_19
  1330. # no_op_flag = True
  1331. #
  1332. # elif config_.AB_EXP_CODE['top_video_relevant_appType_19'] in ab_exp_code_list and page_type == 2:
  1333. # ab_code = config_.AB_CODE['top_video_relevant_appType_19']
  1334. # expire_time = 3600
  1335. # top_K = 1
  1336. # flow_pool_P = config_.P_18_19
  1337. # no_op_flag = True
  1338. #
  1339. # # 票圈最惊奇完整影视资源实验
  1340. # elif config_.AB_EXP_CODE['whole_movies'] in ab_exp_code_list:
  1341. # ab_code = config_.AB_CODE['whole_movies']
  1342. # expire_time = 24 * 3600
  1343. # no_op_flag = True
  1344. # 老视频实验
  1345. # if config_.AB_EXP_CODE['old_video'] in ab_exp_code_list:
  1346. # ab_code = config_.AB_CODE['old_video']
  1347. # no_op_flag = True
  1348. # old_video_index = 2
  1349. # else:
  1350. # old_video_index = -1
  1351. """
  1352. # APP实验组
  1353. if ab_info_data:
  1354. ab_info_app = {}
  1355. for page_code, item in json.loads(ab_info_data).items():
  1356. if not item:
  1357. continue
  1358. ab_info_code = item.get('eventId', None)
  1359. if ab_info_code:
  1360. ab_info_app[page_code] = ab_info_code
  1361. # print(f"======{ab_info_app}")
  1362. # 首页推荐
  1363. if recommend_type == 0:
  1364. app_ab_code = ab_info_app.get('10003', None)
  1365. for code, param in config_.APP_AB_CODE['10003'].items():
  1366. if code == app_ab_code:
  1367. ab_code = param.get('ab_code')
  1368. rule_key = param.get('rule_key')
  1369. data_key = param.get('data_key')
  1370. break
  1371. # # 相关推荐
  1372. # elif recommend_type == 1:
  1373. # if config_.APP_AB_CODE['10037'] == ab_info_app.get('10037', None):
  1374. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
  1375. # expire_time = 3600
  1376. # rule_key = 'rule3'
  1377. # data_key = 'data1'
  1378. # no_op_flag = True
  1379. return top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, no_op_flag, old_video_index, rule_key_30day, shield_config
  1380. def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
  1381. client_info, ab_exp_info, params, ab_info_data, version_audit_status):
  1382. """
  1383. 首页线上推荐逻辑
  1384. :param request_id: request_id
  1385. :param mid: mid type-string
  1386. :param uid: uid type-string
  1387. :param size: 请求视频数量 type-int
  1388. :param app_type: 产品标识 type-int
  1389. :param algo_type: 算法类型 type-string
  1390. :param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
  1391. :param ab_exp_info: ab实验分组参数 [{"expItemId":1, "configValue":{"size":4, "K":3, ...}}, ...]
  1392. :param params:
  1393. :param ab_info_data: app实验分组参数
  1394. :param version_audit_status: 小程序版本审核参数:1-审核中,2-审核通过
  1395. :return:
  1396. """
  1397. # 对 vlog 切换10%的流量做实验
  1398. # 对mid进行哈希
  1399. # hash_mid = hashlib.md5(mid.encode('utf-8')).hexdigest()
  1400. # if app_type in config_.AB_TEST['rank_by_h'] and hash_mid[-1:] in ['8', '0', 'a', 'b']:
  1401. # # 简单召回 - 排序 - 兜底
  1402. # rank_result, last_rov_recall_key = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
  1403. # algo_type=algo_type, client_info=client_info,
  1404. # expire_time=3600,
  1405. # ab_code=config_.AB_CODE['rank_by_h'])
  1406. # # ab-test
  1407. # result = ab_test_op(rank_result=rank_result,
  1408. # ab_code_list=[config_.AB_CODE['position_insert']],
  1409. # app_type=app_type, mid=mid, uid=uid)
  1410. # # redis数据刷新
  1411. # update_redis_data(result=result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
  1412. # expire_time=3600)
  1413. # if app_type == config_.APP_TYPE['APP']:
  1414. # # 票圈视频APP
  1415. # top_K = config_.K
  1416. # flow_pool_P = config_.P
  1417. # # 简单召回 - 排序 - 兜底
  1418. # rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
  1419. # mid=mid, uid=uid, app_type=app_type,
  1420. # size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1421. # algo_type=algo_type, client_info=client_info,
  1422. # expire_time=12 * 3600, params=params)
  1423. # # ab-test
  1424. # # result = ab_test_op(rank_result=rank_result,
  1425. # # ab_code_list=[config_.AB_CODE['position_insert']],
  1426. # # app_type=app_type, mid=mid, uid=uid)
  1427. # # redis数据刷新
  1428. # update_redis_data(result=rank_result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
  1429. # top_K=top_K, expire_time=12 * 3600)
  1430. #
  1431. # else:
  1432. recommend_result = {}
  1433. param_st = time.time()
  1434. # 特殊mid 和 小程序审核版本推荐处理
  1435. if mid in get_special_mid_list() or version_audit_status == 1:
  1436. rank_result = special_mid_recommend(request_id=request_id, mid=mid, uid=uid, app_type=app_type, size=size)
  1437. recommend_result['videos'] = rank_result
  1438. return recommend_result
  1439. # 普通mid推荐处理
  1440. top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, \
  1441. no_op_flag, old_video_index, rule_key_30day, shield_config = \
  1442. get_recommend_params(recommend_type=0, ab_exp_info=ab_exp_info, ab_info_data=ab_info_data, mid=mid,
  1443. app_type=app_type)
  1444. # log_.info({
  1445. # 'logTimestamp': int(time.time() * 1000),
  1446. # 'request_id': request_id,
  1447. # 'app_type': app_type,
  1448. # 'mid': mid,
  1449. # 'uid': uid,
  1450. # 'operation': 'get_recommend_params',
  1451. # 'executeTime': (time.time() - param_st) * 1000
  1452. # })
  1453. recommend_result['getRecommendParamsTime'] = (time.time() - param_st) * 1000
  1454. # 简单召回 - 排序 - 兜底
  1455. get_result_st = time.time()
  1456. #print("ab_code:", ab_code)
  1457. #new pipeline
  1458. if ab_code == 60047 or ab_code == 60048 or ab_code == 60049:
  1459. result = new_video_recommend(request_id=request_id,
  1460. mid=mid, uid=uid, app_type=app_type,
  1461. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1462. algo_type=algo_type, client_info=client_info,
  1463. ab_code=ab_code, expire_time=expire_time,
  1464. rule_key=rule_key, data_key=data_key,
  1465. no_op_flag=no_op_flag, old_video_index=old_video_index,
  1466. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1467. # old base (test4, test5)
  1468. elif ab_code == 60050 or ab_code == 60051:
  1469. result = video_recommend(request_id=request_id,
  1470. mid=mid, uid=uid, app_type=app_type,
  1471. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1472. algo_type=algo_type, client_info=client_info,
  1473. ab_code=ab_code, expire_time=expire_time,
  1474. rule_key=rule_key, data_key=data_key,
  1475. no_op_flag=no_op_flag, old_video_index=old_video_index,
  1476. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1477. # 60052,60053,60057(test7, test8, test6)
  1478. # simrecal: 60054
  1479. # 3 days , 7days recall: 60005, test1
  1480. # old video: 60056, test2
  1481. elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or ab_code == 60056 or ab_code==60057:
  1482. result = video_old_recommend(request_id=request_id,
  1483. mid=mid, uid=uid, app_type=app_type,
  1484. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1485. algo_type='', client_info=client_info,
  1486. ab_code=ab_code, expire_time=expire_time,
  1487. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1488. old_video_index=old_video_index, video_id= None,
  1489. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1490. else:
  1491. result = video_recommend(request_id=request_id,
  1492. mid=mid, uid=uid, app_type=app_type,
  1493. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1494. algo_type=algo_type, client_info=client_info,
  1495. ab_code=ab_code, expire_time=expire_time,
  1496. rule_key=rule_key, data_key=data_key,
  1497. no_op_flag=no_op_flag, old_video_index=old_video_index,
  1498. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1499. # log_.info({
  1500. # 'logTimestamp': int(time.time() * 1000),
  1501. # 'request_id': request_id,
  1502. # 'app_type': app_type,
  1503. # 'mid': mid,
  1504. # 'uid': uid,
  1505. # 'operation': 'get_recommend_result',
  1506. # 'executeTime': (time.time() - get_result_st) * 1000
  1507. # })
  1508. recommend_result['recommendOperation'] = result
  1509. rank_result = result.get('rankResult')
  1510. recommend_result['videos'] = rank_result
  1511. recommend_result['getRecommendResultTime'] = (time.time() - get_result_st) * 1000
  1512. # ab-test
  1513. # result = ab_test_op(rank_result=rank_result,
  1514. # ab_code_list=[config_.AB_CODE['position_insert']],
  1515. # app_type=app_type, mid=mid, uid=uid)
  1516. # redis数据刷新
  1517. update_redis_st = time.time()
  1518. if ab_code == 60047 or ab_code == 60048 or ab_code == 60049:
  1519. update_flow_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1520. elif ab_code == 60050 or ab_code == 60051:
  1521. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1522. elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 \
  1523. or ab_code == 60056 or ab_code==60057:
  1524. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1525. else:
  1526. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1527. # log_.info({
  1528. # 'logTimestamp': int(time.time() * 1000),
  1529. # 'request_id': request_id,
  1530. # 'app_type': app_type,
  1531. # 'mid': mid,
  1532. # 'uid': uid,
  1533. # 'operation': 'update_redis_data',
  1534. # 'executeTime': (time.time() - update_redis_st) * 1000
  1535. # })
  1536. recommend_result['updateRedisDataTime'] = (time.time() - update_redis_st) * 1000
  1537. return recommend_result
  1538. # return rank_result
  1539. def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_exp_info, client_info,
  1540. page_type, params, ab_info_data, version_audit_status):
  1541. """
  1542. 相关推荐逻辑
  1543. :param request_id: request_id
  1544. :param video_id: 相关推荐的头部视频id
  1545. :param mid: mid type-string
  1546. :param uid: uid type-string
  1547. :param size: 请求视频数量 type-int
  1548. :param app_type: 产品标识 type-int
  1549. :param ab_exp_info: ab实验分组参数 [{"expItemId":1, "configValue":{"size":4, "K":3, ...}}, ...]
  1550. :param client_info: 地域参数
  1551. :param page_type: 页面区分参数 1:详情页;2:分享页
  1552. :param params:
  1553. :param ab_info_data: app实验分组参数
  1554. :param version_audit_status: 小程序版本审核参数:1-审核中,2-审核通过
  1555. :return: videos type-list
  1556. """
  1557. recommend_result = {}
  1558. param_st = time.time()
  1559. # 特殊mid 和 小程序审核版本推荐处理
  1560. if mid in get_special_mid_list() or version_audit_status == 1:
  1561. rank_result = special_mid_recommend(request_id=request_id, mid=mid, uid=uid, app_type=app_type, size=size)
  1562. recommend_result['videos'] = rank_result
  1563. return recommend_result
  1564. # return rank_result
  1565. # 普通mid推荐处理
  1566. top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, \
  1567. no_op_flag, old_video_index, rule_key_30day, shield_config = \
  1568. get_recommend_params(recommend_type=1, ab_exp_info=ab_exp_info, ab_info_data=ab_info_data, page_type=page_type,
  1569. mid=mid, app_type=app_type)
  1570. # log_.info({
  1571. # 'logTimestamp': int(time.time() * 1000),
  1572. # 'request_id': request_id,
  1573. # 'app_type': app_type,
  1574. # 'mid': mid,
  1575. # 'uid': uid,
  1576. # 'operation': 'get_recommend_params',
  1577. # 'executeTime': (time.time() - param_st) * 1000
  1578. # })
  1579. recommend_result['getRecommendParamsTime'] = (time.time() - param_st) * 1000
  1580. # 简单召回 - 排序 - 兜底
  1581. get_result_st = time.time()
  1582. #print("ab_code:", ab_code)
  1583. if ab_code == 60047 or ab_code == 60048 or ab_code == 60049:
  1584. result = new_video_recommend(request_id=request_id,
  1585. mid=mid, uid=uid, app_type=app_type,
  1586. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1587. algo_type='', client_info=client_info,
  1588. ab_code=ab_code, expire_time=expire_time,
  1589. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1590. old_video_index=old_video_index, video_id=video_id,
  1591. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1592. # log_.info({
  1593. elif ab_code == 60050 or ab_code == 60051:
  1594. result = video_recommend(request_id=request_id,
  1595. mid=mid, uid=uid, app_type=app_type,
  1596. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1597. algo_type='', client_info=client_info,
  1598. ab_code=ab_code, expire_time=expire_time,
  1599. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1600. old_video_index=old_video_index, video_id=video_id,
  1601. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1602. elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
  1603. ab_code == 60056 or ab_code==60057:
  1604. result = video_old_recommend(request_id=request_id,
  1605. mid=mid, uid=uid, app_type=app_type,
  1606. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1607. algo_type='', client_info=client_info,
  1608. ab_code=ab_code, expire_time=expire_time,
  1609. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1610. old_video_index=old_video_index, video_id=video_id,
  1611. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1612. else:
  1613. result = video_recommend(request_id=request_id,
  1614. mid=mid, uid=uid, app_type=app_type,
  1615. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1616. algo_type='', client_info=client_info,
  1617. ab_code=ab_code, expire_time=expire_time,
  1618. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1619. old_video_index=old_video_index, video_id=video_id,
  1620. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1621. # log_.info({
  1622. # 'logTimestamp': int(time.time() * 1000),
  1623. # 'request_id': request_id,
  1624. # 'app_type': app_type,
  1625. # 'mid': mid,
  1626. # 'uid': uid,
  1627. # 'operation': 'get_recommend_result',
  1628. # 'executeTime': (time.time() - get_result_st) * 1000
  1629. # })
  1630. recommend_result['recommendOperation'] = result
  1631. rank_result = result.get('rankResult')
  1632. recommend_result['videos'] = rank_result
  1633. recommend_result['getRecommendResultTime'] = (time.time() - get_result_st) * 1000
  1634. # ab-test
  1635. # result = ab_test_op(rank_result=rank_result,
  1636. # ab_code_list=[config_.AB_CODE['position_insert'], config_.AB_CODE['relevant_video_op']],
  1637. # app_type=app_type, mid=mid, uid=uid, head_vid=video_id, size=size)
  1638. # redis数据刷新
  1639. update_redis_st = time.time()
  1640. if ab_code == 60047 or ab_code == 60048 or ab_code == 60049:
  1641. update_flow_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1642. elif ab_code == 60050 or ab_code == 60051:
  1643. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1644. elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
  1645. ab_code == 60056 or ab_code==60057:
  1646. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1647. else:
  1648. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1649. # log_.info({
  1650. # 'logTimestamp': int(time.time() * 1000),
  1651. # 'request_id': request_id,
  1652. # 'app_type': app_type,
  1653. # 'mid': mid,
  1654. # 'uid': uid,
  1655. # 'operation': 'update_redis_data',
  1656. # 'executeTime': (time.time() - update_redis_st) * 1000
  1657. # })
  1658. recommend_result['updateRedisDataTime'] = (time.time() - update_redis_st) * 1000
  1659. return recommend_result
  1660. # return rank_result
  1661. def special_mid_recommend(request_id, mid, uid, app_type, size,
  1662. ab_code=config_.AB_CODE['special_mid'],
  1663. push_from=config_.PUSH_FROM['special_mid'],
  1664. expire_time=24*3600):
  1665. redis_helper = RedisHelper()
  1666. # 特殊mid推荐指定视频列表
  1667. pool_recall = PoolRecall(request_id=request_id, app_type=app_type,
  1668. mid=mid, uid=uid, ab_code=ab_code)
  1669. # 获取相关redis key
  1670. special_key_name, redis_date = pool_recall.get_pool_redis_key(pool_type='special')
  1671. # 用户上一次在rov召回池对应的位置
  1672. last_special_recall_key = f'{config_.LAST_VIDEO_FROM_SPECIAL_POOL_PREFIX}{app_type}:{mid}:{redis_date}'
  1673. value = redis_helper.get_data_from_redis(last_special_recall_key)
  1674. if value:
  1675. idx = redis_helper.get_index_with_data(special_key_name, value)
  1676. if not idx:
  1677. idx = 0
  1678. else:
  1679. idx += 1
  1680. else:
  1681. idx = 0
  1682. recall_result = []
  1683. # 每次获取的视频数
  1684. get_size = size * 5
  1685. # 记录获取频次
  1686. freq = 0
  1687. while len(recall_result) < size:
  1688. freq += 1
  1689. if freq > config_.MAX_FREQ_FROM_ROV_POOL:
  1690. break
  1691. # 获取数据
  1692. data = redis_helper.get_data_zset_with_index(key_name=special_key_name,
  1693. start=idx, end=idx + get_size - 1,
  1694. with_scores=True)
  1695. if not data:
  1696. break
  1697. # 获取视频id,并转换类型为int,并存储为key-value{videoId: score}
  1698. # 添加视频源参数 pushFrom, abCode
  1699. temp_result = [{'videoId': int(value[0]), 'rovScore': value[1],
  1700. 'pushFrom': push_from, 'abCode': ab_code}
  1701. for value in data]
  1702. recall_result.extend(temp_result)
  1703. idx += get_size
  1704. # 将此次获取的末位视频id同步刷新到Redis中,方便下次快速定位到召回位置,过期时间为1天
  1705. if mid and recall_result:
  1706. # mid为空时,不做记录
  1707. redis_helper.set_data_to_redis(key_name=last_special_recall_key,
  1708. value=recall_result[:size][-1]['videoId'],
  1709. expire_time=expire_time)
  1710. return recall_result[:size]
  1711. def get_special_mid_list():
  1712. redis_helper = RedisHelper()
  1713. special_mid_list = redis_helper.get_data_from_set(key_name=config_.KEY_NAME_SPECIAL_MID)
  1714. if special_mid_list:
  1715. return special_mid_list
  1716. else:
  1717. return []
  1718. if __name__ == '__main__':
  1719. videos = [
  1720. {"videoId": 10136461, "rovScore": 99.971, "pushFrom": "recall_pool", "abCode": 10000},
  1721. {"videoId": 10239014, "rovScore": 99.97, "pushFrom": "recall_pool", "abCode": 10000},
  1722. {"videoId": 9851154, "rovScore": 99.969, "pushFrom": "recall_pool", "abCode": 10000},
  1723. {"videoId": 10104347, "rovScore": 99.968, "pushFrom": "recall_pool", "abCode": 10000},
  1724. {"videoId": 10141507, "rovScore": 99.967, "pushFrom": "recall_pool", "abCode": 10000},
  1725. {"videoId": 10292817, "flowPool": "2#6#2#1641780979606", "rovScore": 53.926690610816486,
  1726. "pushFrom": "flow_pool", "abCode": 10000},
  1727. {"videoId": 10224932, "flowPool": "2#5#1#1641800279644", "rovScore": 53.47890460059617, "pushFrom": "flow_pool",
  1728. "abCode": 10000},
  1729. {"videoId": 9943255, "rovScore": 99.966, "pushFrom": "recall_pool", "abCode": 10000},
  1730. {"videoId": 10282970, "flowPool": "2#5#1#1641784814103", "rovScore": 52.682815076325575,
  1731. "pushFrom": "flow_pool", "abCode": 10000},
  1732. {"videoId": 10282205, "rovScore": 99.965, "pushFrom": "recall_pool", "abCode": 10000}
  1733. ]
  1734. res = relevant_video_top_recommend(app_type=4, mid='', uid=1111, head_vid=123, videos=videos, size=10)
  1735. print(res)