recommend.py 92 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899
  1. import json
  2. import time
  3. import multiprocessing
  4. import traceback
  5. import hashlib
  6. from datetime import datetime, timedelta
  7. import config
  8. from log import Log
  9. from config import set_config
  10. from video_recall import PoolRecall
  11. from video_rank import video_new_rank2, video_sank_pos_rank,video_new_rank,video_rank,refactor_video_rank, bottom_strategy, video_rank_by_w_h_rate, video_rank_with_old_video, bottom_strategy2
  12. from db_helper import RedisHelper
  13. import gevent
  14. from utils import FilterVideos, get_user_has30day_return
  15. import ast
  16. log_ = Log()
  17. config_ = set_config()
  18. def relevant_video_top_recommend(request_id, app_type, mid, uid, head_vid, videos, size):
  19. """
  20. 相关推荐强插 运营给定置顶相关性视频
  21. :param request_id: request_id
  22. :param app_type: 产品标识 type-int
  23. :param mid: mid
  24. :param uid: uid
  25. :param head_vid: 相关推荐头部视频id type-int
  26. :param videos: 当前相关推荐结果 type-list
  27. :param size: 返回视频个数 type-int
  28. :return: rank_result
  29. """
  30. # 获取头部视频对应的相关性视频
  31. key_name = '{}{}'.format(config_.RELEVANT_VIDEOS_WITH_OP_KEY_NAME, head_vid)
  32. redis_helper = RedisHelper()
  33. relevant_videos = redis_helper.get_data_from_redis(key_name=key_name)
  34. if relevant_videos is None:
  35. # 该视频没有指定的相关性视频
  36. return videos
  37. relevant_videos = json.loads(relevant_videos)
  38. # 按照指定顺序排序
  39. relevant_videos_sorted = sorted(relevant_videos, key=lambda x: x['order'], reverse=False)
  40. # 过滤
  41. relevant_video_ids = [int(item['recommend_vid']) for item in relevant_videos_sorted]
  42. filter_helper = FilterVideos(request_id=request_id, app_type=app_type, video_ids=relevant_video_ids, mid=mid, uid=uid)
  43. filtered_ids = filter_helper.filter_videos()
  44. if filtered_ids is None:
  45. return videos
  46. # 获取生效中的视频
  47. now = int(time.time())
  48. relevant_videos_in_effect = [
  49. {'videoId': int(item['recommend_vid']), 'pushFrom': config_.PUSH_FROM['relevant_video_op'],
  50. 'abCode': config_.AB_CODE['relevant_video_op']}
  51. for item in relevant_videos_sorted
  52. if item['start_time'] < now < item['finish_time'] and int(item['recommend_vid']) in filtered_ids
  53. ]
  54. if len(relevant_videos_in_effect) == 0:
  55. return videos
  56. # 与现有排序结果 进行合并重排
  57. # 获取现有排序结果中流量池视频 及其位置
  58. relevant_ids = [item['videoId'] for item in relevant_videos_in_effect]
  59. flow_pool_videos = []
  60. other_videos = []
  61. for i, item in enumerate(videos):
  62. if item.get('pushFrom', None) == config_.PUSH_FROM['flow_recall'] and item.get('videoId') not in relevant_ids:
  63. flow_pool_videos.append((i, item))
  64. elif item.get('videoId') not in relevant_ids:
  65. other_videos.append(item)
  66. else:
  67. continue
  68. # 重排,保持流量池视频位置不变
  69. rank_result = relevant_videos_in_effect + other_videos
  70. for i, item in flow_pool_videos:
  71. rank_result.insert(i, item)
  72. return rank_result[:size]
  73. def video_position_recommend(request_id, mid, uid, app_type, videos):
  74. # videos = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
  75. # algo_type=algo_type, client_info=client_info)
  76. redis_helper = RedisHelper()
  77. pos1_vids = redis_helper.get_data_from_redis(config.BaseConfig.RECALL_POSITION1_KEY_NAME)
  78. pos2_vids = redis_helper.get_data_from_redis(config.BaseConfig.RECALL_POSITION2_KEY_NAME)
  79. if pos1_vids is not None:
  80. pos1_vids = ast.literal_eval(pos1_vids)
  81. if pos2_vids is not None:
  82. pos2_vids = ast.literal_eval(pos2_vids)
  83. pos1_vids = [] if pos1_vids is None else pos1_vids
  84. pos2_vids = [] if pos2_vids is None else pos2_vids
  85. pos1_vids = [int(i) for i in pos1_vids]
  86. pos2_vids = [int(i) for i in pos2_vids]
  87. filter_1 = FilterVideos(request_id=request_id, app_type=app_type, video_ids=pos1_vids, mid=mid, uid=uid)
  88. filter_2 = FilterVideos(request_id=request_id, app_type=app_type, video_ids=pos2_vids, mid=mid, uid=uid)
  89. t = [gevent.spawn(filter_1.filter_videos), gevent.spawn(filter_2.filter_videos)]
  90. gevent.joinall(t)
  91. filted_list = [i.get() for i in t]
  92. pos1_vids = filted_list[0]
  93. pos2_vids = filted_list[1]
  94. videos = positon_duplicate(pos1_vids, pos2_vids, videos)
  95. if pos1_vids is not None and len(pos1_vids) >0 :
  96. videos.insert(0, {'videoId': int(pos1_vids[0]), 'rovScore': 100,
  97. 'pushFrom': config_.PUSH_FROM['position_insert'], 'abCode': config_.AB_CODE['position_insert']})
  98. if pos2_vids is not None and len(pos2_vids) >0 :
  99. videos.insert(1, {'videoId': int(pos2_vids[0]), 'rovScore': 100,
  100. 'pushFrom': config_.PUSH_FROM['position_insert'], 'abCode': config_.AB_CODE['position_insert']})
  101. return videos[:10]
  102. def positon_duplicate(pos1_vids, pos2_vids, videos):
  103. s = set()
  104. if pos1_vids is not None and len(pos1_vids) >0:
  105. s.add(int(pos1_vids[0]))
  106. if pos2_vids is not None and len(pos2_vids) >0:
  107. s.add(int(pos2_vids[0]))
  108. l = []
  109. for item in videos:
  110. if item['videoId'] in s:
  111. continue
  112. else:
  113. l.append(item)
  114. return l
  115. def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, algo_type, client_info,
  116. expire_time=24*3600, ab_code=config_.AB_CODE['initial'], rule_key='', data_key='',
  117. no_op_flag=False, old_video_index=-1, video_id=None, params=None, rule_key_30day=None,
  118. shield_config=None):
  119. """
  120. 首页线上推荐逻辑
  121. :param request_id: request_id
  122. :param mid: mid type-string
  123. :param uid: uid type-string
  124. :param size: 请求视频数量 type-int
  125. :param top_K: 保证topK为召回池视频 type-int
  126. :param flow_pool_P: size-top_K视频为流量池视频的概率 type-float
  127. :param app_type: 产品标识 type-int
  128. :param algo_type: 算法类型 type-string
  129. :param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
  130. :param expire_time: 末位视频记录redis过期时间
  131. :param ab_code: AB实验code
  132. :param video_id: 相关推荐头部视频id
  133. :param params:
  134. :return:
  135. """
  136. result = {}
  137. # ####### 多进程召回
  138. start_recall = time.time()
  139. # log_.info('====== recall')
  140. '''
  141. cores = multiprocessing.cpu_count()
  142. pool = multiprocessing.Pool(processes=cores)
  143. pool_recall = PoolRecall(app_type=app_type, mid=mid, uid=uid, ab_code=ab_code)
  144. _, last_rov_recall_key, _ = pool_recall.get_video_last_idx()
  145. pool_list = [
  146. # rov召回池
  147. pool.apply_async(pool_recall.rov_pool_recall, (size,)),
  148. # 流量池
  149. pool.apply_async(pool_recall.flow_pool_recall, (size,))
  150. ]
  151. recall_result_list = [p.get() for p in pool_list]
  152. pool.close()
  153. pool.join()
  154. '''
  155. recall_result_list = []
  156. pool_recall = PoolRecall(request_id=request_id,
  157. app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
  158. client_info=client_info, rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  159. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config, video_id= video_id)
  160. # _, last_rov_recall_key, _ = pool_recall.get_video_last_idx()
  161. # # 小时级实验
  162. # if ab_code in [code for _, code in config_.AB_CODE['rank_by_h'].items()]:
  163. # t = [gevent.spawn(pool_recall.rule_recall_by_h, size, expire_time),
  164. # gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  165. # gevent.spawn(pool_recall.flow_pool_recall, size)]
  166. # # 小时级实验
  167. # elif ab_code in [code for _, code in config_.AB_CODE['rank_by_24h'].items()]:
  168. # t = [gevent.spawn(pool_recall.rov_pool_recall_by_h, size, expire_time),
  169. # gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  170. # gevent.spawn(pool_recall.flow_pool_recall, size)]
  171. # 地域分组实验
  172. # if ab_code in [code for _, code in config_.AB_CODE['region_rank_by_h'].items()]:
  173. # load test config
  174. exp_config = None
  175. if ab_code == 60058:
  176. exp_config = pool_recall.get_U2I_config()
  177. elif ab_code == 60059:
  178. exp_config = pool_recall.get_w2v_config()
  179. elif ab_code == 60060:
  180. exp_config = pool_recall.get_test_config()
  181. elif ab_code == 60061:
  182. exp_config = pool_recall.get_simrecall_config()
  183. elif ab_code == 60062:
  184. exp_config = pool_recall.get_u2u2i_config()
  185. #print("exp_config:", exp_config)
  186. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  187. t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config)]
  188. if ab_code==60058:
  189. t.append(gevent.spawn(pool_recall.get_U2I_reall, mid, exp_config))
  190. elif ab_code==60059:
  191. t.append(gevent.spawn(pool_recall.get_word2vec_item_reall, exp_config))
  192. elif ab_code==60061:
  193. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
  194. elif ab_code==60062:
  195. t.append(gevent.spawn(pool_recall.get_U2U2I_reall, mid, exp_config))
  196. else:
  197. t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config),
  198. gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  199. gevent.spawn(pool_recall.flow_pool_recall, size)]
  200. if ab_code==60058:
  201. t.append(gevent.spawn(pool_recall.get_U2I_reall, mid, exp_config))
  202. elif ab_code == 60059:
  203. t.append(gevent.spawn(pool_recall.get_word2vec_item_reall, exp_config))
  204. elif ab_code == 60061:
  205. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
  206. elif ab_code == 60062:
  207. t.append(gevent.spawn(pool_recall.get_U2U2I_reall, mid, exp_config))
  208. # 最惊奇相关推荐实验
  209. # elif ab_code == config_.AB_CODE['top_video_relevant_appType_19']:
  210. # t = [gevent.spawn(pool_recall.relevant_recall_19, video_id, size, expire_time),
  211. # gevent.spawn(pool_recall.flow_pool_recall_18_19, size)]
  212. # 最惊奇完整影视实验
  213. # elif ab_code == config_.AB_CODE['whole_movies']:
  214. # t = [gevent.spawn(pool_recall.rov_pool_recall_19, size, expire_time)]
  215. # 最惊奇/老好看实验
  216. # elif app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  217. # t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
  218. # gevent.spawn(pool_recall.flow_pool_recall_18_19, size)]
  219. # # 天级实验
  220. # elif ab_code in [code for _, code in config_.AB_CODE['rank_by_day'].items()]:
  221. # t = [gevent.spawn(pool_recall.rov_pool_recall_by_day, size, expire_time),
  222. # gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  223. # gevent.spawn(pool_recall.flow_pool_recall, size)]
  224. # 老视频实验
  225. # elif ab_code in [config_.AB_CODE['old_video']]:
  226. # t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
  227. # gevent.spawn(pool_recall.flow_pool_recall, size),
  228. # gevent.spawn(pool_recall.old_videos_recall, size)]
  229. # else:
  230. # if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  231. # t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time)]
  232. # else:
  233. # t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time),
  234. # gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  235. # gevent.spawn(pool_recall.flow_pool_recall, size)]
  236. gevent.joinall(t)
  237. recall_result_list = [i.get() for i in t]
  238. # end_recall = time.time()
  239. # log_.info({
  240. # 'logTimestamp': int(time.time() * 1000),
  241. # 'request_id': request_id,
  242. # 'mid': mid,
  243. # 'uid': uid,
  244. # 'operation': 'recall',
  245. # 'recall_result': recall_result_list,
  246. # 'executeTime': (time.time() - start_recall) * 1000
  247. # })
  248. result['recallResult'] = recall_result_list
  249. result['recallTime'] = (time.time() - start_recall) * 1000
  250. # ####### 排序
  251. start_rank = time.time()
  252. # log_.info('====== rank')
  253. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  254. if ab_code in [
  255. config_.AB_CODE['rov_rank_appType_18_19'],
  256. config_.AB_CODE['rov_rank_appType_19'],
  257. config_.AB_CODE['top_video_relevant_appType_19']
  258. ]:
  259. data = {
  260. 'rov_pool_recall': recall_result_list[0],
  261. 'flow_pool_recall': recall_result_list[1]
  262. }
  263. else:
  264. data = {
  265. 'rov_pool_recall': recall_result_list[0],
  266. 'flow_pool_recall': []
  267. }
  268. else:
  269. if recall_result_list[1]:
  270. redis_helper = RedisHelper()
  271. quick_flow_pool_P = redis_helper.get_data_from_redis(
  272. key_name=f"{config_.QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX}{config_.QUICK_FLOW_POOL_ID}"
  273. )
  274. if quick_flow_pool_P:
  275. flow_pool_P = quick_flow_pool_P
  276. data = {
  277. 'rov_pool_recall': recall_result_list[0],
  278. 'flow_pool_recall': recall_result_list[1]
  279. }
  280. else:
  281. data = {
  282. 'rov_pool_recall': recall_result_list[0],
  283. 'flow_pool_recall': recall_result_list[2]
  284. }
  285. data['u2i_recall'] = []
  286. data['w2v_recall'] = []
  287. data['sim_recall'] = []
  288. data['u2u2i_recall'] = []
  289. if ab_code == 60058:
  290. if len(recall_result_list)>=4:
  291. data['u2i_recall'] = recall_result_list[3]
  292. elif ab_code == 60059:
  293. if len(recall_result_list)>=4:
  294. data['w2v_recall'] = recall_result_list[3]
  295. elif ab_code == 60061:
  296. if len(recall_result_list)>=4:
  297. data['sim_recall'] = recall_result_list[3]
  298. elif ab_code == 60062:
  299. if len(recall_result_list)>=4:
  300. data['u2u2i_recall'] = recall_result_list[3]
  301. #if ab_code=="ab_new_test":
  302. # rank_result = video_new_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
  303. #else:
  304. #print("data['hot_recall']", data['hot_recall'])
  305. # 60058: u2itag, 60059:word2vec, 60061: sim_recall, 60062: u2u2i
  306. if ab_code == 60058 or ab_code == 60059 or ab_code == 60060 or ab_code == 60061 or ab_code == 60062:
  307. rank_result, flow_num = video_sank_pos_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P), ab_Code=ab_code, exp_config=exp_config)
  308. result['flow_num'] = flow_num
  309. if rank_result:
  310. result['rank_num'] = len(rank_result)
  311. else:
  312. rank_result = video_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
  313. # 老视频实验
  314. # if ab_code in [config_.AB_CODE['old_video']]:
  315. # rank_result = video_rank_with_old_video(rank_result=rank_result, old_video_recall=recall_result_list[2],
  316. # size=size, top_K=top_K, old_video_index=old_video_index)
  317. # end_rank = time.time()
  318. # log_.info({
  319. # 'logTimestamp': int(time.time() * 1000),
  320. # 'request_id': request_id,
  321. # 'mid': mid,
  322. # 'uid': uid,
  323. # 'operation': 'rank',
  324. # 'rank_result': rank_result,
  325. # 'executeTime': (time.time() - start_rank) * 1000
  326. # })
  327. result['rankResult'] = rank_result
  328. result['rankTime'] = (time.time() - start_rank) * 1000
  329. # if not rank_result:
  330. # # 兜底策略
  331. # # log_.info('====== bottom strategy')
  332. # start_bottom = time.time()
  333. # rank_result = bottom_strategy2(
  334. # size=size, app_type=app_type, mid=mid, uid=uid, ab_code=ab_code, client_info=client_info, params=params
  335. # )
  336. #
  337. # # if ab_code == config_.AB_CODE['region_rank_by_h'].get('abtest_130'):
  338. # # rank_result = bottom_strategy2(
  339. # # size=size, app_type=app_type, mid=mid, uid=uid, ab_code=ab_code, client_info=client_info, params=params
  340. # # )
  341. # # else:
  342. # # rank_result = bottom_strategy(
  343. # # request_id=request_id, size=size, app_type=app_type, ab_code=ab_code, params=params
  344. # # )
  345. #
  346. # # log_.info({
  347. # # 'logTimestamp': int(time.time() * 1000),
  348. # # 'request_id': request_id,
  349. # # 'mid': mid,
  350. # # 'uid': uid,
  351. # # 'operation': 'bottom',
  352. # # 'bottom_result': rank_result,
  353. # # 'executeTime': (time.time() - start_bottom) * 1000
  354. # # })
  355. # result['bottomResult'] = rank_result
  356. # result['bottomTime'] = (time.time() - start_bottom) * 1000
  357. #
  358. # result['rankResult'] = rank_result
  359. return result
  360. # return rank_result, last_rov_recall_key
  361. def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, algo_type, client_info,
  362. expire_time=24*3600, ab_code=config_.AB_CODE['initial'], rule_key='', data_key='',
  363. no_op_flag=False, old_video_index=-1, video_id=None, params=None, rule_key_30day=None,
  364. shield_config=None):
  365. """
  366. 首页线上推荐逻辑
  367. :param request_id: request_id
  368. :param mid: mid type-string
  369. :param uid: uid type-string
  370. :param size: 请求视频数量 type-int
  371. :param top_K: 保证topK为召回池视频 type-int
  372. :param flow_pool_P: size-top_K视频为流量池视频的概率 type-float
  373. :param app_type: 产品标识 type-int
  374. :param algo_type: 算法类型 type-string
  375. :param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
  376. :param expire_time: 末位视频记录redis过期时间
  377. :param ab_code: AB实验code
  378. :param video_id: 相关推荐头部视频id
  379. :param params:
  380. :return:
  381. """
  382. result = {}
  383. # ####### 多进程召回
  384. start_recall = time.time()
  385. # log_.info('====== recall')
  386. #print("abcode",ab_code)
  387. recall_result_list = []
  388. pool_recall = PoolRecall(request_id=request_id,
  389. app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
  390. client_info=client_info, rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  391. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config, video_id= video_id)
  392. exp_config = None
  393. if ab_code == 60052:
  394. exp_config = pool_recall.get_flow_exp_7_config()
  395. elif ab_code == 60053:
  396. exp_config = pool_recall.get_flow_exp_8_config()
  397. elif ab_code == 60057:
  398. exp_config = pool_recall.get_flow_exp_6_config()
  399. #print("exp_config:", exp_config)
  400. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  401. t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config)]
  402. if ab_code ==60054:
  403. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
  404. if ab_code == 60055:
  405. t.append(gevent.spawn(pool_recall.get_3days_hot_item_reall, exp_config))
  406. if ab_code == 60056:
  407. t.append(gevent.spawn(pool_recall.get_hot_item_reall, exp_config))
  408. else:
  409. t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config),
  410. gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  411. gevent.spawn(pool_recall.flow_pool_recall, size)]
  412. if ab_code == 60054:
  413. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter))
  414. if ab_code == 60055:
  415. t.append(gevent.spawn(pool_recall.get_3days_hot_item_reall))
  416. if ab_code == 60056:
  417. t.append(gevent.spawn(pool_recall.get_hot_item_reall))
  418. gevent.joinall(t)
  419. recall_result_list = [i.get() for i in t]
  420. #print("recall:",recall_result_list)
  421. if len(recall_result_list)<0:
  422. result['recallResult']= []
  423. result['rankResult'] = []
  424. return result
  425. #1. merge simrecall
  426. if ab_code == 60054:
  427. rov_pool_recall = []
  428. if len(recall_result_list) >= 2:
  429. region_recall = recall_result_list[0]
  430. sim_recall = []
  431. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  432. sim_recall = recall_result_list[1]
  433. else:
  434. if len(recall_result_list)>=4:
  435. sim_recall = recall_result_list[3]
  436. now_video_ids = set('')
  437. if len(region_recall) > 0:
  438. for video in region_recall:
  439. video_id = video.get('videoId')
  440. if video_id not in now_video_ids:
  441. rov_pool_recall.append(video)
  442. now_video_ids.add(video_id)
  443. if len(sim_recall) > 0:
  444. for video in sim_recall:
  445. video_id = video.get('videoId')
  446. # print("sim video_id:", video_id)
  447. if video_id not in now_video_ids:
  448. rov_pool_recall.append(video)
  449. now_video_ids.add(video_id)
  450. if len(rov_pool_recall) > 0:
  451. recall_result_list[0] = rov_pool_recall
  452. # merge hot 3day recall
  453. if ab_code == 60055:
  454. rov_pool_recall = []
  455. if len(recall_result_list)>=2:
  456. region_recall = recall_result_list[0]
  457. hot_3_day_recall = []
  458. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  459. hot_3_day_recall = recall_result_list[1]
  460. else:
  461. if len(recall_result_list)>=4:
  462. hot_3_day_recall = recall_result_list[3]
  463. #print("sim:",sim_recall)
  464. now_video_ids = set('')
  465. if len(region_recall)>0:
  466. for video in region_recall:
  467. video_id = video.get('videoId')
  468. if video_id not in now_video_ids:
  469. rov_pool_recall.append(video)
  470. now_video_ids.add(video_id)
  471. if len(hot_3_day_recall)>0:
  472. for video in hot_3_day_recall:
  473. video_id = video.get('videoId')
  474. #print("sim video_id:", video_id)
  475. if video_id not in now_video_ids:
  476. rov_pool_recall.append(video)
  477. now_video_ids.add(video_id)
  478. if len(rov_pool_recall)>0:
  479. recall_result_list[0] = rov_pool_recall
  480. # merge old day recall
  481. if ab_code == 60056:
  482. rov_pool_recall = []
  483. if len(recall_result_list)>=2:
  484. region_recall = recall_result_list[0]
  485. hot_7_day_recall = []
  486. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  487. hot_7_day_recall = recall_result_list[1]
  488. else:
  489. if len(recall_result_list) >= 3:
  490. hot_7_day_recall = recall_result_list[3]
  491. now_video_ids = set('')
  492. if len(region_recall)>0:
  493. for video in region_recall:
  494. video_id = video.get('videoId')
  495. if video_id not in now_video_ids:
  496. rov_pool_recall.append(video)
  497. now_video_ids.add(video_id)
  498. if len(hot_7_day_recall)>0:
  499. for video in hot_7_day_recall:
  500. video_id = video.get('videoId')
  501. #print("sim video_id:", video_id)
  502. if video_id not in now_video_ids:
  503. rov_pool_recall.append(video)
  504. now_video_ids.add(video_id)
  505. if len(rov_pool_recall)>0:
  506. recall_result_list[0] = rov_pool_recall
  507. result['recallResult'] = recall_result_list
  508. result['recallTime'] = (time.time() - start_recall) * 1000
  509. #print("recall:", recall_result_list)
  510. # ####### 排序
  511. start_rank = time.time()
  512. # log_.info('====== rank')
  513. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  514. if ab_code in [
  515. config_.AB_CODE['rov_rank_appType_18_19'],
  516. config_.AB_CODE['rov_rank_appType_19'],
  517. config_.AB_CODE['top_video_relevant_appType_19']
  518. ]:
  519. data = {
  520. 'rov_pool_recall': recall_result_list[0],
  521. 'flow_pool_recall': recall_result_list[1]
  522. }
  523. else:
  524. data = {
  525. 'rov_pool_recall': recall_result_list[0],
  526. 'flow_pool_recall': []
  527. }
  528. else:
  529. if recall_result_list[1]:
  530. redis_helper = RedisHelper()
  531. quick_flow_pool_P = redis_helper.get_data_from_redis(
  532. key_name=f"{config_.QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX}{config_.QUICK_FLOW_POOL_ID}"
  533. )
  534. if quick_flow_pool_P:
  535. flow_pool_P = quick_flow_pool_P
  536. data = {
  537. 'rov_pool_recall': recall_result_list[0],
  538. 'flow_pool_recall': recall_result_list[1]
  539. }
  540. else:
  541. data = {
  542. 'rov_pool_recall': recall_result_list[0],
  543. 'flow_pool_recall': recall_result_list[2]
  544. }
  545. #if ab_code=="ab_new_test":
  546. #print("before data:", data)
  547. rank_result, flow_num = video_new_rank2(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P), ab_code=ab_code, exp_config=exp_config)
  548. #print(rank_result)
  549. if rank_result:
  550. result['rank_num'] = len(rank_result)
  551. result['rankResult'] = rank_result
  552. result['flow_num'] = flow_num
  553. result['rankTime'] = (time.time() - start_rank) * 1000
  554. return result
  555. # return rank_result, last_rov_recall_key
  556. def new_video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, algo_type, client_info,
  557. expire_time=24*3600, ab_code=config_.AB_CODE['initial'], rule_key='', data_key='',
  558. no_op_flag=False, old_video_index=-1, video_id=None, params=None, rule_key_30day=None,
  559. shield_config=None):
  560. """
  561. 首页线上推荐逻辑
  562. :param request_id: request_id
  563. :param mid: mid type-string
  564. :param uid: uid type-string
  565. :param size: 请求视频数量 type-int
  566. :param top_K: 保证topK为召回池视频 type-int
  567. :param flow_pool_P: size-top_K视频为流量池视频的概率 type-float
  568. :param app_type: 产品标识 type-int
  569. :param algo_type: 算法类型 type-string
  570. :param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
  571. :param expire_time: 末位视频记录redis过期时间
  572. :param ab_code: AB实验code
  573. :param video_id: 相关推荐头部视频id
  574. :param params:
  575. :return:
  576. """
  577. #1. recall
  578. result = {}
  579. result['rankResult'] = []
  580. # ####### 多进程召回
  581. start_recall = time.time()
  582. # 1. 根据城市或者省份获取region_code
  583. city_code_list = [code for _, code in config_.CITY_CODE.items()]
  584. # 获取provinceCode
  585. province_code = client_info.get('provinceCode', '-1')
  586. # 获取cityCode
  587. city_code = client_info.get('cityCode', '-1')
  588. if city_code in city_code_list:
  589. # 分城市数据存在时,获取城市分组数据
  590. region_code = city_code
  591. else:
  592. region_code = province_code
  593. if region_code == '':
  594. region_code = '-1'
  595. #print("region_code:", region_code)
  596. #size =1000
  597. pool_recall = PoolRecall(request_id=request_id,
  598. app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
  599. client_info=client_info, rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  600. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config, video_id= video_id)
  601. if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  602. t = [gevent.spawn(pool_recall.get_region_hour_recall, size, region_code),
  603. gevent.spawn(pool_recall.get_region_day_recall, size, region_code),
  604. gevent.spawn(pool_recall.get_selected_recall, size, region_code),
  605. gevent.spawn(pool_recall.get_no_selected_recall, size, region_code)
  606. ]
  607. if ab_code == 60049:
  608. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall))
  609. else:
  610. t = [
  611. gevent.spawn(pool_recall.get_region_hour_recall, size, region_code),
  612. gevent.spawn(pool_recall.get_region_day_recall, size, region_code),
  613. gevent.spawn(pool_recall.get_selected_recall, size, region_code),
  614. gevent.spawn(pool_recall.get_no_selected_recall, size, region_code),
  615. gevent.spawn(pool_recall.new_flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
  616. gevent.spawn(pool_recall.new_flow_pool_recall, size)]
  617. if ab_code ==60049:
  618. t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall))
  619. gevent.joinall(t)
  620. # all recall_result
  621. all_recall_result_list = [i.get() for i in t]
  622. all_recall_result = []
  623. #print(all_recall_result_list)
  624. result['recallTime'] = (time.time() - start_recall) * 1000
  625. #print("recall time:", result['recallTime'])
  626. if not all_recall_result_list or len(all_recall_result_list)==0:
  627. return result
  628. for recall_item in all_recall_result_list:
  629. if not recall_item or len(recall_item)==0:
  630. continue
  631. for per_item in recall_item:
  632. all_recall_result.append(per_item)
  633. #print("all_recall_result:", all_recall_result)
  634. #2. duplicate
  635. dup_time = time.time()
  636. recall_dict = {}
  637. fast_flow_set = set('')
  638. flow_flow_set = set('')
  639. region_h_recall = []
  640. region_day_recall = []
  641. select_day_recall = []
  642. no_selected_recall = []
  643. sim_hot_recall = []
  644. flow_recall = []
  645. flowFlag_dict = {}
  646. for per_item in all_recall_result:
  647. #print(per_item)
  648. try:
  649. vId = int(per_item.get("videoId",0))
  650. if vId==0:
  651. continue
  652. recall_name = per_item.get("pushFrom",'')
  653. flow_pool = per_item.get("flowPool", '')
  654. if flow_pool != '':
  655. flow_pool_id = int(flow_pool.split('#')[0])
  656. if flow_pool_id == config_.QUICK_FLOW_POOL_ID:
  657. fast_flow_set.add(vId)
  658. else:
  659. flow_flow_set.add(vId)
  660. flowFlag_dict[vId] = flow_pool
  661. #duplicate divide into
  662. if vId not in recall_dict:
  663. if recall_name == config_.PUSH_FROM['rov_recall_region_h']:
  664. region_h_recall.append(per_item)
  665. elif recall_name == config_.PUSH_FROM['rov_recall_region_24h']:
  666. region_day_recall.append(per_item)
  667. elif recall_name == config_.PUSH_FROM['rov_recall_24h']:
  668. select_day_recall.append(per_item)
  669. elif recall_name == config_.PUSH_FROM['rov_recall_24h_dup']:
  670. no_selected_recall.append(per_item)
  671. elif recall_name == config_.PUSH_FROM['sim_hot_vid_recall']:
  672. sim_hot_recall.append(per_item)
  673. elif recall_name == config_.PUSH_FROM['flow_recall']:
  674. flow_recall.append(per_item)
  675. if vId not in recall_dict:
  676. recall_dict[vId] = recall_name
  677. else:
  678. recall_name = recall_dict[vId] + "," + recall_name
  679. recall_dict[vId] = recall_name
  680. except:
  681. continue
  682. #print("recall_dict:", recall_dict)
  683. #print("recall time:", (time.time()-dup_time)*1000)
  684. #3. filter video, 先过预曝光
  685. filter_time = time.time()
  686. filter_ = FilterVideos(request_id=request_id,
  687. app_type=app_type, mid=mid, uid=uid, video_ids=list(recall_dict.keys()))
  688. #print("filer:", list(recall_dict.keys()))
  689. #a).expose filter
  690. #all_recall_list = list(recall_dict.keys())
  691. all_recall_list = filter_.filter_videos_new(region_code=region_code, shield_config=shield_config, flow_set=flowFlag_dict.keys())
  692. #print("filer after:", all_recall_list)
  693. #print("filter_ time:", (time.time() - filter_time) * 1000)
  694. #4. sort: old sort: flow 按概率出
  695. start_rank = time.time()
  696. #quick_flow_pool_P get from redis
  697. redis_helper = RedisHelper()
  698. quick_flow_pool_P = redis_helper.get_data_from_redis(
  699. key_name=f"{config_.QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX}{config_.QUICK_FLOW_POOL_ID}"
  700. )
  701. if quick_flow_pool_P:
  702. flow_pool_P = quick_flow_pool_P
  703. rank_result= []
  704. if ab_code==60048 or ab_code==60049:
  705. rank_ids, add_flow_set = video_new_rank(videoIds=all_recall_list,fast_flow_set=fast_flow_set, flow_set=flow_flow_set,size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
  706. #print("rank_ids:", rank_ids)
  707. for rank_item in rank_ids:
  708. rank_id = rank_item[0]
  709. rank_score = rank_item[1]
  710. pushFrom = recall_dict.get(rank_id, '')
  711. #print(pushFrom, rank_id)
  712. flowPoolFlag = ''
  713. if rank_id in add_flow_set:
  714. flowPoolFlag = flowFlag_dict.get(rank_id,'')
  715. rank_result.append({'videoId': rank_id, 'flowPool': flowPoolFlag,
  716. 'rovScore': rank_score, 'pushFrom': pushFrom,
  717. 'abCode': ab_code})
  718. #
  719. #print("rank_result:", rank_result)
  720. else:
  721. all_dup_recall_result = region_h_recall+region_day_recall+select_day_recall+no_selected_recall+flow_recall
  722. rank_result = refactor_video_rank(rov_recall_rank=all_dup_recall_result,fast_flow_set=fast_flow_set, flow_set=flow_flow_set, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
  723. result['rankResult'] = rank_result
  724. result['rankTime'] = (time.time() - start_rank) * 1000
  725. #print("rank time:", result['rankTime'])
  726. return result
  727. # return rank_result, last_rov_recall_key
  728. def ab_test_op(rank_result, ab_code_list, app_type, mid, uid, **kwargs):
  729. """
  730. 对排序后的结果 按照AB实验进行对应的分组操作
  731. :param rank_result: 排序后的结果
  732. :param ab_code_list: 此次请求参与的 ab实验组
  733. :param app_type: 产品标识
  734. :param mid: mid
  735. :param uid: uid
  736. :param kwargs: 其他参数
  737. :return:
  738. """
  739. # ####### 视频宽高比AB实验
  740. # 对内容精选进行 视频宽高比分发实验
  741. # if config_.AB_CODE['w_h_rate'] in ab_code_list and app_type in config_.AB_TEST.get('w_h_rate', []):
  742. # rank_result = video_rank_by_w_h_rate(videos=rank_result)
  743. # log_.info('app_type: {}, mid: {}, uid: {}, rank_by_w_h_rate_result: {}'.format(
  744. # app_type, mid, uid, rank_result))
  745. # 按position位置排序
  746. if config_.AB_CODE['position_insert'] in ab_code_list and app_type in config_.AB_TEST.get('position_insert', []):
  747. rank_result = video_position_recommend(mid, uid, app_type, rank_result)
  748. print('===========================')
  749. print(rank_result)
  750. log_.info('app_type: {}, mid: {}, uid: {}, rank_by_position_insert_result: {}'.format(
  751. app_type, mid, uid, rank_result))
  752. # 相关推荐强插
  753. # if config_.AB_CODE['relevant_video_op'] in ab_code_list \
  754. # and app_type in config_.AB_TEST.get('relevant_video_op', []):
  755. # head_vid = kwargs['head_vid']
  756. # size = kwargs['size']
  757. # rank_result = relevant_video_top_recommend(
  758. # app_type=app_type, mid=mid, uid=uid, head_vid=head_vid, videos=rank_result, size=size
  759. # )
  760. # log_.info('app_type: {}, mid: {}, uid: {}, head_vid: {}, rank_by_relevant_video_op_result: {}'.format(
  761. # app_type, mid, uid, head_vid, rank_result))
  762. return rank_result
  763. def update_redis_data(result, app_type, mid, top_K, expire_time=24*3600):
  764. """
  765. 根据最终的排序结果更新相关redis数据
  766. :param result: 排序结果
  767. :param app_type: 产品标识
  768. :param mid: mid
  769. :param top_K: 保证topK为召回池视频 type-int
  770. :param expire_time: 末位视频记录redis过期时间
  771. :return: None
  772. """
  773. # ####### redis数据刷新
  774. try:
  775. redis_helper = RedisHelper()
  776. # log_.info('====== update redis')
  777. if mid and mid != 'null':
  778. # mid为空时,不做预曝光和定位数据更新
  779. # 预曝光数据同步刷新到Redis, 过期时间为0.5h
  780. preview_key_name = f"{config_.PREVIEW_KEY_PREFIX}{app_type}:{mid}"
  781. preview_video_ids = [int(item['videoId']) for item in result]
  782. if preview_video_ids:
  783. # log_.error('key_name = {} \n values = {}'.format(preview_key_name, tuple(preview_video_ids)))
  784. redis_helper.add_data_with_set(key_name=preview_key_name, values=tuple(preview_video_ids), expire_time=30 * 60)
  785. # log_.info('preview redis update success!')
  786. # # 将此次获取的ROV召回池top_K末位视频id同步刷新到Redis中,方便下次快速定位到召回位置,过期时间为1天
  787. # rov_recall_video = [item['videoId'] for item in result[:top_K]
  788. # if item['pushFrom'] == config_.PUSH_FROM['rov_recall']]
  789. # if len(rov_recall_video) > 0:
  790. # if app_type == config_.APP_TYPE['APP']:
  791. # key_name = config_.UPDATE_ROV_KEY_NAME_APP
  792. # else:
  793. # key_name = config_.UPDATE_ROV_KEY_NAME
  794. # if not redis_helper.get_score_with_value(key_name=key_name, value=rov_recall_video[-1]):
  795. # redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=rov_recall_video[-1],
  796. # expire_time=expire_time)
  797. # log_.info('last video redis update success!')
  798. # 将此次获取的 地域分组小时级数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  799. rov_recall_h_video = [item['videoId'] for item in result[:top_K]
  800. if item['pushFrom'] == config_.PUSH_FROM['rov_recall_region_h']]
  801. if len(rov_recall_h_video) > 0:
  802. last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_H_PREFIX}{app_type}:{mid}'
  803. redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_h_video[-1],
  804. expire_time=expire_time)
  805. # 将此次获取的 地域分组相对24h数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  806. rov_recall_24h_dup1_video = [item['videoId'] for item in result[:top_K]
  807. if item['pushFrom'] == config_.PUSH_FROM['rov_recall_region_24h']]
  808. if len(rov_recall_24h_dup1_video) > 0:
  809. last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP1_24H_PREFIX}{app_type}:{mid}'
  810. redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup1_video[-1],
  811. expire_time=expire_time)
  812. # 将此次获取的 相对24h筛选数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  813. rov_recall_24h_dup2_video = [item['videoId'] for item in result[:top_K]
  814. if item['pushFrom'] == config_.PUSH_FROM['rov_recall_24h']]
  815. if len(rov_recall_24h_dup2_video) > 0:
  816. last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP2_24H_PREFIX}{app_type}:{mid}'
  817. redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup2_video[-1],
  818. expire_time=expire_time)
  819. # 将此次获取的 相对24h筛选后剩余数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  820. rov_recall_24h_dup3_video = [item['videoId'] for item in result[:top_K]
  821. if item['pushFrom'] == config_.PUSH_FROM['rov_recall_24h_dup']]
  822. if len(rov_recall_24h_dup3_video) > 0:
  823. last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP3_24H_PREFIX}{app_type}:{mid}'
  824. redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_24h_dup3_video[-1],
  825. expire_time=expire_time)
  826. # # 将此次获取的 相对48h筛选数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  827. # rov_recall_48h_dup2_video = [item['videoId'] for item in result[:top_K]
  828. # if item['pushFrom'] == config_.PUSH_FROM['rov_recall_48h']]
  829. # if len(rov_recall_48h_dup2_video) > 0:
  830. # last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP2_48H_PREFIX}{app_type}:{mid}'
  831. # redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_48h_dup2_video[-1],
  832. # expire_time=expire_time)
  833. #
  834. # # 将此次获取的 相对48h筛选后剩余数据列表 中的视频id同步刷新到redis中,方便下次快速定位到召回位置
  835. # rov_recall_48h_dup3_video = [item['videoId'] for item in result[:top_K]
  836. # if item['pushFrom'] == config_.PUSH_FROM['rov_recall_48h_dup']]
  837. # if len(rov_recall_48h_dup3_video) > 0:
  838. # last_video_key = f'{config_.LAST_VIDEO_FROM_REGION_DUP3_48H_PREFIX}{app_type}:{mid}'
  839. # redis_helper.set_data_to_redis(key_name=last_video_key, value=rov_recall_48h_dup3_video[-1],
  840. # expire_time=expire_time)
  841. # 将此次分发的流量池视频,对 本地分发数-1 进行记录
  842. if app_type not in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  843. # 获取本地分发数-1策略开关
  844. switch = redis_helper.get_data_from_redis(key_name=config_.IN_FLOW_POOL_COUNT_SWITCH_KEY_NAME)
  845. if switch is not None:
  846. if int(switch) == 1:
  847. flow_recall_video = [item for item in result if item.get('flowPool', None) is not None]
  848. else:
  849. flow_recall_video = [item for item in result if
  850. item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
  851. else:
  852. flow_recall_video = [item for item in result if item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
  853. if flow_recall_video:
  854. update_local_distribute_count(flow_recall_video)
  855. # log_.info('update local distribute count success!')
  856. # 限流视频分发数记录
  857. if app_type == config_.APP_TYPE['APP']:
  858. # APP 不计入
  859. return
  860. limit_video_id_list = redis_helper.get_data_from_set(
  861. key_name=f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_SET}{datetime.today().strftime('%Y%m%d')}"
  862. )
  863. if limit_video_id_list is not None:
  864. limit_video_id_list = [int(item) for item in limit_video_id_list]
  865. for item in result:
  866. video_id = item['videoId']
  867. if video_id in limit_video_id_list:
  868. key_name = f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_DISTRIBUTE_COUNT}{video_id}"
  869. redis_helper.setnx_key(key_name=key_name, value=0, expire_time=24*2600)
  870. redis_helper.incr_key(key_name=key_name, amount=1, expire_time=24*3600)
  871. except Exception as e:
  872. log_.error("update redis data fail!")
  873. log_.error(traceback.format_exc())
  874. def update_flow_redis_data(result, app_type, mid, top_K, expire_time=24*3600):
  875. """
  876. 根据最终的排序结果更新相关redis数据
  877. :param result: 排序结果
  878. :param app_type: 产品标识
  879. :param mid: mid
  880. :param top_K: 保证topK为召回池视频 type-int
  881. :param expire_time: 末位视频记录redis过期时间
  882. :return: None
  883. """
  884. # ####### redis数据刷新
  885. try:
  886. redis_helper = RedisHelper()
  887. # log_.info('====== update redis')
  888. if mid and mid != 'null':
  889. # mid为空时,不做预曝光和定位数据更新
  890. # 预曝光数据同步刷新到Redis, 过期时间为0.5h
  891. preview_key_name = f"{config_.PREVIEW_KEY_PREFIX}{app_type}:{mid}"
  892. preview_video_ids = [int(item['videoId']) for item in result]
  893. if preview_video_ids:
  894. # log_.error('key_name = {} \n values = {}'.format(preview_key_name, tuple(preview_video_ids)))
  895. redis_helper.add_data_with_set(key_name=preview_key_name, values=tuple(preview_video_ids), expire_time=30 * 60)
  896. # log_.info('preview redis update success!')
  897. # 将此次分发的流量池视频,对 本地分发数-1 进行记录
  898. if app_type not in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
  899. # 获取本地分发数-1策略开关
  900. switch = redis_helper.get_data_from_redis(key_name=config_.IN_FLOW_POOL_COUNT_SWITCH_KEY_NAME)
  901. if switch is not None:
  902. if int(switch) == 1:
  903. flow_recall_video = [item for item in result if item.get('flowPool', None) is not None]
  904. else:
  905. flow_recall_video = [item for item in result if
  906. item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
  907. else:
  908. flow_recall_video = [item for item in result if item['pushFrom'] == config_.PUSH_FROM['flow_recall']]
  909. if flow_recall_video:
  910. update_local_distribute_count(flow_recall_video)
  911. # log_.info('update local distribute count success!')
  912. # 限流视频分发数记录
  913. if app_type == config_.APP_TYPE['APP']:
  914. # APP 不计入
  915. return
  916. limit_video_id_list = redis_helper.get_data_from_set(
  917. key_name=f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_SET}{datetime.today().strftime('%Y%m%d')}"
  918. )
  919. if limit_video_id_list is not None:
  920. limit_video_id_list = [int(item) for item in limit_video_id_list]
  921. for item in result:
  922. video_id = item['videoId']
  923. if video_id in limit_video_id_list:
  924. key_name = f"{config_.KEY_NAME_PREFIX_LIMIT_VIDEO_DISTRIBUTE_COUNT}{video_id}"
  925. redis_helper.setnx_key(key_name=key_name, value=0, expire_time=24*2600)
  926. redis_helper.incr_key(key_name=key_name, amount=1, expire_time=24*3600)
  927. except Exception as e:
  928. log_.error("update redis data fail!")
  929. log_.error(traceback.format_exc())
  930. def update_local_distribute_count(videos):
  931. """
  932. 更新本地分发数
  933. :param videos: 视频列表 type-list [{'videoId':'', 'flowPool':'', 'distributeCount': '',
  934. 'rovScore': '', 'pushFrom': 'flow_pool', 'abCode': self.ab_code}, ....]
  935. :return:
  936. """
  937. try:
  938. redis_helper = RedisHelper()
  939. for item in videos:
  940. video_id, flow_pool = item['videoId'], item['flowPool']
  941. key_name = f"{config_.LOCAL_DISTRIBUTE_COUNT_PREFIX}{video_id}:{flow_pool}"
  942. # 本地记录的分发数 - 1
  943. redis_helper.decr_key(key_name=key_name, amount=1, expire_time=15 * 60)
  944. # 对该视频做分发数检查
  945. cur_count = redis_helper.get_data_from_redis(key_name=key_name)
  946. # 无记录
  947. if cur_count is None:
  948. continue
  949. # 本地分发数 cur_count <= 0,从所有的流量召回池移除,删除本地分发记录key
  950. if int(cur_count) <= 0:
  951. add_remove_log = False
  952. redis_helper.del_keys(key_name=key_name)
  953. for app_name in config_.APP_TYPE:
  954. app_type = config_.APP_TYPE.get(app_name)
  955. flow_pool_key_list = [
  956. f"{config_.FLOWPOOL_KEY_NAME_PREFIX}{app_type}",
  957. f"{config_.QUICK_FLOWPOOL_KEY_NAME_PREFIX}{app_type}:{config_.QUICK_FLOW_POOL_ID}"
  958. ]
  959. for key in flow_pool_key_list:
  960. remove_res = redis_helper.remove_value_from_zset(key_name=key, value=f"{video_id}-{flow_pool}")
  961. if remove_res > 0:
  962. add_remove_log = True
  963. video_flow_pool_key_list = [
  964. f"{config_.QUICK_FLOWPOOL_VIDEO_INFO_KEY_NAME_PREFIX}{app_type}:{config_.QUICK_FLOW_POOL_ID}:{video_id}",
  965. f"{config_.FLOWPOOL_VIDEO_INFO_KEY_NAME_PREFIX}{app_type}:{video_id}"
  966. ]
  967. for key in video_flow_pool_key_list:
  968. redis_helper.remove_value_from_set(key_name=key, values=(flow_pool, ))
  969. if add_remove_log is True:
  970. log_.info({'tag': 'remove video_id from flow_pool', 'video_id': video_id, 'flow_pool': flow_pool})
  971. # if redis_helper.key_exists(key_name=key_name):
  972. # # 该视频本地有记录,本地记录的分发数 - 1
  973. # redis_helper.decr_key(key_name=key_name, amount=1, expire_time=5 * 60)
  974. # else:
  975. # # 该视频本地无记录,接口获取的分发数 - 1
  976. # redis_helper.incr_key(key_name=key_name, amount=int(item['distributeCount']) - 1, expire_time=5 * 60)
  977. except Exception as e:
  978. log_.error('update_local_distribute_count error...')
  979. log_.error(traceback.format_exc())
  980. def get_religion_class_with_mid(mid, religion_class_name):
  981. """
  982. 判断用户是否属于对应的宗教类型
  983. :param mid: mid type-string
  984. :param religion_class_name: 宗教类型, type-string, (catholicism-天主教, christianity-基督教)
  985. :return: religion_class_flag, type-int, (0-否,1-是), 默认: 0
  986. """
  987. religion_class_flag = 0
  988. now_date = datetime.today()
  989. redis_helper = RedisHelper()
  990. if mid:
  991. hash_mid = hashlib.md5(mid.encode('utf-8')).hexdigest()
  992. hash_tag = hash_mid[-1:]
  993. key_name_prefix = config_.KEY_NAME_PREFIX_RELIGION_USER.get(religion_class_name, None)
  994. if key_name_prefix is None:
  995. return religion_class_flag
  996. key_name = f"{key_name_prefix}{hash_tag}:{datetime.strftime(now_date, '%Y%m%d')}"
  997. if not redis_helper.key_exists(key_name=key_name):
  998. key_name = f"{key_name_prefix}{hash_tag}:{datetime.strftime(now_date - timedelta(days=1), '%Y%m%d')}"
  999. if redis_helper.data_exists_with_set(key_name=key_name, value=mid):
  1000. religion_class_flag = 1
  1001. return religion_class_flag
  1002. def get_recommend_params(recommend_type, ab_exp_info, ab_info_data, mid, app_type, page_type=0):
  1003. """
  1004. 根据实验分组给定对应的推荐参数
  1005. :param recommend_type: 首页推荐和相关推荐区分参数(0-首页推荐,1-相关推荐)
  1006. :param ab_exp_info: AB实验组参数
  1007. :param ab_info_data: app实验组参数
  1008. :param mid: mid
  1009. :param app_type: app_type, type-int
  1010. :param page_type: 页面区分参数,默认:0(首页)
  1011. :return:
  1012. """
  1013. top_K = config_.K
  1014. flow_pool_P = config_.P
  1015. # 不获取人工干预数据标记
  1016. no_op_flag = True
  1017. expire_time = 3600
  1018. old_video_index = -1
  1019. # 获取对应的默认配置
  1020. ab_initial_config = config_.INITIAL_CONFIG.get(app_type, None)
  1021. if ab_initial_config is None:
  1022. ab_initial_config = config_.INITIAL_CONFIG.get('other')
  1023. param = config_.AB_EXP_CODE[ab_initial_config]
  1024. ab_code = param.get('ab_code')
  1025. rule_key = param.get('rule_key')
  1026. data_key = param.get('data_key')
  1027. rule_key_30day = param.get('30day_rule_key')
  1028. shield_config = config_.SHIELD_CONFIG
  1029. # 默认使用 095 实验的配置
  1030. # ab_code = config_.AB_EXP_CODE['095'].get('ab_code')
  1031. # rule_key = config_.AB_EXP_CODE['095'].get('rule_key')
  1032. # data_key = config_.AB_EXP_CODE['095'].get('data_key')
  1033. # rule_key_30day = None
  1034. # 获取用户近30天是否有回流
  1035. # user_30day_return_result = get_user_has30day_return(mid=mid)
  1036. # 获取实验配置
  1037. if ab_exp_info:
  1038. ab_exp_code_list = []
  1039. config_value_dict = {}
  1040. for _, item in ab_exp_info.items():
  1041. if not item:
  1042. continue
  1043. for ab_item in item:
  1044. ab_exp_code = ab_item.get('abExpCode', None)
  1045. if not ab_exp_code:
  1046. continue
  1047. ab_exp_code_list.append(str(ab_exp_code))
  1048. config_value_dict[str(ab_exp_code)] = ab_item.get('configValue', None)
  1049. # 流量池视频分发概率实验
  1050. if '211' in ab_exp_code_list:
  1051. flow_pool_P = 0.9
  1052. elif '221' in ab_exp_code_list:
  1053. flow_pool_P = 0.7
  1054. elif '299' in ab_exp_code_list:
  1055. flow_pool_P = 0.5
  1056. elif '300' in ab_exp_code_list:
  1057. flow_pool_P = 0.4
  1058. elif '301' in ab_exp_code_list:
  1059. flow_pool_P = 0.6
  1060. elif '339' in ab_exp_code_list:
  1061. flow_pool_P = 0
  1062. # if '136' in ab_exp_code_list:
  1063. # # 无回流 - 消费人群
  1064. # if user_30day_return_result == 0:
  1065. # param = config_.AB_EXP_CODE.get('136')
  1066. # ab_code = param.get('ab_code')
  1067. # expire_time = 3600
  1068. # rule_key = param.get('rule_key')
  1069. # data_key = param.get('data_key')
  1070. # no_op_flag = True
  1071. # elif '137' in ab_exp_code_list:
  1072. # # 有回流 - 分享人群
  1073. # if user_30day_return_result == 1:
  1074. # param = config_.AB_EXP_CODE.get('137')
  1075. # ab_code = param.get('ab_code')
  1076. # expire_time = 3600
  1077. # rule_key = param.get('rule_key')
  1078. # data_key = param.get('data_key')
  1079. # no_op_flag = True
  1080. # elif '161' in ab_exp_code_list:
  1081. # # 无回流 - 消费人群
  1082. # if user_30day_return_result == 0:
  1083. # param = config_.AB_EXP_CODE.get('136')
  1084. # ab_code = param.get('ab_code')
  1085. # expire_time = 3600
  1086. # rule_key = param.get('rule_key')
  1087. # data_key = param.get('data_key')
  1088. # no_op_flag = True
  1089. # # 有回流 - 分享人群
  1090. # else:
  1091. # param = config_.AB_EXP_CODE.get('137')
  1092. # ab_code = param.get('ab_code')
  1093. # expire_time = 3600
  1094. # rule_key = param.get('rule_key')
  1095. # data_key = param.get('data_key')
  1096. # no_op_flag = True
  1097. # elif '162' in ab_exp_code_list:
  1098. # # 有回流
  1099. # if user_30day_return_result == 1:
  1100. # param = config_.AB_EXP_CODE.get('162')
  1101. # ab_code = param.get('ab_code')
  1102. # expire_time = 3600
  1103. # rule_key = param.get('rule_key')
  1104. # data_key = param.get('data_key')
  1105. # no_op_flag = True
  1106. # 老好看视频 宗教人群实验
  1107. if '228' in ab_exp_code_list:
  1108. # 天主教
  1109. religion_param = config_.AB_EXP_CODE['228']
  1110. religion_class_name = religion_param.get('religion_class_name')
  1111. religion_class_flag = get_religion_class_with_mid(mid=mid, religion_class_name=religion_class_name)
  1112. if religion_class_flag == 1:
  1113. ab_code = religion_param.get('ab_code')
  1114. rule_key = religion_param.get('rule_key')
  1115. data_key = religion_param.get('data_key')
  1116. rule_key_30day = religion_param.get('30day_rule_key')
  1117. elif '229' in ab_exp_code_list:
  1118. # 基督教
  1119. religion_param = config_.AB_EXP_CODE['229']
  1120. religion_class_name = religion_param.get('religion_class_name')
  1121. religion_class_flag = get_religion_class_with_mid(mid=mid, religion_class_name=religion_class_name)
  1122. if religion_class_flag == 1:
  1123. ab_code = religion_param.get('ab_code')
  1124. rule_key = religion_param.get('rule_key')
  1125. data_key = religion_param.get('data_key')
  1126. rule_key_30day = religion_param.get('30day_rule_key')
  1127. else:
  1128. for code, param in config_.AB_EXP_CODE.items():
  1129. if code in ab_exp_code_list:
  1130. ab_code = param.get('ab_code')
  1131. rule_key = param.get('rule_key')
  1132. data_key = param.get('data_key')
  1133. rule_key_30day = param.get('30day_rule_key')
  1134. shield_config = param.get('shield_config', config_.SHIELD_CONFIG)
  1135. break
  1136. """
  1137. # 推荐条数 10->4 实验
  1138. # if config_.AB_EXP_CODE['rec_size_home'] in ab_exp_code_list:
  1139. # config_value = config_value_dict.get(config_.AB_EXP_CODE['rec_size_home'], None)
  1140. # if config_value:
  1141. # config_value = eval(str(config_value))
  1142. # else:
  1143. # config_value = {}
  1144. # log_.info(f'config_value: {config_value}, type: {type(config_value)}')
  1145. # size = int(config_value.get('size', 4))
  1146. # top_K = int(config_value.get('K', 3))
  1147. # flow_pool_P = float(config_value.get('P', 0.3))
  1148. # else:
  1149. # size = size
  1150. # top_K = config_.K
  1151. # flow_pool_P = config_.P
  1152. # 算法实验相对对照组
  1153. # if config_.AB_EXP_CODE['ab_initial'] in ab_exp_code_list:
  1154. # ab_code = config_.AB_CODE['ab_initial']
  1155. # expire_time = 24 * 3600
  1156. # rule_key = config_.RULE_KEY['initial']
  1157. # no_op_flag = True
  1158. # 小时级更新-规则1 实验
  1159. # elif config_.AB_EXP_CODE['rule_rank1'] in ab_exp_code_list:
  1160. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank1')
  1161. # expire_time = 3600
  1162. # rule_key = config_.RULE_KEY['rule_rank1']
  1163. # no_op_flag = True
  1164. # elif config_.AB_EXP_CODE['rule_rank2'] in ab_exp_code_list:
  1165. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank2')
  1166. # expire_time = 3600
  1167. # rule_key = config_.RULE_KEY['rule_rank2']
  1168. # elif config_.AB_EXP_CODE['rule_rank3'] in ab_exp_code_list:
  1169. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank3')
  1170. # expire_time = 3600
  1171. # rule_key = config_.RULE_KEY['rule_rank3']
  1172. # no_op_flag = True
  1173. # elif config_.AB_EXP_CODE['rule_rank4'] in ab_exp_code_list:
  1174. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank4')
  1175. # expire_time = 3600
  1176. # rule_key = config_.RULE_KEY['rule_rank4']
  1177. # elif config_.AB_EXP_CODE['rule_rank5'] in ab_exp_code_list:
  1178. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank5')
  1179. # expire_time = 3600
  1180. # rule_key = config_.RULE_KEY['rule_rank5']
  1181. # elif config_.AB_EXP_CODE['day_rule_rank1'] in ab_exp_code_list:
  1182. # ab_code = config_.AB_CODE['rank_by_day'].get('day_rule_rank1')
  1183. # expire_time = 24 * 3600
  1184. # rule_key = config_.RULE_KEY_DAY['day_rule_rank1']
  1185. # no_op_flag = True
  1186. # if config_.AB_EXP_CODE['rule_rank6'] in ab_exp_code_list:
  1187. # ab_code = config_.AB_CODE['rank_by_h'].get('rule_rank6')
  1188. # expire_time = 3600
  1189. # rule_key = config_.RULE_KEY['rule_rank6']
  1190. # no_op_flag = True
  1191. # elif config_.AB_EXP_CODE['day_rule_rank2'] in ab_exp_code_list:
  1192. # ab_code = config_.AB_CODE['rank_by_day'].get('day_rule_rank2')
  1193. # expire_time = 24 * 3600
  1194. # rule_key = config_.RULE_KEY_DAY['day_rule_rank2']
  1195. # no_op_flag = True
  1196. # elif config_.AB_EXP_CODE['region_rule_rank1'] in ab_exp_code_list:
  1197. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank1')
  1198. # expire_time = 3600
  1199. # rule_key = config_.RULE_KEY_REGION['region_rule_rank1']
  1200. # no_op_flag = True
  1201. # elif config_.AB_EXP_CODE['24h_rule_rank1'] in ab_exp_code_list:
  1202. # ab_code = config_.AB_CODE['rank_by_24h'].get('24h_rule_rank1')
  1203. # expire_time = 3600
  1204. # rule_key = config_.RULE_KEY_24H['24h_rule_rank1']
  1205. # no_op_flag = True
  1206. # elif config_.AB_EXP_CODE['24h_rule_rank2'] in ab_exp_code_list:
  1207. # ab_code = config_.AB_CODE['rank_by_24h'].get('24h_rule_rank2')
  1208. # expire_time = 3600
  1209. # rule_key = config_.RULE_KEY_24H['24h_rule_rank2']
  1210. # no_op_flag = True
  1211. # elif config_.AB_EXP_CODE['region_rule_rank2'] in ab_exp_code_list:
  1212. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank2')
  1213. # expire_time = 3600
  1214. # rule_key = config_.RULE_KEY_REGION['region_rule_rank2']
  1215. # no_op_flag = True
  1216. # if config_.AB_EXP_CODE['region_rule_rank3'] in ab_exp_code_list or\
  1217. # config_.AB_EXP_CODE['region_rule_rank3_appType_19'] in ab_exp_code_list or\
  1218. # config_.AB_EXP_CODE['region_rule_rank3_appType_4'] in ab_exp_code_list or\
  1219. # config_.AB_EXP_CODE['region_rule_rank3_appType_6'] in ab_exp_code_list or\
  1220. # config_.AB_EXP_CODE['region_rule_rank3_appType_18'] in ab_exp_code_list:
  1221. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank3')
  1222. # expire_time = 3600
  1223. # rule_key = config_.RULE_KEY_REGION['region_rule_rank3'].get('rule_key')
  1224. # data_key = config_.RULE_KEY_REGION['region_rule_rank3'].get('data_key')
  1225. # no_op_flag = True
  1226. # if config_.AB_EXP_CODE['region_rule_rank4'] in ab_exp_code_list or\
  1227. if config_.AB_EXP_CODE['region_rule_rank4_appType_19'] in ab_exp_code_list or \
  1228. config_.AB_EXP_CODE['region_rule_rank4_appType_4'] in ab_exp_code_list or\
  1229. config_.AB_EXP_CODE['region_rule_rank4_appType_6'] in ab_exp_code_list or\
  1230. config_.AB_EXP_CODE['region_rule_rank4_appType_18'] in ab_exp_code_list:
  1231. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
  1232. expire_time = 3600
  1233. rule_key = config_.RULE_KEY_REGION['region_rule_rank4'].get('rule_key')
  1234. data_key = config_.RULE_KEY_REGION['region_rule_rank4'].get('data_key')
  1235. no_op_flag = True
  1236. # elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data1'] in ab_exp_code_list:
  1237. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
  1238. # expire_time = 3600
  1239. # rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data1'].get('rule_key')
  1240. # data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data1'].get('data_key')
  1241. # no_op_flag = True
  1242. # elif config_.AB_EXP_CODE['region_rule_rank3_appType_5_data2'] in ab_exp_code_list:
  1243. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank3_appType_5_data2')
  1244. # expire_time = 3600
  1245. # rule_key = config_.RULE_KEY_REGION['region_rule_rank3_appType_5_data2'].get('rule_key')
  1246. # data_key = config_.RULE_KEY_REGION['region_rule_rank3_appType_5_data2'].get('data_key')
  1247. # no_op_flag = True
  1248. elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data3'] in ab_exp_code_list:
  1249. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_5_data3')
  1250. expire_time = 3600
  1251. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data3'].get('rule_key')
  1252. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data3'].get('data_key')
  1253. no_op_flag = True
  1254. elif config_.AB_EXP_CODE['region_rule_rank4_appType_5_data4'] in ab_exp_code_list:
  1255. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_5_data4')
  1256. expire_time = 3600
  1257. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data4'].get('rule_key')
  1258. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_5_data4'].get('data_key')
  1259. no_op_flag = True
  1260. elif config_.AB_EXP_CODE['region_rule_rank4_appType_0_data2'] in ab_exp_code_list:
  1261. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_0_data2')
  1262. expire_time = 3600
  1263. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_0_data2'].get('rule_key')
  1264. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_0_data2'].get('data_key')
  1265. no_op_flag = True
  1266. # elif config_.AB_EXP_CODE['region_rule_rank4_appType_19_data2'] in ab_exp_code_list:
  1267. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_19_data2')
  1268. # expire_time = 3600
  1269. # rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data2'].get('rule_key')
  1270. # data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data2'].get('data_key')
  1271. # no_op_flag = True
  1272. # elif config_.AB_EXP_CODE['region_rule_rank4_appType_19_data3'] in ab_exp_code_list:
  1273. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_19_data3')
  1274. # expire_time = 3600
  1275. # rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data3'].get('rule_key')
  1276. # data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_19_data3'].get('data_key')
  1277. # no_op_flag = True
  1278. elif config_.AB_EXP_CODE['region_rule_rank5_appType_0_data1'] in ab_exp_code_list:
  1279. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank5_appType_0_data1')
  1280. expire_time = 3600
  1281. rule_key = config_.RULE_KEY_REGION['region_rule_rank5_appType_0_data1'].get('rule_key')
  1282. data_key = config_.RULE_KEY_REGION['region_rule_rank5_appType_0_data1'].get('data_key')
  1283. no_op_flag = True
  1284. elif config_.AB_EXP_CODE['region_rule_rank4_appType_4_data2'] in ab_exp_code_list:
  1285. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_4_data2')
  1286. expire_time = 3600
  1287. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data2'].get('rule_key')
  1288. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data2'].get('data_key')
  1289. no_op_flag = True
  1290. elif config_.AB_EXP_CODE['region_rule_rank4_appType_4_data3'] in ab_exp_code_list:
  1291. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_4_data3')
  1292. expire_time = 3600
  1293. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data3'].get('rule_key')
  1294. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_4_data3'].get('data_key')
  1295. no_op_flag = True
  1296. elif config_.AB_EXP_CODE['region_rule_rank4_appType_6_data2'] in ab_exp_code_list:
  1297. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_6_data2')
  1298. expire_time = 3600
  1299. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data2'].get('rule_key')
  1300. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data2'].get('data_key')
  1301. no_op_flag = True
  1302. elif config_.AB_EXP_CODE['region_rule_rank4_appType_6_data3'] in ab_exp_code_list:
  1303. ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_6_data3')
  1304. expire_time = 3600
  1305. rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data3'].get('rule_key')
  1306. data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_6_data3'].get('data_key')
  1307. no_op_flag = True
  1308. # elif config_.AB_EXP_CODE['region_rule_rank4_appType_18_data2'] in ab_exp_code_list:
  1309. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4_appType_18_data2')
  1310. # expire_time = 3600
  1311. # rule_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_18_data2'].get('rule_key')
  1312. # data_key = config_.RULE_KEY_REGION['region_rule_rank4_appType_18_data2'].get('data_key')
  1313. # no_op_flag = True
  1314. # elif config_.AB_EXP_CODE['region_rule_rank6_appType_0_data1'] in ab_exp_code_list:
  1315. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank6_appType_0_data1')
  1316. # expire_time = 3600
  1317. # rule_key = config_.RULE_KEY_REGION['region_rule_rank6_appType_0_data1'].get('rule_key')
  1318. # data_key = config_.RULE_KEY_REGION['region_rule_rank6_appType_0_data1'].get('data_key')
  1319. # no_op_flag = True
  1320. else:
  1321. ab_code = config_.AB_CODE['initial']
  1322. expire_time = 24 * 3600
  1323. rule_key = config_.RULE_KEY_REGION['initial'].get('rule_key')
  1324. data_key = config_.RULE_KEY_REGION['initial'].get('data_key')
  1325. # # 老好看视频 / 票圈最惊奇 首页/相关推荐逻辑更新实验
  1326. # if config_.AB_EXP_CODE['rov_rank_appType_18_19'] in ab_exp_code_list:
  1327. # ab_code = config_.AB_CODE['rov_rank_appType_18_19']
  1328. # expire_time = 3600
  1329. # flow_pool_P = config_.P_18_19
  1330. # no_op_flag = True
  1331. #
  1332. # elif config_.AB_EXP_CODE['rov_rank_appType_19'] in ab_exp_code_list:
  1333. # ab_code = config_.AB_CODE['rov_rank_appType_19']
  1334. # expire_time = 3600
  1335. # top_K = 0
  1336. # flow_pool_P = config_.P_18_19
  1337. # no_op_flag = True
  1338. #
  1339. # elif config_.AB_EXP_CODE['top_video_relevant_appType_19'] in ab_exp_code_list and page_type == 2:
  1340. # ab_code = config_.AB_CODE['top_video_relevant_appType_19']
  1341. # expire_time = 3600
  1342. # top_K = 1
  1343. # flow_pool_P = config_.P_18_19
  1344. # no_op_flag = True
  1345. #
  1346. # # 票圈最惊奇完整影视资源实验
  1347. # elif config_.AB_EXP_CODE['whole_movies'] in ab_exp_code_list:
  1348. # ab_code = config_.AB_CODE['whole_movies']
  1349. # expire_time = 24 * 3600
  1350. # no_op_flag = True
  1351. # 老视频实验
  1352. # if config_.AB_EXP_CODE['old_video'] in ab_exp_code_list:
  1353. # ab_code = config_.AB_CODE['old_video']
  1354. # no_op_flag = True
  1355. # old_video_index = 2
  1356. # else:
  1357. # old_video_index = -1
  1358. """
  1359. # APP实验组
  1360. if ab_info_data:
  1361. ab_info_app = {}
  1362. for page_code, item in json.loads(ab_info_data).items():
  1363. if not item:
  1364. continue
  1365. ab_info_code = item.get('eventId', None)
  1366. if ab_info_code:
  1367. ab_info_app[page_code] = ab_info_code
  1368. # print(f"======{ab_info_app}")
  1369. # 首页推荐
  1370. if recommend_type == 0:
  1371. app_ab_code = ab_info_app.get('10003', None)
  1372. for code, param in config_.APP_AB_CODE['10003'].items():
  1373. if code == app_ab_code:
  1374. ab_code = param.get('ab_code')
  1375. rule_key = param.get('rule_key')
  1376. data_key = param.get('data_key')
  1377. break
  1378. # # 相关推荐
  1379. # elif recommend_type == 1:
  1380. # if config_.APP_AB_CODE['10037'] == ab_info_app.get('10037', None):
  1381. # ab_code = config_.AB_CODE['region_rank_by_h'].get('region_rule_rank4')
  1382. # expire_time = 3600
  1383. # rule_key = 'rule3'
  1384. # data_key = 'data1'
  1385. # no_op_flag = True
  1386. return top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, no_op_flag, old_video_index, rule_key_30day, shield_config
  1387. def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
  1388. client_info, ab_exp_info, params, ab_info_data, version_audit_status):
  1389. """
  1390. 首页线上推荐逻辑
  1391. :param request_id: request_id
  1392. :param mid: mid type-string
  1393. :param uid: uid type-string
  1394. :param size: 请求视频数量 type-int
  1395. :param app_type: 产品标识 type-int
  1396. :param algo_type: 算法类型 type-string
  1397. :param client_info: 用户位置信息 {"country": "国家", "province": "省份", "city": "城市"}
  1398. :param ab_exp_info: ab实验分组参数 [{"expItemId":1, "configValue":{"size":4, "K":3, ...}}, ...]
  1399. :param params:
  1400. :param ab_info_data: app实验分组参数
  1401. :param version_audit_status: 小程序版本审核参数:1-审核中,2-审核通过
  1402. :return:
  1403. """
  1404. # 对 vlog 切换10%的流量做实验
  1405. # 对mid进行哈希
  1406. # hash_mid = hashlib.md5(mid.encode('utf-8')).hexdigest()
  1407. # if app_type in config_.AB_TEST['rank_by_h'] and hash_mid[-1:] in ['8', '0', 'a', 'b']:
  1408. # # 简单召回 - 排序 - 兜底
  1409. # rank_result, last_rov_recall_key = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
  1410. # algo_type=algo_type, client_info=client_info,
  1411. # expire_time=3600,
  1412. # ab_code=config_.AB_CODE['rank_by_h'])
  1413. # # ab-test
  1414. # result = ab_test_op(rank_result=rank_result,
  1415. # ab_code_list=[config_.AB_CODE['position_insert']],
  1416. # app_type=app_type, mid=mid, uid=uid)
  1417. # # redis数据刷新
  1418. # update_redis_data(result=result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
  1419. # expire_time=3600)
  1420. # if app_type == config_.APP_TYPE['APP']:
  1421. # # 票圈视频APP
  1422. # top_K = config_.K
  1423. # flow_pool_P = config_.P
  1424. # # 简单召回 - 排序 - 兜底
  1425. # rank_result, last_rov_recall_key = video_recommend(request_id=request_id,
  1426. # mid=mid, uid=uid, app_type=app_type,
  1427. # size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1428. # algo_type=algo_type, client_info=client_info,
  1429. # expire_time=12 * 3600, params=params)
  1430. # # ab-test
  1431. # # result = ab_test_op(rank_result=rank_result,
  1432. # # ab_code_list=[config_.AB_CODE['position_insert']],
  1433. # # app_type=app_type, mid=mid, uid=uid)
  1434. # # redis数据刷新
  1435. # update_redis_data(result=rank_result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
  1436. # top_K=top_K, expire_time=12 * 3600)
  1437. #
  1438. # else:
  1439. recommend_result = {}
  1440. param_st = time.time()
  1441. # 特殊mid 和 小程序审核版本推荐处理
  1442. if mid in get_special_mid_list() or version_audit_status == 1:
  1443. rank_result = special_mid_recommend(request_id=request_id, mid=mid, uid=uid, app_type=app_type, size=size)
  1444. recommend_result['videos'] = rank_result
  1445. return recommend_result
  1446. # 普通mid推荐处理
  1447. top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, \
  1448. no_op_flag, old_video_index, rule_key_30day, shield_config = \
  1449. get_recommend_params(recommend_type=0, ab_exp_info=ab_exp_info, ab_info_data=ab_info_data, mid=mid,
  1450. app_type=app_type)
  1451. # log_.info({
  1452. # 'logTimestamp': int(time.time() * 1000),
  1453. # 'request_id': request_id,
  1454. # 'app_type': app_type,
  1455. # 'mid': mid,
  1456. # 'uid': uid,
  1457. # 'operation': 'get_recommend_params',
  1458. # 'executeTime': (time.time() - param_st) * 1000
  1459. # })
  1460. recommend_result['getRecommendParamsTime'] = (time.time() - param_st) * 1000
  1461. # 简单召回 - 排序 - 兜底
  1462. get_result_st = time.time()
  1463. #print("ab_code:", ab_code)
  1464. #new pipeline
  1465. if ab_code == 60047 or ab_code == 60048 or ab_code == 60049:
  1466. result = new_video_recommend(request_id=request_id,
  1467. mid=mid, uid=uid, app_type=app_type,
  1468. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1469. algo_type=algo_type, client_info=client_info,
  1470. ab_code=ab_code, expire_time=expire_time,
  1471. rule_key=rule_key, data_key=data_key,
  1472. no_op_flag=no_op_flag, old_video_index=old_video_index,
  1473. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1474. # old base (test4, test5)
  1475. # elif ab_code == 60050 or ab_code == 60051:
  1476. # result = video_recommend(request_id=request_id,
  1477. # mid=mid, uid=uid, app_type=app_type,
  1478. # size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1479. # algo_type=algo_type, client_info=client_info,
  1480. # ab_code=ab_code, expire_time=expire_time,
  1481. # rule_key=rule_key, data_key=data_key,
  1482. # no_op_flag=no_op_flag, old_video_index=old_video_index,
  1483. # params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1484. # 60052,60053,60057(test7, test8, test6)
  1485. # simrecal: 60054
  1486. # 3 days , 7days recall: 60005, test1
  1487. # old video: 60056, test2
  1488. elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or ab_code == 60056 or ab_code==60057:
  1489. result = video_old_recommend(request_id=request_id,
  1490. mid=mid, uid=uid, app_type=app_type,
  1491. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1492. algo_type='', client_info=client_info,
  1493. ab_code=ab_code, expire_time=expire_time,
  1494. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1495. old_video_index=old_video_index, video_id= None,
  1496. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1497. else:
  1498. result = video_recommend(request_id=request_id,
  1499. mid=mid, uid=uid, app_type=app_type,
  1500. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1501. algo_type=algo_type, client_info=client_info,
  1502. ab_code=ab_code, expire_time=expire_time,
  1503. rule_key=rule_key, data_key=data_key,
  1504. no_op_flag=no_op_flag, old_video_index=old_video_index,
  1505. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1506. # log_.info({
  1507. # 'logTimestamp': int(time.time() * 1000),
  1508. # 'request_id': request_id,
  1509. # 'app_type': app_type,
  1510. # 'mid': mid,
  1511. # 'uid': uid,
  1512. # 'operation': 'get_recommend_result',
  1513. # 'executeTime': (time.time() - get_result_st) * 1000
  1514. # })
  1515. recommend_result['recommendOperation'] = result
  1516. rank_result = result.get('rankResult')
  1517. for i, item in enumerate(rank_result):
  1518. item['position'] = i + 1
  1519. recommend_result['videos'] = rank_result
  1520. recommend_result['getRecommendResultTime'] = (time.time() - get_result_st) * 1000
  1521. # ab-test
  1522. # result = ab_test_op(rank_result=rank_result,
  1523. # ab_code_list=[config_.AB_CODE['position_insert']],
  1524. # app_type=app_type, mid=mid, uid=uid)
  1525. # redis数据刷新
  1526. update_redis_st = time.time()
  1527. if ab_code == 60047 or ab_code == 60048 or ab_code == 60049:
  1528. update_flow_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1529. # elif ab_code == 60050 or ab_code == 60051:
  1530. # update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1531. elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 \
  1532. or ab_code == 60056 or ab_code==60057:
  1533. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1534. else:
  1535. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1536. # log_.info({
  1537. # 'logTimestamp': int(time.time() * 1000),
  1538. # 'request_id': request_id,
  1539. # 'app_type': app_type,
  1540. # 'mid': mid,
  1541. # 'uid': uid,
  1542. # 'operation': 'update_redis_data',
  1543. # 'executeTime': (time.time() - update_redis_st) * 1000
  1544. # })
  1545. recommend_result['updateRedisDataTime'] = (time.time() - update_redis_st) * 1000
  1546. return recommend_result
  1547. # return rank_result
  1548. def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_exp_info, client_info,
  1549. page_type, params, ab_info_data, version_audit_status):
  1550. """
  1551. 相关推荐逻辑
  1552. :param request_id: request_id
  1553. :param video_id: 相关推荐的头部视频id
  1554. :param mid: mid type-string
  1555. :param uid: uid type-string
  1556. :param size: 请求视频数量 type-int
  1557. :param app_type: 产品标识 type-int
  1558. :param ab_exp_info: ab实验分组参数 [{"expItemId":1, "configValue":{"size":4, "K":3, ...}}, ...]
  1559. :param client_info: 地域参数
  1560. :param page_type: 页面区分参数 1:详情页;2:分享页
  1561. :param params:
  1562. :param ab_info_data: app实验分组参数
  1563. :param version_audit_status: 小程序版本审核参数:1-审核中,2-审核通过
  1564. :return: videos type-list
  1565. """
  1566. recommend_result = {}
  1567. param_st = time.time()
  1568. # 特殊mid 和 小程序审核版本推荐处理
  1569. if mid in get_special_mid_list() or version_audit_status == 1:
  1570. rank_result = special_mid_recommend(request_id=request_id, mid=mid, uid=uid, app_type=app_type, size=size)
  1571. recommend_result['videos'] = rank_result
  1572. return recommend_result
  1573. # return rank_result
  1574. # 普通mid推荐处理
  1575. top_K, flow_pool_P, ab_code, rule_key, data_key, expire_time, \
  1576. no_op_flag, old_video_index, rule_key_30day, shield_config = \
  1577. get_recommend_params(recommend_type=1, ab_exp_info=ab_exp_info, ab_info_data=ab_info_data, page_type=page_type,
  1578. mid=mid, app_type=app_type)
  1579. # log_.info({
  1580. # 'logTimestamp': int(time.time() * 1000),
  1581. # 'request_id': request_id,
  1582. # 'app_type': app_type,
  1583. # 'mid': mid,
  1584. # 'uid': uid,
  1585. # 'operation': 'get_recommend_params',
  1586. # 'executeTime': (time.time() - param_st) * 1000
  1587. # })
  1588. recommend_result['getRecommendParamsTime'] = (time.time() - param_st) * 1000
  1589. # 简单召回 - 排序 - 兜底
  1590. get_result_st = time.time()
  1591. #print("ab_code:", ab_code)
  1592. if ab_code == 60047 or ab_code == 60048 or ab_code == 60049:
  1593. result = new_video_recommend(request_id=request_id,
  1594. mid=mid, uid=uid, app_type=app_type,
  1595. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1596. algo_type='', client_info=client_info,
  1597. ab_code=ab_code, expire_time=expire_time,
  1598. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1599. old_video_index=old_video_index, video_id=video_id,
  1600. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1601. # log_.info({
  1602. # elif ab_code == 60050 or ab_code == 60051:
  1603. # result = video_recommend(request_id=request_id,
  1604. # mid=mid, uid=uid, app_type=app_type,
  1605. # size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1606. # algo_type='', client_info=client_info,
  1607. # ab_code=ab_code, expire_time=expire_time,
  1608. # rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1609. # old_video_index=old_video_index, video_id=video_id,
  1610. # params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1611. elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
  1612. ab_code == 60056 or ab_code==60057:
  1613. result = video_old_recommend(request_id=request_id,
  1614. mid=mid, uid=uid, app_type=app_type,
  1615. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1616. algo_type='', client_info=client_info,
  1617. ab_code=ab_code, expire_time=expire_time,
  1618. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1619. old_video_index=old_video_index, video_id=video_id,
  1620. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1621. else:
  1622. result = video_recommend(request_id=request_id,
  1623. mid=mid, uid=uid, app_type=app_type,
  1624. size=size, top_K=top_K, flow_pool_P=flow_pool_P,
  1625. algo_type='', client_info=client_info,
  1626. ab_code=ab_code, expire_time=expire_time,
  1627. rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
  1628. old_video_index=old_video_index, video_id=video_id,
  1629. params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
  1630. # log_.info({
  1631. # 'logTimestamp': int(time.time() * 1000),
  1632. # 'request_id': request_id,
  1633. # 'app_type': app_type,
  1634. # 'mid': mid,
  1635. # 'uid': uid,
  1636. # 'operation': 'get_recommend_result',
  1637. # 'executeTime': (time.time() - get_result_st) * 1000
  1638. # })
  1639. recommend_result['recommendOperation'] = result
  1640. rank_result = result.get('rankResult')
  1641. for i, item in enumerate(rank_result):
  1642. item['position'] = i+1
  1643. recommend_result['videos'] = rank_result
  1644. recommend_result['getRecommendResultTime'] = (time.time() - get_result_st) * 1000
  1645. # ab-test
  1646. # result = ab_test_op(rank_result=rank_result,
  1647. # ab_code_list=[config_.AB_CODE['position_insert'], config_.AB_CODE['relevant_video_op']],
  1648. # app_type=app_type, mid=mid, uid=uid, head_vid=video_id, size=size)
  1649. # redis数据刷新
  1650. update_redis_st = time.time()
  1651. if ab_code == 60047 or ab_code == 60048 or ab_code == 60049:
  1652. update_flow_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1653. # elif ab_code == 60050 or ab_code == 60051:
  1654. # update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1655. elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
  1656. ab_code == 60056 or ab_code==60057:
  1657. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1658. else:
  1659. update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
  1660. # log_.info({
  1661. # 'logTimestamp': int(time.time() * 1000),
  1662. # 'request_id': request_id,
  1663. # 'app_type': app_type,
  1664. # 'mid': mid,
  1665. # 'uid': uid,
  1666. # 'operation': 'update_redis_data',
  1667. # 'executeTime': (time.time() - update_redis_st) * 1000
  1668. # })
  1669. recommend_result['updateRedisDataTime'] = (time.time() - update_redis_st) * 1000
  1670. return recommend_result
  1671. # return rank_result
  1672. def special_mid_recommend(request_id, mid, uid, app_type, size,
  1673. ab_code=config_.AB_CODE['special_mid'],
  1674. push_from=config_.PUSH_FROM['special_mid'],
  1675. expire_time=24*3600):
  1676. redis_helper = RedisHelper()
  1677. # 特殊mid推荐指定视频列表
  1678. pool_recall = PoolRecall(request_id=request_id, app_type=app_type,
  1679. mid=mid, uid=uid, ab_code=ab_code)
  1680. # 获取相关redis key
  1681. special_key_name, redis_date = pool_recall.get_pool_redis_key(pool_type='special')
  1682. # 用户上一次在rov召回池对应的位置
  1683. last_special_recall_key = f'{config_.LAST_VIDEO_FROM_SPECIAL_POOL_PREFIX}{app_type}:{mid}:{redis_date}'
  1684. value = redis_helper.get_data_from_redis(last_special_recall_key)
  1685. if value:
  1686. idx = redis_helper.get_index_with_data(special_key_name, value)
  1687. if not idx:
  1688. idx = 0
  1689. else:
  1690. idx += 1
  1691. else:
  1692. idx = 0
  1693. recall_result = []
  1694. # 每次获取的视频数
  1695. get_size = size * 5
  1696. # 记录获取频次
  1697. freq = 0
  1698. while len(recall_result) < size:
  1699. freq += 1
  1700. if freq > config_.MAX_FREQ_FROM_ROV_POOL:
  1701. break
  1702. # 获取数据
  1703. data = redis_helper.get_data_zset_with_index(key_name=special_key_name,
  1704. start=idx, end=idx + get_size - 1,
  1705. with_scores=True)
  1706. if not data:
  1707. break
  1708. # 获取视频id,并转换类型为int,并存储为key-value{videoId: score}
  1709. # 添加视频源参数 pushFrom, abCode
  1710. temp_result = [{'videoId': int(value[0]), 'rovScore': value[1],
  1711. 'pushFrom': push_from, 'abCode': ab_code}
  1712. for value in data]
  1713. recall_result.extend(temp_result)
  1714. idx += get_size
  1715. # 将此次获取的末位视频id同步刷新到Redis中,方便下次快速定位到召回位置,过期时间为1天
  1716. if mid and recall_result:
  1717. # mid为空时,不做记录
  1718. redis_helper.set_data_to_redis(key_name=last_special_recall_key,
  1719. value=recall_result[:size][-1]['videoId'],
  1720. expire_time=expire_time)
  1721. return recall_result[:size]
  1722. def get_special_mid_list():
  1723. redis_helper = RedisHelper()
  1724. special_mid_list = redis_helper.get_data_from_set(key_name=config_.KEY_NAME_SPECIAL_MID)
  1725. if special_mid_list:
  1726. return special_mid_list
  1727. else:
  1728. return []
  1729. if __name__ == '__main__':
  1730. videos = [
  1731. {"videoId": 10136461, "rovScore": 99.971, "pushFrom": "recall_pool", "abCode": 10000},
  1732. {"videoId": 10239014, "rovScore": 99.97, "pushFrom": "recall_pool", "abCode": 10000},
  1733. {"videoId": 9851154, "rovScore": 99.969, "pushFrom": "recall_pool", "abCode": 10000},
  1734. {"videoId": 10104347, "rovScore": 99.968, "pushFrom": "recall_pool", "abCode": 10000},
  1735. {"videoId": 10141507, "rovScore": 99.967, "pushFrom": "recall_pool", "abCode": 10000},
  1736. {"videoId": 10292817, "flowPool": "2#6#2#1641780979606", "rovScore": 53.926690610816486,
  1737. "pushFrom": "flow_pool", "abCode": 10000},
  1738. {"videoId": 10224932, "flowPool": "2#5#1#1641800279644", "rovScore": 53.47890460059617, "pushFrom": "flow_pool",
  1739. "abCode": 10000},
  1740. {"videoId": 9943255, "rovScore": 99.966, "pushFrom": "recall_pool", "abCode": 10000},
  1741. {"videoId": 10282970, "flowPool": "2#5#1#1641784814103", "rovScore": 52.682815076325575,
  1742. "pushFrom": "flow_pool", "abCode": 10000},
  1743. {"videoId": 10282205, "rovScore": 99.965, "pushFrom": "recall_pool", "abCode": 10000}
  1744. ]
  1745. res = relevant_video_top_recommend(app_type=4, mid='', uid=1111, head_vid=123, videos=videos, size=10)
  1746. print(res)