Merge branch 'u2i_sort_20230726' of algorithm/rov-server into master

config.py

@@ -165,6 +165,9 @@ class BaseConfig(object):
             'abtest_394': 60066,
             'abtest_395': 60067,
             'abtest_396': 60068,
+            'abtest_407': 60069,
+            'abtest_408': 60070,
+            'abtest_409': 60071,
         },  # 地域分组小时级规则实验
 
         'rank_by_24h': {
@@ -424,7 +427,19 @@ class BaseConfig(object):
         '396': {
             'data_key': 'data10', 'rule_key': 'rule7',
             'ab_code': AB_CODE['region_rank_by_h'].get('abtest_396')
-        }
+        },
+        '407': {
+            'data_key': 'data10', 'rule_key': 'rule7',
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_407')
+        },
+        '408': {
+            'data_key': 'data10', 'rule_key': 'rule7',
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_408')
+        },
+        '409': {
+            'data_key': 'data10', 'rule_key': 'rule7',
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_409')
+        },
     }
 
     # APP ab实验配置

rank_service.py

@@ -21,6 +21,7 @@ def get_featurs(mid, data, size, top_K, flow_pool_P, env_dict=None):
     machineinfo_model ='M2006C3LC'
     city_code = -1
     province_code = -1
+    relevant_video_id = -1
     if env_dict and len(env_dict)>0:
         apptype = env_dict.get('app_type',4)
         pagesource = env_dict.get('pagesource', '')
@@ -32,6 +33,7 @@ def get_featurs(mid, data, size, top_K, flow_pool_P, env_dict=None):
         sencetype = env_dict.get('sencetype', '')
         city_code = env_dict.get('city_code', -1)
         province_code = env_dict.get('province_code', -1)
+        relevant_video_id = env_dict.get('relevant_video_id', -1)
         if city_code and city_code == "":
             city_code = -1
         if province_code and province_code == "":
@@ -44,6 +46,10 @@ def get_featurs(mid, data, size, top_K, flow_pool_P, env_dict=None):
             province_code = int(province_code)
         except:
             province_code = -1
+        try:
+            relevant_video_id = int(relevant_video_id)
+        except:
+            relevant_video_id = -1
     mid_list = []
     videoid_list = []
     apptype_list = []
@@ -66,6 +72,7 @@ def get_featurs(mid, data, size, top_K, flow_pool_P, env_dict=None):
     day_view_users_list = []
     day_share_users = []
     recommendsource_list = []
+    relevant_video_list = []
     recall_list = env_dict.get('recall_list', [])
     city_list = []
     province_list = []
@@ -83,6 +90,7 @@ def get_featurs(mid, data, size, top_K, flow_pool_P, env_dict=None):
             sencetype_list.append(sencetype)
             city_list.append(city_code)
             province_list.append(province_code)
+            relevant_video_list.append(relevant_video_id)
     video_static_info = env_dict.get('vid_day_fea_list', [])
     video_hour_static_info = env_dict.get('vid_hour_fea_list', [])
     #print("video_static_info:",video_static_info)
@@ -184,6 +192,7 @@ def get_featurs(mid, data, size, top_K, flow_pool_P, env_dict=None):
     feature_dict["hour_ctr_score_list"] = hour_ctr_score_list
     feature_dict["city_code"] = city_list
     feature_dict["province_code"] = province_list
+    feature_dict["relevant_video_id"] = relevant_video_list
     return feature_dict
 
 
@@ -235,7 +244,8 @@ def get_tf_serving_sores(feature_dict):
                     #"hour_return_rate": feature_dict["hour_return_rate_list"],
                     #"hour_ctr_score": feature_dict["hour_ctr_score_list"],
                     "city_code": feature_dict['city_code'],
-                    "province_code": feature_dict['province_code']
+                    "province_code": feature_dict['province_code'],
+                    "relevant_video_id":feature_dict['relevant_video_id']
     }
     request_data_dict= {}
     request_data_dict["inputs"] = inputs_data

recommend.py

@@ -454,17 +454,13 @@ def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
         t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config)]
         if ab_code ==60054 or ab_code==60066:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
-        if ab_code == 60055:
-            t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config))
-        if ab_code == 60056:
+        if ab_code == 60056 or ab_code == 60071:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
             t.append(gevent.spawn(pool_recall.get_U2I_reall, mid , exp_config))
-        if ab_code == 60065 :
-            t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config, 'rv2:'))
-        if ab_code ==60067 :
+        if ab_code ==60067  or ab_code == 60069:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
             t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config))
-        if ab_code == 60068:
+        if ab_code == 60068 or ab_code == 60070:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
             t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config, 'rv2:'))
     else:
@@ -473,17 +469,13 @@ def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
              gevent.spawn(pool_recall.flow_pool_recall, size)]
         if  ab_code == 60054 or ab_code==60066:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter))
-        if ab_code == 60055:
-            t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config))
-        if ab_code == 60056:
+        if ab_code == 60056 or ab_code == 60071:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
             t.append(gevent.spawn(pool_recall.get_U2I_reall, mid , exp_config))
-        if ab_code == 60065:
-            t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config, 'rv2:'))
-        if ab_code == 60067:
+        if ab_code == 60067 or ab_code == 60069:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
             t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config))
-        if ab_code == 60068:
+        if ab_code == 60068 or ab_code == 60070:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
             t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config, 'rv2:'))
 
@@ -495,7 +487,7 @@ def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
         result['recallResult']= []
         result['rankResult'] = []
         return result
-    #1. merge simrecall
+    #1. merge simrecall or  deepfm
     if ab_code == 60054 or ab_code==60066:
         rov_pool_recall = []
         if len(recall_result_list) >= 2:
@@ -522,35 +514,8 @@ def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
                         now_video_ids.add(video_id)
             if len(rov_pool_recall) > 0:
                 recall_result_list[0] = rov_pool_recall
-    # merge return video
-    if ab_code == 60055 or ab_code==60065:
-        rov_pool_recall = []
-        if len(recall_result_list)>=2:
-            region_recall = recall_result_list[0]
-            return_video_reall = []
-            if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
-                return_video_reall, = recall_result_list[1]
-            else:
-                if len(recall_result_list)>=4:
-                    return_video_reall = recall_result_list[3]
-            #print("sim:",sim_recall)
-            now_video_ids = set('')
-            if len(region_recall)>0:
-                for video in region_recall:
-                    video_id = video.get('videoId')
-                    if video_id not in now_video_ids:
-                        rov_pool_recall.append(video)
-                        now_video_ids.add(video_id)
-            if len(return_video_reall)>0:
-                for video in return_video_reall:
-                    video_id = video.get('videoId')
-                    if video_id not in now_video_ids:
-                        rov_pool_recall.append(video)
-                        now_video_ids.add(video_id)
-            if len(rov_pool_recall)>0:
-                recall_result_list[0] = rov_pool_recall
-    # merge simrecall, merge u2i title recall
-    if ab_code == 60056:
+    # merge simrecall, merge u2i title recall, deepfm
+    if ab_code == 60056 or ab_code == 60071:
         rov_pool_recall = []
         if len(recall_result_list)>=2:
             region_recall = recall_result_list[0]
@@ -565,6 +530,7 @@ def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
                     sim_recall = recall_result_list[3]
                 if len(recall_result_list) >= 5:
                     u2i_title_recall = recall_result_list[4]
+            #print("u2i_title_recall:", u2i_title_recall)
             now_video_ids = set('')
             if len(region_recall)>0:
                 for video in region_recall:
@@ -589,7 +555,7 @@ def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
                 recall_result_list[0] = rov_pool_recall
 
     #2. merge simrecall, return video
-    if ab_code == 60067 or ab_code==60068:
+    if ab_code == 60067 or ab_code==60068 or ab_code==60069 or ab_code==60070:
         rov_pool_recall = []
         if len(recall_result_list)>=2:
             region_recall = recall_result_list[0]
@@ -1730,9 +1696,9 @@ def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
     # return video, return video2
     # old video: 60056, test2
     elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 \
-            or ab_code == 60055 or ab_code == 60056 or ab_code==60057  \
-            or ab_code==60065 or ab_code == 60067 or ab_code==60068 \
-            or ab_code==60066:
+             or ab_code == 60056 or ab_code==60057  \
+             or ab_code == 60067 or ab_code==60068 \
+            or ab_code==60066 or ab_code == 60069 or ab_code == 60070 or ab_code == 60071:
         result, fea_info = video_old_recommend(request_id=request_id,
                                      mid=mid, uid=uid, app_type=app_type,
                                      size=size, top_K=top_K, flow_pool_P=flow_pool_P,
@@ -1850,9 +1816,9 @@ def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_
                                  rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
                                  old_video_index=old_video_index, video_id=video_id,
                                  params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
-    elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
-            ab_code == 60056 or ab_code==60057 or ab_code ==60065 or ab_code ==60067 \
-            or ab_code ==60068 or ab_code==60066:
+    elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054  or \
+            ab_code == 60056 or ab_code==60057  or ab_code ==60067 \
+            or ab_code ==60068 or ab_code==60066 or ab_code == 60069 or ab_code == 60070 or ab_code == 60071:
         result, fea_info = video_old_recommend(request_id=request_id,
                                  mid=mid, uid=uid, app_type=app_type,
                                  size=size, top_K=top_K, flow_pool_P=flow_pool_P,
@@ -1896,11 +1862,6 @@ def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_
     update_redis_st = time.time()
     if ab_code == 60047 or ab_code == 60048 or  ab_code == 60049:
          update_flow_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
-    # elif ab_code == 60050 or ab_code == 60051:
-    #      update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
-    # elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
-    #         ab_code == 60056 or ab_code==60057 or ab_code ==60065:
-    #      update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
     else:
          update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
     # log_.info({

video_rank.py

@@ -627,7 +627,7 @@ def video_new_rank2(data, size, top_K, flow_pool_P, ab_code, mid, exp_config=Non
     rov_recall_rank = rec_recall_item_list
     #call rank service
     #flag_call_service = 0
-    if ab_code == 60066:
+    if ab_code == 60066 or ab_code == 60069 or ab_code == 60070 or ab_code == 60071:
         feature_dict = get_featurs(mid, data, size, top_K, flow_pool_P, env_dict)
         score_result = get_tf_serving_sores(feature_dict)
         if rec_recall_item_list and score_result and len(score_result) > 0 and len(score_result) == len(rec_recall_item_list):