Merge branch 'master_2023_06_19' of algorithm/rov-server into master

config.py

@@ -158,7 +158,8 @@ class BaseConfig(object):
             'abtest_336': 60059,
             'abtest_337': 60060,
             'abtest_338': 60061,
-            'abtest_339': 60062,
+            'abtest_367': 60062,
+            'abtest_368': 60063,
         },  # 地域分组小时级规则实验
 
         'rank_by_24h': {
@@ -391,9 +392,13 @@ class BaseConfig(object):
             'data_key': 'data10', 'rule_key': 'rule7',
             'ab_code': AB_CODE['region_rank_by_h'].get('abtest_338')
         },
-        '339': {
+        '367': {
             'data_key': 'data10', 'rule_key': 'rule7',
-            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_339')
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_367')
+        },
+        '368': {
+            'data_key': 'data10', 'rule_key': 'rule7',
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_368')
         }
     }
 

recommend.py

@@ -209,6 +209,8 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
         exp_config = pool_recall.get_simrecall_config()
     elif ab_code == 60062:
         exp_config = pool_recall.get_u2u2i_config()
+    elif ab_code == 60063:
+        exp_config = pool_recall.get_simrecall_config_new()
     #print("exp_config:", exp_config)
     if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
         t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config)]
@@ -216,7 +218,7 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
             t.append(gevent.spawn(pool_recall.get_U2I_reall, mid, exp_config))
         elif  ab_code==60059:
             t.append(gevent.spawn(pool_recall.get_word2vec_item_reall, exp_config))
-        elif  ab_code==60061:
+        elif  ab_code==60061 or ab_code==60063:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
         elif  ab_code==60062:
             t.append(gevent.spawn(pool_recall.get_U2U2I_reall, mid, exp_config))
@@ -228,7 +230,7 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
             t.append(gevent.spawn(pool_recall.get_U2I_reall, mid, exp_config))
         elif ab_code == 60059:
             t.append(gevent.spawn(pool_recall.get_word2vec_item_reall, exp_config))
-        elif ab_code == 60061:
+        elif ab_code == 60061 or ab_code==60063:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
         elif ab_code == 60062:
             t.append(gevent.spawn(pool_recall.get_U2U2I_reall, mid, exp_config))
@@ -322,7 +324,7 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
     elif ab_code == 60059:
         if len(recall_result_list)>=4:
             data['w2v_recall'] = recall_result_list[3]
-    elif ab_code == 60061:
+    elif (ab_code == 60061 or ab_code == 60063):
         if len(recall_result_list)>=4:
             data['sim_recall'] = recall_result_list[3]
     elif ab_code == 60062:
@@ -333,7 +335,8 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
     #else:
     #print("data['hot_recall']", data['hot_recall'])
     # 60058: u2itag, 60059:word2vec, 60061: sim_recall, 60062: u2u2i
-    if ab_code == 60058 or ab_code == 60059 or ab_code == 60060 or ab_code == 60061 or ab_code == 60062:
+    if ab_code == 60058 or ab_code == 60059 or ab_code == 60060 or ab_code == 60061 \
+        or ab_code == 60062 or ab_code== 60063:
         rank_result, flow_num = video_sank_pos_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P), ab_Code=ab_code, exp_config=exp_config)
         result['flow_num'] = flow_num
         if rank_result:
@@ -438,7 +441,7 @@ def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
 
     if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
         t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config)]
-        if ab_code ==60054:
+        if ab_code ==60054 :
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
         if ab_code == 60055:
             t.append(gevent.spawn(pool_recall.get_3days_hot_item_reall, exp_config))

video_rank.py

@@ -758,7 +758,7 @@ def video_sanke_rank(data, size, top_K, flow_pool_P, ab_Code='', exp_config=None
             recall_dict['w2v_recall'] = data['w2v_recall']
         else:
             recall_dict['w2v_recall'] = w2v_recall
-    elif ab_Code==60061:
+    elif ab_Code==60061 or ab_Code==60063:
         if len(data['sim_recall'])>0:
             recall_dict['sim_recall'] = data['sim_recall']
         else:
@@ -928,7 +928,7 @@ def video_sank_pos_rank(data, size, top_K, flow_pool_P, ab_Code='', exp_config=N
             recall_dict['w2v_recall'] = data['w2v_recall']
         else:
             recall_dict['w2v_recall'] = w2v_recall
-    elif ab_Code==60061:
+    elif ab_Code==60061 or ab_Code==60063:
         if len(data['sim_recall'])>0:
             recall_dict['sim_recall'] = data['sim_recall']
         else:
@@ -963,24 +963,39 @@ def video_sank_pos_rank(data, size, top_K, flow_pool_P, ab_Code='', exp_config=N
     recall_list.append(recall_pos3)
     recall_list.append(recall_pos4)
     select_ids = set('')
+    recall_num_limit_dict = {}
+    if 'recall_num_limit' in exp_config:
+        recall_num_limit_dict = exp_config['recall_num_limit']
+    exp_recall_dict = {}
     for j in range(3):
         if len(rov_recall_rank)>12:
             break
         for recall_pos_config in recall_list:
             rand_num = random.random()
+            #print("rand_num:", rand_num)
             for per_recall_item in recall_pos_config:
                 per_recall_name = per_recall_item[0]
                 per_recall_freq = per_recall_item[1]
+                per_recall_num = exp_recall_dict.get(per_recall_name, 0)
+                per_recall_total_num = recall_num_limit_dict.get(per_recall_name, 0)
+                # recall set total num
+                if len(recall_num_limit_dict)>0 and per_recall_total_num>0 and per_recall_num>= per_recall_total_num:
+                    continue
                 if rand_num < per_recall_freq and per_recall_name in recall_dict:
                     per_recall = recall_dict[per_recall_name]
                     for recall_item in per_recall:
                         vid = recall_item['videoId']
                         if vid in select_ids:
                             continue
+                        #recall_item['rand'] = rand_num
                         rov_recall_rank.append(recall_item)
                         select_ids.add(vid)
+                        if per_recall_name in exp_recall_dict:
+                            exp_recall_dict[per_recall_name] +=1
+                        else:
+                            exp_recall_dict[per_recall_name] = 1
                         break
-
+    #print("rov_recall_rank:", rov_recall_rank)
     if len(rov_recall_rank)<4:
         rov_doudi_rank = region_h_recall_rank + sim_recall + u2i_recall + u2u2i_recall + w2v_recall + region_24h_recall_rank + rule_24h_recall_rank + rule_24h_dup_recall_rank
         for recall_item in rov_doudi_rank:

video_recall.py

@@ -2602,6 +2602,21 @@ class PoolRecall(object):
         else:
             return None
 
+    def get_simrecall_config_new(self):
+        recall_key = "simrecall_exp_config_pos_new"
+        #print("recall_key:", recall_key)
+        data = self.redis_helper.get_data_from_redis(key_name=recall_key)
+        if data is not None:
+            try:
+                # print(data)
+                json_result = json.loads(data)
+                # print(json_result)
+                return json_result
+            except Exception as e:
+                return None
+        else:
+            return None
+
     def get_hotrecall_config(self):
         recall_key = "ht_exp_config"
         data = self.redis_helper.get_data_from_redis(key_name=recall_key)