Merge branch 'feature_2023062711_liqian_add_ad_abtest' into pre-master

config.py

@@ -158,7 +158,9 @@ class BaseConfig(object):
             'abtest_336': 60059,
             'abtest_337': 60060,
             'abtest_338': 60061,
-            'abtest_339': 60062,
+            'abtest_367': 60062,
+            'abtest_368': 60063,
+            'abtest_373': 60064,
         },  # 地域分组小时级规则实验
 
         'rank_by_24h': {
@@ -391,9 +393,17 @@ class BaseConfig(object):
             'data_key': 'data10', 'rule_key': 'rule7',
             'ab_code': AB_CODE['region_rank_by_h'].get('abtest_338')
         },
-        '339': {
+        '367': {
             'data_key': 'data10', 'rule_key': 'rule7',
-            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_339')
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_367')
+        },
+        '368': {
+            'data_key': 'data10', 'rule_key': 'rule7',
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_368')
+        },
+        '373': {
+            'data_key': 'data10', 'rule_key': 'rule7',
+            'ab_code': AB_CODE['region_rank_by_h'].get('abtest_373')
         }
     }
 
@@ -476,7 +486,8 @@ class BaseConfig(object):
         'hot_recall': 'hot_recall',  # hot召回
         'w2v_recall': 'w2v_recall', #word2vec
         'u2i_tag_recall':'u2i_tag_recall', #u2i_tag_recall
-        'u2u2i_recall':'u2u2i_recall', #u2u2i_recall
+        'u2u2i_recall':'u2u2i_recall', #u2u2i_recall,
+        'return_video_recall': 'return_video_recall',  # return_video_recall
     }
 
     # category id mapping
@@ -908,6 +919,14 @@ class BaseConfig(object):
             'group_class_key': 'class1',
             'no_ad_mid_group_list': NO_AD_MID_GROUP_LIST['class1']
         },  # 所有广告类型本端视频数据 + 优化1
+        '190-f': {
+            'video': {'data': 'videos21'},
+            'user': {'data': 'user21', 'rule': 'rule2'},
+            'group_class_key': 'class1',
+            'no_ad_mid_group_list': NO_AD_MID_GROUP_LIST['class1'],
+            'care_model_status_param': 1,
+            'care_model_ab_mid_group': 'mean_group',
+        },  # 所有广告类型本端视频数据 + 优化1 + 未开启关怀模式人群多出广告
 
         # 票圈视频
         '194-a': {
@@ -1034,6 +1053,14 @@ class BaseConfig(object):
             'group_class_key': 'class1',
             'no_ad_mid_group_list': NO_AD_MID_GROUP_LIST['class1']
         },  # 所有广告类型本端数据
+        '196-f': {
+            'video': {'data': 'videos6'},
+            'user': {'data': 'user6', 'rule': 'rule1'},
+            'group_class_key': 'class1',
+            'no_ad_mid_group_list': NO_AD_MID_GROUP_LIST['class1'],
+            'care_model_status_param': 1,
+            'care_model_ab_mid_group': 'mean_group',
+        },  # 所有广告类型本端数据 + else非关怀模式人群多出广告
 
         # 老好看视频
         '197-a': {
@@ -1118,6 +1145,14 @@ class BaseConfig(object):
             'group_class_key': 'class1',
             'no_ad_mid_group_list': NO_AD_MID_GROUP_LIST['class1']
         },  # 所有广告类型本端数据 + 优化阈值计算方式
+        '242-d': {
+            'video': {'data': 'videos22'},
+            'user': {'data': 'user22', 'rule': 'rule2'},
+            'group_class_key': 'class1',
+            'no_ad_mid_group_list': NO_AD_MID_GROUP_LIST['class1'],
+            'care_model_status_param': 1,
+            'care_model_ab_mid_group': 'mean_group',
+        },  # 所有广告类型本端数据 + 优化阈值计算方式 + else非关怀模式人群多出广告
 
         # 票圈福年
         '243-a': {
@@ -1138,6 +1173,14 @@ class BaseConfig(object):
             'group_class_key': 'class1',
             'no_ad_mid_group_list': NO_AD_MID_GROUP_LIST['class1']
         },  # 所有广告类型本端数据
+        '243-d': {
+            'video': {'data': 'videos3'},
+            'user': {'data': 'user3', 'rule': 'rule1'},
+            'group_class_key': 'class1',
+            'no_ad_mid_group_list': NO_AD_MID_GROUP_LIST['class1'],
+            'care_model_status_param': 1,
+            'care_model_ab_mid_group': 'mean_group',
+        },  # 所有广告类型本端数据 + else非关怀模式人群多出广告
 
         # 票圈|信仰之路
         '324-a': {

recommend.py

@@ -209,6 +209,10 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
         exp_config = pool_recall.get_simrecall_config()
     elif ab_code == 60062:
         exp_config = pool_recall.get_u2u2i_config()
+    elif ab_code == 60063:
+        exp_config = pool_recall.get_simrecall_config_new()
+    elif ab_code == 60064:
+        exp_config = pool_recall.get_video_recall_config()
     #print("exp_config:", exp_config)
     if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
         t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config)]
@@ -216,10 +220,12 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
             t.append(gevent.spawn(pool_recall.get_U2I_reall, mid, exp_config))
         elif  ab_code==60059:
             t.append(gevent.spawn(pool_recall.get_word2vec_item_reall, exp_config))
-        elif  ab_code==60061:
+        elif  ab_code==60061 or ab_code==60063:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
         elif  ab_code==60062:
             t.append(gevent.spawn(pool_recall.get_U2U2I_reall, mid, exp_config))
+        elif  ab_code==60064:
+            t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config))
     else:
         t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config),
              gevent.spawn(pool_recall.flow_pool_recall, size, config_.QUICK_FLOW_POOL_ID),
@@ -228,11 +234,12 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
             t.append(gevent.spawn(pool_recall.get_U2I_reall, mid, exp_config))
         elif ab_code == 60059:
             t.append(gevent.spawn(pool_recall.get_word2vec_item_reall, exp_config))
-        elif ab_code == 60061:
+        elif ab_code == 60061 or ab_code==60063:
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
         elif ab_code == 60062:
             t.append(gevent.spawn(pool_recall.get_U2U2I_reall, mid, exp_config))
-
+        elif  ab_code==60064:                                                                         
+            t.append(gevent.spawn(pool_recall.get_return_video_reall, exp_config))
     # 最惊奇相关推荐实验
     # elif ab_code == config_.AB_CODE['top_video_relevant_appType_19']:
     #     t = [gevent.spawn(pool_recall.relevant_recall_19, video_id, size, expire_time),
@@ -322,18 +329,20 @@ def video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type, al
     elif ab_code == 60059:
         if len(recall_result_list)>=4:
             data['w2v_recall'] = recall_result_list[3]
-    elif ab_code == 60061:
+    elif (ab_code == 60061 or ab_code == 60063):
         if len(recall_result_list)>=4:
             data['sim_recall'] = recall_result_list[3]
     elif ab_code == 60062:
         if len(recall_result_list)>=4:
             data['u2u2i_recall'] = recall_result_list[3]
-    #if ab_code=="ab_new_test":
-    #    rank_result = video_new_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
+    elif ab_code == 60064:
+        if len(recall_result_list)>=4:                     #if ab_code=="ab_new_test":
+            data['return_video_recall'] = recall_result_list[3]   #    rank_result = video_new_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P))
     #else:
     #print("data['hot_recall']", data['hot_recall'])
     # 60058: u2itag, 60059:word2vec, 60061: sim_recall, 60062: u2u2i
-    if ab_code == 60058 or ab_code == 60059 or ab_code == 60060 or ab_code == 60061 or ab_code == 60062:
+    if ab_code == 60058 or ab_code == 60059 or ab_code == 60060 or ab_code == 60061 \
+        or ab_code == 60062 or ab_code== 60063 or ab_code == 60064:
         rank_result, flow_num = video_sank_pos_rank(data=data, size=size, top_K=top_K, flow_pool_P=float(flow_pool_P), ab_Code=ab_code, exp_config=exp_config)
         result['flow_num'] = flow_num
         if rank_result:
@@ -438,7 +447,7 @@ def video_old_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
 
     if app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
         t = [gevent.spawn(pool_recall.rov_pool_recall_with_region, size, expire_time, ab_code, exp_config)]
-        if ab_code ==60054:
+        if ab_code ==60054 :
             t.append(gevent.spawn(pool_recall.get_sim_hot_item_reall_filter, exp_config))
         if ab_code == 60055:
             t.append(gevent.spawn(pool_recall.get_3days_hot_item_reall, exp_config))

video_rank.py

@@ -758,7 +758,7 @@ def video_sanke_rank(data, size, top_K, flow_pool_P, ab_Code='', exp_config=None
             recall_dict['w2v_recall'] = data['w2v_recall']
         else:
             recall_dict['w2v_recall'] = w2v_recall
-    elif ab_Code==60061:
+    elif ab_Code==60061 or ab_Code==60063:
         if len(data['sim_recall'])>0:
             recall_dict['sim_recall'] = data['sim_recall']
         else:
@@ -919,6 +919,7 @@ def video_sank_pos_rank(data, size, top_K, flow_pool_P, ab_Code='', exp_config=N
     w2v_recall =[]
     sim_recall = []
     u2u2i_recall = []
+    return_video_recall = []
     if ab_Code==60058:
         if len(data['u2i_recall'])>0:
             u2i_recall = sorted(data['u2i_recall'], key=lambda k: k.get('rovScore', 0), reverse=True)
@@ -928,7 +929,7 @@ def video_sank_pos_rank(data, size, top_K, flow_pool_P, ab_Code='', exp_config=N
             recall_dict['w2v_recall'] = data['w2v_recall']
         else:
             recall_dict['w2v_recall'] = w2v_recall
-    elif ab_Code==60061:
+    elif ab_Code==60061 or ab_Code==60063:
         if len(data['sim_recall'])>0:
             recall_dict['sim_recall'] = data['sim_recall']
         else:
@@ -938,15 +939,20 @@ def video_sank_pos_rank(data, size, top_K, flow_pool_P, ab_Code='', exp_config=N
             recall_dict['u2u2i_recall'] = data['u2u2i_recall']
         else:
             recall_dict['u2u2i_recall'] = u2u2i_recall
+    elif ab_Code==60064:
+        if len(data['return_video_recall'])>0:
+            recall_dict['return_video_recall'] = data['return_video_recall']
+        else:
+            recall_dict['return_video_recall'] = return_video_recall
 
-    recall_pos1 = [('rov_recall_region_h',0.98),('rov_recall_24h',0.02),('rov_recall_region_24h',1),
-                   ('rov_recall_24h',1),('rov_recall_24h_dup',1)]
-    recall_pos2 =  [('rov_recall_region_h',0.98),('rov_recall_24h',0.02),('rov_recall_region_24h',1),
-                   ('rov_recall_24h',1),('rov_recall_24h_dup',1)]
-    recall_pos3 = [('rov_recall_region_h', 0.98), ('rov_recall_24h', 0.02), ('rov_recall_region_24h', 1),
-                   ('rov_recall_24h', 1), ('rov_recall_24h_dup', 1)]
-    recall_pos4 = [('rov_recall_region_h', 0.98), ('rov_recall_24h', 0.02), ('rov_recall_region_24h', 1),
-                   ('rov_recall_24h', 1), ('rov_recall_24h_dup', 1)]
+    recall_pos1 = [('rov_recall_region_h',0, 0.98),('rov_recall_24h',0.98, 1),('rov_recall_region_24h',0,1),
+                   ('rov_recall_24h',0,1), ('rov_recall_24h_dup',0,1)]
+    recall_pos2 =  [('rov_recall_region_h',0,0.98),('rov_recall_24h',0.98,1),('rov_recall_region_24h',0,1),
+                   ('rov_recall_24h',0,1),('rov_recall_24h_dup',0,1)]
+    recall_pos3 = [('rov_recall_region_h', 0,0.98), ('rov_recall_24h', 0.98,1), ('rov_recall_region_24h', 0,1),
+                   ('rov_recall_24h', 0,1), ('rov_recall_24h_dup', 0,1)]
+    recall_pos4 = [('rov_recall_region_h', 0,0.98), ('rov_recall_24h', 0,0.02), ('rov_recall_region_24h', 0,1),
+                   ('rov_recall_24h', 0,1), ('rov_recall_24h_dup', 0,1)]
     if exp_config  and 'recall_pos1' in exp_config \
             and 'recall_pos2' in exp_config \
             and 'recall_pos3' in exp_config \
@@ -963,26 +969,50 @@ def video_sank_pos_rank(data, size, top_K, flow_pool_P, ab_Code='', exp_config=N
     recall_list.append(recall_pos3)
     recall_list.append(recall_pos4)
     select_ids = set('')
+    recall_num_limit_dict = {}
+    if exp_config and 'recall_num_limit' in exp_config:
+        recall_num_limit_dict = exp_config['recall_num_limit']
+    exp_recall_dict = {}
+   #index_pos = 0
     for j in range(3):
         if len(rov_recall_rank)>12:
             break
+        # choose pos
         for recall_pos_config in recall_list:
             rand_num = random.random()
+            index_pos = 0
+            # choose pos recall
             for per_recall_item in recall_pos_config:
+                if index_pos == 1:
+                    break
+                if len(per_recall_item)<3:
+                    continue
                 per_recall_name = per_recall_item[0]
-                per_recall_freq = per_recall_item[1]
-                if rand_num < per_recall_freq and per_recall_name in recall_dict:
+                per_recall_min = float(per_recall_item[1])
+                per_recall_max = float(per_recall_item[2])
+                per_recall_num = exp_recall_dict.get(per_recall_name, 0)
+                per_recall_total_num = recall_num_limit_dict.get(per_recall_name, 0)
+                # recall set total num
+                if len(recall_num_limit_dict)>0 and per_recall_total_num>0 and per_recall_num>= per_recall_total_num:
+                    continue
+                if rand_num >= per_recall_min and rand_num < per_recall_max and per_recall_name in recall_dict:
                     per_recall = recall_dict[per_recall_name]
                     for recall_item in per_recall:
                         vid = recall_item['videoId']
                         if vid in select_ids:
                             continue
+                        recall_item['rand'] = rand_num
                         rov_recall_rank.append(recall_item)
                         select_ids.add(vid)
+                        if per_recall_name in exp_recall_dict:
+                            exp_recall_dict[per_recall_name] +=1
+                        else:
+                            exp_recall_dict[per_recall_name] = 1
+                        index_pos = 1
                         break
-
+    #print("rov_recall_rank:", rov_recall_rank)
     if len(rov_recall_rank)<4:
-        rov_doudi_rank = region_h_recall_rank + sim_recall + u2i_recall + u2u2i_recall + w2v_recall + region_24h_recall_rank + rule_24h_recall_rank + rule_24h_dup_recall_rank
+        rov_doudi_rank = region_h_recall_rank + sim_recall + u2i_recall + u2u2i_recall + w2v_recall +return_video_recall+ region_24h_recall_rank + rule_24h_recall_rank + rule_24h_dup_recall_rank
         for recall_item in rov_doudi_rank:
             vid = recall_item['videoId']
             if vid in select_ids:

video_recall.py

@@ -2579,7 +2579,7 @@ class PoolRecall(object):
             return None
 
     def get_w2v_config(self):
-        recall_key = "w2v_exp_config_pos"
+        recall_key = "w2v_exp_config_pos_range"
         data = self.redis_helper.get_data_from_redis(key_name=recall_key)
         if data is not None:
             try:
@@ -2607,6 +2607,21 @@ class PoolRecall(object):
         else:
             return None
 
+    def get_simrecall_config_new(self):
+        recall_key = "simrecall_exp_config_range"
+        #print("recall_key:", recall_key)
+        data = self.redis_helper.get_data_from_redis(key_name=recall_key)
+        if data is not None:
+            try:
+                # print(data)
+                json_result = json.loads(data)
+                # print(json_result)
+                return json_result
+            except Exception as e:
+                return None
+        else:
+            return None
+
     def get_hotrecall_config(self):
         recall_key = "ht_exp_config"
         data = self.redis_helper.get_data_from_redis(key_name=recall_key)
@@ -2622,7 +2637,7 @@ class PoolRecall(object):
             return None
 
     def get_U2I_config(self):
-        recall_key = "u2i_exp_config_pos"
+        recall_key = "u2i_exp_config_pos_range"
         data = self.redis_helper.get_data_from_redis(key_name=recall_key)
         if data is not None:
             try:
@@ -2636,7 +2651,7 @@ class PoolRecall(object):
             return None
 
     def get_u2u2i_config(self):
-        recall_key = "u2u2i_exp_config"
+        recall_key = "u2u2i_exp_config_range"
         data = self.redis_helper.get_data_from_redis(key_name=recall_key)
         if data is not None:
             try:
@@ -2804,4 +2819,64 @@ class PoolRecall(object):
             if vid in recall_dict:
                 recall_result.append(recall_dict[vid])
         #print("u2i recall_result:", recall_result)
-        return recall_result
+        return recall_result
+
+    def get_video_recall_config(self):
+        recall_key = "vr_exp_pos_config_range"
+        data = self.redis_helper.get_data_from_redis(key_name=recall_key)
+        if data is not None:
+            try:
+                # print(data)
+                json_result = json.loads(data)
+                # print(json_result)
+                return json_result
+            except Exception as e:
+                return None
+        else:
+            return None
+
+    def get_return_video_reall(self, exp_config=None):
+        if self.video_id is None:
+            return  []
+        recall_key = "rv:"+ str(self.video_id)
+        #print("recall_key:", recall_key)
+        data = self.redis_helper.get_data_from_redis(key_name=recall_key)
+
+        #print(data)
+        recall_result = []
+        recall_dict  = {}
+        video_ids = []
+        if data is not None:
+            json_result = json.loads(data)
+            #print("json_result:", json_result)
+            for per_item in json_result:
+                try:
+                    vid = int(per_item[0])
+                    video_ids.append(vid)
+                    recall_dict[vid] = {'videoId': vid, 'flowPool': '',
+                         'rovScore': float(per_item[1]), 'pushFrom': config_.PUSH_FROM['return_video_recall'],
+                         'abCode': self.ab_code}
+                except Exception as e:
+                    continue
+        if len(video_ids)<=0:
+            return  recall_result
+        recall_num = 20
+        try:
+            if exp_config and exp_config['recall_get_num']:
+                recall_num = int(exp_config['recall_get_num'])
+        except:
+            recall_num = 20
+        #print("recall_num:", recall_num)
+        video_ids = video_ids[:recall_num]
+        #print(video_ids)
+        filter_ = FilterVideos(request_id=self.request_id,
+                               app_type=self.app_type, mid=self.mid, uid=self.uid, video_ids=video_ids)
+        filtered_viewed_videos = filter_.filter_videos_status(pool_type='normal')
+        if filtered_viewed_videos is None:
+            return recall_result
+        #print("filtered_viewed_videos:", filtered_viewed_videos)
+        for vid in filtered_viewed_videos:
+            if vid in recall_dict:
+                recall_result.append(recall_dict[vid])
+        #print("u2i recall_result:", recall_result)
+        return recall_result