add test rule_rank1_50

config.py

@@ -15,6 +15,15 @@ class BaseConfig(object):
         'ZUI_JING_QI': 19,  # 票圈最惊奇
         'APP': 13,  # 票圈视频APP
     }
+
+    # AB实验配置
+    AB_EXP_CODE = {
+        'rec_size_home': '003',
+        'rec_size_relevant': '004',
+        'rule_rank1_20': '011',
+        'rule_rank1_50': '016'
+    }
+
     # abTest
     AB_TEST = {
         'w_h_rate': [APP_TYPE['LONG_VIDEO']],  # 视频宽高比实验(每组的前两个视频调整为横屏视频), 已下线
@@ -28,7 +37,7 @@ class BaseConfig(object):
         'w_h_rate': 10001,  # 视频宽高比实验(每组的前两个视频调整为横屏视频)，已下线
         'position_insert': 10002,  # 按位置插入
         'relevant_video_op': 10003,  # 运营对某些视频给定一些相关视频，调整为对应视频相关推荐的头部
-        'rank_by_h': 20001,  # 小时级别更新rov列表实验
+        'rank_by_h': [20001, 20002],  # 小时级别更新rov列表实验
     }
 
     # pushFrom
@@ -56,10 +65,11 @@ class BaseConfig(object):
     # ROV召回池redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.{date}
     RECALL_KEY_NAME_PREFIX = 'com.weiqu.video.recall.hot.item.score.'
 
-    # 小程序小时级更新结果存放 redis key前缀，完整格式：com.weiqu.video.recall.item.score.h.{date}.{h}
+    # 小程序小时级更新结果存放 redis key前缀，完整格式：com.weiqu.video.recall.item.score.h.{return_count}.{date}.{h}
     RECALL_KEY_NAME_PREFIX_BY_H = 'com.weiqu.video.recall.item.score.h.'
 
-    # 小程序离线ROV模型结果与小程序小时级更新结果去重后 存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.dup.h.{date}.{h}
+    # 小程序离线ROV模型结果与小程序小时级更新结果去重后 存放 redis key前缀，
+    # 完整格式：com.weiqu.video.recall.hot.item.score.dup.h.{return_count}{date}.{h}
     RECALL_KEY_NAME_PREFIX_DUP_H = 'com.weiqu.video.recall.hot.item.score.dup.h.'
 
     # 每个mid存储对应小时级更新结果 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.h.mid.{mid}
@@ -67,8 +77,8 @@ class BaseConfig(object):
     # 记录 mid-小时级key 中数据所属(date，h)，完整格式：com.weiqu.video.h.record.mid.{mid}
     H_WITH_MID_RECORD_KEY_NAME_PREFIX = 'com.weiqu.video.h.record.mid.'
 
-    # 小时级视频状态不符合推荐要求的列表 redis key，完整格式：com.weiqu.video.filter.h.item
-    H_VIDEO_FILER = 'com.weiqu.video.filter.h.item'
+    # 小时级视频状态不符合推荐要求的列表 redis key，完整格式：com.weiqu.video.filter.h.item.{return_count}
+    H_VIDEO_FILER = 'com.weiqu.video.filter.h.item.'
 
     # app应用 小程序离线ROV模型结果存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.app.{date}
     RECALL_KEY_NAME_PREFIX_APP = 'com.weiqu.video.recall.hot.item.score.app.'
@@ -179,12 +189,13 @@ class DevelopmentConfig(BaseConfig):
         'PROJECT': 'rov-server-test',
     }
 
-    # AB实验配置
-    AB_EXP_CODE = {
-        'rec_size_home': '003',
-        'rec_size_relevant': '004',
-        'rule_rank1': '011'
-    }
+    # # AB实验配置
+    # AB_EXP_CODE = {
+    #     'rec_size_home': '003',
+    #     'rec_size_relevant': '004',
+    #     'rule_rank1_20': '011',
+    #     'rule_rank1_50': '016'
+    # }
 
 
 class TestConfig(BaseConfig):
@@ -235,12 +246,13 @@ class TestConfig(BaseConfig):
         'PROJECT': 'rov-server-test',
     }
 
-    # AB实验配置
-    AB_EXP_CODE = {
-        'rec_size_home': '003',
-        'rec_size_relevant': '004',
-        'rule_rank1': '011'
-    }
+    # # AB实验配置
+    # AB_EXP_CODE = {
+    #     'rec_size_home': '003',
+    #     'rec_size_relevant': '004',
+    #     'rule_rank1_20': '011',
+    #     'rule_rank1_50': '016'
+    # }
 
 
 class PreProductionConfig(BaseConfig):
@@ -291,12 +303,12 @@ class PreProductionConfig(BaseConfig):
         'PROJECT': 'rov-server',
     }
 
-    # AB实验配置
-    AB_EXP_CODE = {
-        'rec_size_home': '003',
-        'rec_size_relevant': '004',
-        'rule_rank1': '011'
-    }
+    # # AB实验配置
+    # AB_EXP_CODE = {
+    #     'rec_size_home': '003',
+    #     'rec_size_relevant': '004',
+    #     'rule_rank1': '011'
+    # }
 
 
 class ProductionConfig(BaseConfig):
@@ -347,12 +359,12 @@ class ProductionConfig(BaseConfig):
         'PROJECT': 'rov-server',
     }
 
-    # AB实验配置
-    AB_EXP_CODE = {
-        'rec_size_home': '003',
-        'rec_size_relevant': '004',
-        'rule_rank1': '011'
-    }
+    # # AB实验配置
+    # AB_EXP_CODE = {
+    #     'rec_size_home': '003',
+    #     'rec_size_relevant': '004',
+    #     'rule_rank1': '011'
+    # }
 
 
 def set_config():

recommend.py

@@ -136,7 +136,7 @@ def positon_duplicate(pos1_vids, pos2_vids, videos):
 
 
 def video_recommend(mid, uid, size, top_K, flow_pool_P, app_type, algo_type, client_info, expire_time=24*3600,
-                    ab_code=config_.AB_CODE['initial']):
+                    ab_code=config_.AB_CODE['initial'], return_count=-1):
     """
     首页线上推荐逻辑
     :param mid: mid type-string
@@ -171,9 +171,9 @@ def video_recommend(mid, uid, size, top_K, flow_pool_P, app_type, algo_type, cli
     '''
     recall_result_list = []
     pool_recall = PoolRecall(app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
-                             client_info=client_info)
+                             client_info=client_info, return_count=return_count)
     _, last_rov_recall_key, _ = pool_recall.get_video_last_idx()
-    if ab_code == config_.AB_CODE['rank_by_h']:
+    if ab_code in config_.AB_CODE['rank_by_h']:
         t = [gevent.spawn(pool_recall.rov_pool_recall_by_h, size, expire_time),
              gevent.spawn(pool_recall.flow_pool_recall, size)]
     else:
@@ -378,6 +378,7 @@ def video_homepage_recommend(mid, uid, size, app_type, algo_type, client_info, a
             flow_pool_P = config_.P
             ab_code = config_.AB_CODE['initial']
             expire_time = 24 * 3600
+            return_count = -1
         else:
             ab_exp_code_list = []
             config_value_dict = {}
@@ -407,18 +408,25 @@ def video_homepage_recommend(mid, uid, size, app_type, algo_type, client_info, a
                 flow_pool_P = config_.P
 
             # 小时级更新-规则1 实验
-            if config_.AB_EXP_CODE['rule_rank1'] in ab_exp_code_list:
-                ab_code = config_.AB_CODE['rank_by_h']
+            if config_.AB_EXP_CODE['rule_rank1_20'] in ab_exp_code_list:
+                ab_code = config_.AB_CODE['rank_by_h'][0]
                 expire_time = 3600
+                return_count = 20
+            elif config_.AB_EXP_CODE['rule_rank1_50'] in ab_exp_code_list:
+                ab_code = config_.AB_CODE['rank_by_h'][1]
+                expire_time = 3600
+                return_count = 50
             else:
                 ab_code = config_.AB_CODE['initial']
                 expire_time = 24 * 3600
+                return_count = -1
 
         # 简单召回 - 排序 - 兜底
         rank_result, last_rov_recall_key = video_recommend(mid=mid, uid=uid, app_type=app_type,
                                                            size=size, top_K=top_K, flow_pool_P=flow_pool_P,
                                                            algo_type=algo_type, client_info=client_info,
-                                                           ab_code=ab_code, expire_time=expire_time)
+                                                           ab_code=ab_code, expire_time=expire_time,
+                                                           return_count=return_count)
         # ab-test
         # result = ab_test_op(rank_result=rank_result,
         #                     ab_code_list=[config_.AB_CODE['position_insert']],
@@ -448,6 +456,7 @@ def video_relevant_recommend(video_id, mid, uid, size, app_type, ab_exp_info):
         flow_pool_P = config_.P
         ab_code = config_.AB_CODE['initial']
         expire_time = 24 * 3600
+        return_count = -1
     else:
         ab_exp_code_list = []
         config_value_dict = {}
@@ -478,18 +487,25 @@ def video_relevant_recommend(video_id, mid, uid, size, app_type, ab_exp_info):
             flow_pool_P = config_.P
 
         # 小时级更新-规则1 实验
-        if config_.AB_EXP_CODE['rule_rank1'] in ab_exp_code_list:
-            ab_code = config_.AB_CODE['rank_by_h']
+        if config_.AB_EXP_CODE['rule_rank1_20'] in ab_exp_code_list:
+            ab_code = config_.AB_CODE['rank_by_h'][0]
+            expire_time = 3600
+            return_count = 20
+        elif config_.AB_EXP_CODE['rule_rank1_50'] in ab_exp_code_list:
+            ab_code = config_.AB_CODE['rank_by_h'][1]
             expire_time = 3600
+            return_count = 50
         else:
             ab_code = config_.AB_CODE['initial']
             expire_time = 24 * 3600
+            return_count = -1
 
     # 简单召回 - 排序 - 兜底
     rank_result, last_rov_recall_key = video_recommend(mid=mid, uid=uid, app_type=app_type,
                                                        size=size, top_K=top_K, flow_pool_P=flow_pool_P,
                                                        algo_type='', client_info=None,
-                                                       ab_code=ab_code, expire_time=expire_time)
+                                                       ab_code=ab_code, expire_time=expire_time,
+                                                       return_count=return_count)
     # ab-test
     # result = ab_test_op(rank_result=rank_result,
     #                     ab_code_list=[config_.AB_CODE['position_insert'], config_.AB_CODE['relevant_video_op']],

utils.py

@@ -148,19 +148,19 @@ class FilterVideos(object):
         self.uid = uid
         self.video_ids = video_ids
 
-    def filter_video_status_h(self, video_ids):
+    def filter_video_status_h(self, video_ids, return_count):
         """召回小时级更新的视频状态过滤"""
         # 根据Redis缓存中的数据过滤
         redis_helper = RedisHelper()
         # 获取不符合推荐状态的视频
-        filter_videos_list = redis_helper.get_data_from_set(key_name=config_.H_VIDEO_FILER)
+        filter_videos_list = redis_helper.get_data_from_set(key_name=f"{config_.H_VIDEO_FILER}{return_count}")
         if not filter_videos_list:
             return video_ids
         filter_videos = [int(video) for video in filter_videos_list]
         filtered_videos = [video_id for video_id in video_ids if video_id not in filter_videos]
         return filtered_videos
 
-    def filter_videos_h(self):
+    def filter_videos_h(self, return_count):
         """召回小时级更新的视频过滤"""
         # 预曝光过滤
         st_pre = time.time()
@@ -173,7 +173,7 @@ class FilterVideos(object):
 
         # 视频状态过滤
         st_status = time.time()
-        filtered_status_result = self.filter_video_status_h(video_ids=filtered_pre_result)
+        filtered_status_result = self.filter_video_status_h(video_ids=filtered_pre_result, return_count=return_count)
         et_status = time.time()
         log_.info('filter by video status: result = {}, execute time = {}ms'.format(
             filtered_status_result, (et_status - st_status) * 1000))

video_recall.py

@@ -14,7 +14,7 @@ config_ = set_config()
 
 class PoolRecall(object):
     """召回"""
-    def __init__(self, app_type, client_info=None, mid='', uid='', ab_code=''):
+    def __init__(self, app_type, client_info=None, mid='', uid='', ab_code='', return_count=-1):
         """
         初始化
         :param app_type: 产品标识 type-int
@@ -28,6 +28,7 @@ class PoolRecall(object):
         self.uid = uid
         self.ab_code = ab_code
         self.client_info = client_info
+        self.return_count = return_count
         self.redis_helper = RedisHelper()
 
     def copy_redis_zset_data(self, from_key_name, to_key_name):
@@ -54,7 +55,7 @@ class PoolRecall(object):
         now_date = datetime.today()
         h = datetime.now().hour
         now_dt = datetime.strftime(now_date, '%Y%m%d')
-        now_h_recall_key = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{now_dt}.{h}"
+        now_h_recall_key = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{self.return_count}.{now_dt}.{h}"
         if self.redis_helper.key_exists(key_name=now_h_recall_key):
             flag = self.copy_redis_zset_data(from_key_name=now_h_recall_key, to_key_name=h_recall_mid_key)
             if flag:
@@ -67,7 +68,7 @@ class PoolRecall(object):
             else:
                 redis_dt = now_dt
                 redis_h = h - 1
-            now_h_recall_key = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{redis_dt}.{redis_h}"
+            now_h_recall_key = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{self.return_count}.{redis_dt}.{redis_h}"
             flag = self.copy_redis_zset_data(from_key_name=now_h_recall_key, to_key_name=h_recall_mid_key)
             if flag:
                 value = {'date': redis_dt, 'h': redis_h}
@@ -97,7 +98,7 @@ class PoolRecall(object):
                 # return h_recall_mid_key
             elif (record_dt == now_dt and h-int(record_h) == 1) or (h == 0 and int(record_h) == 23):
                 # 记录的h - 当前h = 1，判断当前h数据是否已更新
-                now_h_recall_key = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{now_dt}.{h}"
+                now_h_recall_key = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{self.return_count}.{now_dt}.{h}"
                 # if not self.redis_helper.key_exists(key_name=now_h_recall_key):
                     # 未更新
                     # return h_recall_mid_key
@@ -155,7 +156,7 @@ class PoolRecall(object):
                     video_score[video_id] = value[1]
                 # 过滤
                 filter_ = FilterVideos(app_type=self.app_type, mid=self.mid, uid=self.uid, video_ids=video_ids)
-                ge = gevent.spawn(filter_.filter_videos_h)
+                ge = gevent.spawn(filter_.filter_videos_h, self.return_count)
                 ge.join()
                 filtered_result = ge.get()
 
@@ -475,7 +476,7 @@ class PoolRecall(object):
 
     def get_video_last_idx(self):
         """获取用户上一次在rov召回池对应的位置"""
-        if self.ab_code == config_.AB_CODE['rank_by_h'] or self.app_type == config_.APP_TYPE['APP']:
+        if self.ab_code in config_.AB_CODE['rank_by_h'] or self.app_type == config_.APP_TYPE['APP']:
             rov_pool_key, redis_date = self.get_pool_redis_key_with_h('rov')
         else:
             rov_pool_key, redis_date = self.get_pool_redis_key('rov')
@@ -651,7 +652,7 @@ class PoolRecall(object):
 
             else:
                 # 判断热度列表是否更新，未更新则使用前一小时的热度列表
-                key_name = f"{config_.RECALL_KEY_NAME_PREFIX_DUP_H}{now_date}.{h}"
+                key_name = f"{config_.RECALL_KEY_NAME_PREFIX_DUP_H}{self.return_count}.{now_date}.{h}"
                 if self.redis_helper.key_exists(key_name):
                     return key_name, h
                 else:
@@ -661,7 +662,7 @@ class PoolRecall(object):
                     else:
                         redis_h = h - 1
                         redis_date = now_date
-                    key_name = f"{config_.RECALL_KEY_NAME_PREFIX_DUP_H}{redis_date}.{redis_h}"
+                    key_name = f"{config_.RECALL_KEY_NAME_PREFIX_DUP_H}{self.return_count}.{redis_date}.{redis_h}"
                     # 判断当前时间是否晚于数据正常更新时间，发送消息到飞书
                     now_m = datetime.now().minute
                     feishu_text = '{} —— appType = {}, h = {} 数据未按时更新，请及时查看解决。'.format(