add h_ab test

config.py

@@ -20,6 +20,7 @@ class BaseConfig(object):
         'w_h_rate': [APP_TYPE['LONG_VIDEO']],  # 视频宽高比实验(每组的前两个视频调整为横屏视频), 已下线
         'position_insert': [APP_TYPE['SHORT_VIDEO'], APP_TYPE['LOVE_LIVE']],
         'relevant_video_op': [APP_TYPE['LONG_VIDEO']],  # 相关推荐强插实验(运营对某些视频给定一些相关视频，调整为对应视频相关推荐的头部)
+        'rank_by_h': [APP_TYPE['VLOG']],  # 小时级别更新rov列表
     }
     # abCode
     AB_CODE = {
@@ -27,6 +28,7 @@ class BaseConfig(object):
         'w_h_rate': 10001,  # 视频宽高比实验(每组的前两个视频调整为横屏视频)，已下线
         'position_insert': 10002,  # 按位置插入
         'relevant_video_op': 10003,  # 运营对某些视频给定一些相关视频，调整为对应视频相关推荐的头部
+        'rank_by_h': 20001,  # 小时级别更新rov列表实验
     }
 
     # pushFrom
@@ -39,6 +41,7 @@ class BaseConfig(object):
         'position_insert': 'position_insert',  # 按位置插入
         'relevant_video_op': 'relevant_video_op',  # 相关推荐强插
     }
+
     # category id mapping
     CATEGORY = {
         'recommend': [55],  # 推荐
@@ -55,8 +58,8 @@ class BaseConfig(object):
     # app应用 小程序离线ROV模型结果存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.app.{date}
     RECALL_KEY_NAME_PREFIX_APP = 'com.weiqu.video.recall.hot.item.score.app.'
 
-    # appType = 6, ROV召回池redis key前缀，完整格式：com.weiqu.video.recall.hot.apptype.h.item.score.{appType}.{h}
-    RECALL_KEY_NAME_PREFIX_APP_TYPE = 'com.weiqu.video.recall.hot.apptype.h.item.score.'
+    # appType = 6, ROV召回池redis key前缀，完整格式：com.weiqu.video.recall.hot.apptype.h.item.score.6.{h}
+    RECALL_KEY_NAME_PREFIX_APP_TYPE = 'com.weiqu.video.recall.hot.apptype.h.item.score.6.'
 
     # 流量池redis key前缀，完整格式 com.weiqu.video.flowpool.hot.item.score.{appType}
     FLOW_POOL_KEY_NAME_PREFIX = 'com.weiqu.video.flowpool.hot.item.score.'
@@ -308,8 +311,8 @@ class ProductionConfig(BaseConfig):
 
 def set_config():
     # 获取环境变量 ROV_SERVER_ENV
-    env = os.environ.get('ROV_SERVER_ENV')
-    # env = 'dev'
+    # env = os.environ.get('ROV_SERVER_ENV')
+    env = 'dev'
     if env is None:
         # log_.error('ENV ERROR: is None!')
         return

db_helper.py

@@ -254,7 +254,9 @@ class RedisHelper(object):
 #conn = psycopg2.connect(**hologres_info)
 #cur = conn.cursor()
 
-connectPool = pgpool.SimpleConnectionPool(1, 30, **config_.HOLOGRES_INFO)
+
+# holo连接
+# connectPool = pgpool.SimpleConnectionPool(1, 30, **config_.HOLOGRES_INFO)
 
 
 class HologresHelper(object):

recommend.py

@@ -134,7 +134,8 @@ def positon_duplicate(pos1_vids, pos2_vids, videos):
     return l
 
 
-def video_recommend(mid, uid, size, app_type, algo_type, client_info):
+def video_recommend(mid, uid, size, app_type, algo_type, client_info, expire_time=24*3600,
+                    ab_code=config_.AB_CODE['initial']):
     """
     首页线上推荐逻辑
     :param mid: mid type-string
@@ -143,9 +144,10 @@ def video_recommend(mid, uid, size, app_type, algo_type, client_info):
     :param app_type: 产品标识  type-int
     :param algo_type: 算法类型  type-string
     :param client_info: 用户位置信息 {"country": "国家",  "province": "省份",  "city": "城市"}
+    :param expire_time: 末位视频记录redis过期时间
+    :param ab_code: AB实验code
     :return:
     """
-    ab_code = config_.AB_CODE['initial']
     # ####### 多进程召回
     start_recall = time.time()
     # log_.info('====== recall')
@@ -165,9 +167,10 @@ def video_recommend(mid, uid, size, app_type, algo_type, client_info):
     pool.join()
     '''
     recall_result_list = []
-    pool_recall = PoolRecall(app_type=app_type, mid=mid, uid=uid, ab_code=ab_code, client_info=client_info)
+    pool_recall = PoolRecall(app_type=app_type, mid=mid, uid=uid, ab_code=ab_code,
+                             client_info=client_info)
     _, last_rov_recall_key, _ = pool_recall.get_video_last_idx()
-    t = [gevent.spawn(pool_recall.rov_pool_recall, size), gevent.spawn(pool_recall.flow_pool_recall, size)]
+    t = [gevent.spawn(pool_recall.rov_pool_recall, size, expire_time), gevent.spawn(pool_recall.flow_pool_recall, size)]
     gevent.joinall(t)
     recall_result_list = [i.get() for i in t]
 
@@ -239,13 +242,14 @@ def ab_test_op(rank_result, ab_code_list, app_type, mid, uid, **kwargs):
     return rank_result
 
 
-def update_redis_data(result, app_type, mid, last_rov_recall_key):
+def update_redis_data(result, app_type, mid, last_rov_recall_key, expire_time=24*3600):
     """
     根据最终的排序结果更新相关redis数据
     :param result: 排序结果
     :param app_type: 产品标识
     :param mid: mid
     :param last_rov_recall_key: 用户上一次在rov召回池对应的位置 redis key
+    :param expire_time: 末位视频记录redis过期时间
     :return: None
     """
     # ####### redis数据刷新
@@ -269,7 +273,8 @@ def update_redis_data(result, app_type, mid, last_rov_recall_key):
             else:
                 key_name = config_.UPDATE_ROV_KEY_NAME
             if not redis_helper.get_score_with_value(key_name=key_name, value=rov_recall_video[-1]):
-                redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=rov_recall_video[-1])
+                redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=rov_recall_video[-1],
+                                               expire_time=expire_time)
             log_.info('last video redis update success!')
 
         # 将此次分发的流量池视频，对 本地分发数-1 进行记录
@@ -320,15 +325,34 @@ def video_homepage_recommend(mid, uid, size, app_type, algo_type, client_info):
     :param client_info: 用户位置信息 {"country": "国家",  "province": "省份",  "city": "城市"}
     :return:
     """
-    # 简单召回 - 排序 - 兜底
-    rank_result, last_rov_recall_key = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
-                                                       algo_type=algo_type, client_info=client_info)
-    # ab-test
-    result = ab_test_op(rank_result=rank_result,
-                        ab_code_list=[config_.AB_CODE['position_insert']],
-                        app_type=app_type, mid=mid, uid=uid)
-    # redis数据刷新
-    update_redis_data(result=result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key)
+    # 对 vlog 切换10%的流量做实验
+    # 对mid进行哈希
+    print(hash(mid))
+    print(abs(hash(mid)) % 10)
+    if app_type in config_.AB_TEST['rank_by_h'] and abs(hash(mid)) % 10 in [0, 1, 7, 8, 4, ]:
+        print('in')
+        # 简单召回 - 排序 - 兜底
+        rank_result, last_rov_recall_key = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
+                                                           algo_type=algo_type, client_info=client_info,
+                                                           expire_time=3600,
+                                                           ab_code=config_.AB_CODE['rank_by_h'])
+        # ab-test
+        result = ab_test_op(rank_result=rank_result,
+                            ab_code_list=[config_.AB_CODE['position_insert']],
+                            app_type=app_type, mid=mid, uid=uid)
+        # redis数据刷新
+        update_redis_data(result=result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key,
+                          expire_time=3600)
+    else:
+        # 简单召回 - 排序 - 兜底
+        rank_result, last_rov_recall_key = video_recommend(mid=mid, uid=uid, size=size, app_type=app_type,
+                                                           algo_type=algo_type, client_info=client_info)
+        # ab-test
+        result = ab_test_op(rank_result=rank_result,
+                            ab_code_list=[config_.AB_CODE['position_insert']],
+                            app_type=app_type, mid=mid, uid=uid)
+        # redis数据刷新
+        update_redis_data(result=result, app_type=app_type, mid=mid, last_rov_recall_key=last_rov_recall_key)
 
     return result
 

video_recall.py

@@ -30,8 +30,13 @@ class PoolRecall(object):
         self.client_info = client_info
         self.redis_helper = RedisHelper()
 
-    def rov_pool_recall(self, size=10):
-        """从ROV召回池中获取视频"""
+    def rov_pool_recall(self, size=10, expire_time=24*3600):
+        """
+        从ROV召回池中获取视频
+        :param size: 获取视频个数
+        :param expire_time: 末位视频记录redis过期时间
+        :return:
+        """
         # log_.info('====== rov pool recall')
 
         # 获取生效中的置顶视频
@@ -104,7 +109,8 @@ class PoolRecall(object):
                 rov_pool_recall_result.extend(temp_result)
             else:
                 # 将此次获取的末位视频id同步刷新到Redis中，方便下次快速定位到召回位置，过期时间为1天
-                self.redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=data[-1][0])
+                self.redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=data[-1][0],
+                                                    expire_time=expire_time)
             idx += get_size
 
         # 生效中的置顶视频、被修改rov视频、rov召回池视频 归并排序
@@ -271,60 +277,6 @@ class PoolRecall(object):
         :return: key_name
         """
         if pool_type == 'rov':
-            # appType = 6也使用统一的ROV列表视频
-            """
-            # appType = 6
-            if self.app_type == config_.APP_TYPE['SHORT_VIDEO']:
-                # 获取当前所在小时
-                h = datetime.now().hour
-                # 判断热度列表是否更新，未更新则使用前一小时的热度列表
-                key_name = '{}{}.{}'.format(config_.RECALL_KEY_NAME_PREFIX_APP_TYPE, self.app_type, h)
-                if self.redis_helper.key_exists(key_name):
-                    return key_name, h
-                else:
-                    if h == 0:
-                        redis_h = 23
-                    else:
-                        redis_h = h - 1
-                    key_name = '{}{}.{}'.format(config_.RECALL_KEY_NAME_PREFIX_APP_TYPE, self.app_type, redis_h)
-                    # 判断当前时间是否晚于数据正常更新时间，发送消息到飞书
-                    now_m = datetime.now().minute
-                    feishu_text = '{} —— appType = {}, h = {} 数据未按时更新，请及时查看解决。'.format(
-                        config_.ENV_TEXT, self.app_type, h)
-                    if now_m > config_.ROV_UPDATE_MINUTE_6 and h > 0:
-                        # h=0时，因数据首次更新耗时长，不做报警
-                        send_msg_to_feishu(feishu_text)
-                    return key_name, redis_h
-                # 其他
-            else:
-                # appType = 13  票圈视频app
-                if self.app_type == config_.APP_TYPE['APP']:
-                    key_name_prefix = config_.RECALL_KEY_NAME_PREFIX_APP
-                # 其他
-                else:
-                    key_name_prefix = config_.RECALL_KEY_NAME_PREFIX
-
-                # 判断热度列表是否更新，未更新则使用前一天的热度列表
-                key_name = key_name_prefix + time.strftime('%Y%m%d')
-                if self.redis_helper.key_exists(key_name):
-                    redis_date = date.today().strftime('%Y%m%d')
-                else:
-                    redis_date = (date.today() - timedelta(days=1)).strftime('%Y%m%d')
-                    key_name = key_name_prefix + redis_date
-                    # if not self.redis_helper.key_exists(key_name):
-                    #     return None, None
-
-                    # 判断当前时间是否晚于rov召回池更新时间 + 1h，发送消息到飞书
-                    now_h = datetime.now().hour
-                    now_m = datetime.now().minute
-                    feishu_text = '{} —— 今日ROV召回池数据未按时更新，请及时查看解决。'.format(config_.ENV_TEXT)
-                    if now_h == config_.ROV_UPDATE_H + 1 and now_m > config_.ROV_UPDATE_MINUTE:
-                        send_msg_to_feishu(feishu_text)
-                    elif now_h > config_.ROV_UPDATE_H + 2:
-                        send_msg_to_feishu(feishu_text)
-
-                return key_name, redis_date
-            """
 
             # appType = 13  票圈视频app
             if self.app_type == config_.APP_TYPE['APP']:
@@ -363,7 +315,11 @@ class PoolRecall(object):
 
     def get_video_last_idx(self):
         """获取用户上一次在rov召回池对应的位置"""
-        rov_pool_key, redis_date = self.get_pool_redis_key('rov')
+        if self.ab_code == config_.AB_CODE['rank_by_h']:
+            rov_pool_key, redis_date = self.get_pool_redis_key_with_h('rov')
+            print('in {}, {}'.format(rov_pool_key, redis_date))
+        else:
+            rov_pool_key, redis_date = self.get_pool_redis_key('rov')
         if not rov_pool_key:
             return None, None, None
         last_rov_recall_key = config_.LAST_VIDEO_FROM_ROV_POOL_PREFIX + '{}.{}.{}'.format(
@@ -491,3 +447,38 @@ class PoolRecall(object):
         except Exception as e:
             log_.error(traceback.format_exc())
             return [], []
+
+    def get_pool_redis_key_with_h(self, pool_type):
+        """
+        拼接key，获取以小时级别更新的视频列表
+        :param pool_type: type-string {'rov': rov召回池, 'flow': 流量池}
+        :return: key_name
+        """
+        if pool_type == 'rov':
+            # 获取当前所在小时
+            h = datetime.now().hour
+            # 判断热度列表是否更新，未更新则使用前一小时的热度列表
+            key_name = '{}{}'.format(config_.RECALL_KEY_NAME_PREFIX_APP_TYPE, h)
+            if self.redis_helper.key_exists(key_name):
+                return key_name, h
+            else:
+                if h == 0:
+                    redis_h = 23
+                else:
+                    redis_h = h - 1
+                key_name = '{}{}'.format(config_.RECALL_KEY_NAME_PREFIX_APP_TYPE, redis_h)
+                # 判断当前时间是否晚于数据正常更新时间，发送消息到飞书
+                now_m = datetime.now().minute
+                feishu_text = '{} —— appType = {}, h = {} 数据未按时更新，请及时查看解决。'.format(
+                    config_.ENV_TEXT, self.app_type, h)
+                if now_m > config_.ROV_UPDATE_MINUTE_6 and h > 0:
+                    # h=0时，因数据首次更新耗时长，不做报警
+                    send_msg_to_feishu(feishu_text)
+                return key_name, redis_h
+
+        elif pool_type == 'flow':
+            return config_.FLOW_POOL_KEY_NAME_PREFIX + str(self.app_type)
+
+        else:
+            log_.error('pool type error')
+            return None, None