debug

config.py

@@ -47,6 +47,9 @@ class DevelopmentConfig(BaseConfig):
     # 参数types: 1-已观看 2-视频状态 3-是否进入老年人社区过滤 4-话题状态
     VIDEO_FILTER_URL = 'http://videotest-internal.yishihui.com/longvideoapi/openapi/recommend/filterVideos'
 
+    # 获取视频在流量池中的剩余可分发数接口地址
+    GET_REMAIN_VIEW_COUNT_URL = 'http://testapi-internal.piaoquantv.com/flowpool/video/remainViewCount'
+
 
 class TestConfig(BaseConfig):
     """测试环境配置"""
@@ -66,6 +69,77 @@ class TestConfig(BaseConfig):
         'password': '4BEcOgxREOPq7t3A7EWkjciVULeQGj'
     }
 
+    # Hologres视频状态存储表名
+    VIDEO_STATUS = 'longvideo_test.dwd_mdm_item_video_stat'
+
+    # 用户已观看视频过滤 & 视频审核条件过滤 & 是否进入老年人社区过滤 & 话题状态过滤 接口地址
+    # 参数types: 1-已观看 2-视频状态 3-是否进入老年人社区过滤 4-话题状态
+    VIDEO_FILTER_URL = 'http://videotest-internal.yishihui.com/longvideoapi/openapi/recommend/filterVideos'
+
+    # 获取视频在流量池中的剩余可分发数接口地址
+    GET_REMAIN_VIEW_COUNT_URL = 'http://testapi-internal.piaoquantv.com/flowpool/video/remainViewCount'
+
+
+class PreProductionConfig(BaseConfig):
+    """测试环境配置"""
+    # 线上环境redis地址
+    REDIS_INFO = {
+        'host': 'r-bp1fogs2mflr1ybfot.redis.rds.aliyuncs.com',
+        'port': 6379,
+        'password': 'Wqsd@2019',
+    }
+
+    # Hologres连接参数，服务器使用
+    HOLOGRES_INFO = {
+        'host': 'hgprecn-cn-7pp28y18c00c-cn-hangzhou-vpc.hologres.aliyuncs.com',
+        'port': 80,
+        'dbname': 'dssm',
+        'user': 'LTAI5tMPqPy9yboQAf1mBCCN',
+        'password': '4BEcOgxREOPq7t3A7EWkjciVULeQGj'
+    }
+
+    # Hologres视频状态存储表名
+    VIDEO_STATUS = 'longvideo.dwd_mdm_item_video_stat'
+
+    # 用户已观看视频过滤 & 视频审核条件过滤 & 是否进入老年人社区过滤 & 话题状态过滤 接口地址
+    # 参数types: 1-已观看 2-视频状态 3-是否进入老年人社区过滤 4-话题状态
+    VIDEO_FILTER_URL = 'http://speedpre.wx.com/longvideoapi/openapi/recommend/filterVideos'
+
+    # 获取视频在流量池中的剩余可分发数接口地址
+    GET_REMAIN_VIEW_COUNT_URL = 'http://preapi-internal.piaoquantv.com/flowpool/video/remainViewCount'
+
+
+class ProductionConfig(BaseConfig):
+    """测试环境配置"""
+    # 线上环境redis地址
+    REDIS_INFO = {
+        'host': 'r-bp1fogs2mflr1ybfot.redis.rds.aliyuncs.com',
+        'port': 6379,
+        'password': 'Wqsd@2019',
+    }
+
+    # Hologres连接参数，服务器使用
+    HOLOGRES_INFO = {
+        'host': 'hgprecn-cn-7pp28y18c00c-cn-hangzhou-vpc.hologres.aliyuncs.com',
+        'port': 80,
+        'dbname': 'dssm',
+        'user': 'LTAI5tMPqPy9yboQAf1mBCCN',
+        'password': '4BEcOgxREOPq7t3A7EWkjciVULeQGj'
+    }
+
+    # Hologres视频状态存储表名
+    VIDEO_STATUS = 'longvideo.dwd_mdm_item_video_stat'
+
+    # 用户已观看视频过滤 & 视频审核条件过滤 & 是否进入老年人社区过滤 & 话题状态过滤 接口地址
+    # 参数types: 1-已观看 2-视频状态 3-是否进入老年人社区过滤 4-话题状态
+    VIDEO_FILTER_URL = 'http://recommend-common-internal.piaoquantv.com/longvideoapi/openapi/recommend/filterVideos'
+
+    # 获取视频在流量池中的剩余可分发数接口地址
+    GET_REMAIN_VIEW_COUNT_URL = 'http://api-internal.piaoquantv.com/flowpool/video/remainViewCount'
+
 
 def set_config():
     return DevelopmentConfig()
+    # return TestConfig()
+    # return PreProductionConfig()
+    # return ProductionConfig()

db_helper.py

@@ -167,7 +167,7 @@ class RedisHelper(object):
         """
         新增数据，set
         :param key_name: key
-        :param values: 要添加的元素  类型-set
+        :param values: 要添加的元素  类型-tuple
         :param expire_time: 过期时间，单位：s，默认0.5小时
         :return: None
         """
@@ -190,7 +190,7 @@ class RedisHelper(object):
         """
         删除set中的指定元素
         :param key_name: key
-        :param values: 元素的值, 类型-set
+        :param values: 元素的值, 类型-tuple
         :return: None
         """
         conn = self.connect()

recommend.py

@@ -70,10 +70,12 @@ def video_recommend(mid, uid, size, app_type, algo_type):
     preview_key_name = config_.PREVIEW_KEY_PREFIX + '{}.{}'.format(app_type, mid)
     preview_video_ids = [item['videoId'] for item in rank_result]
     if preview_video_ids:
-        redis_helper.add_data_with_set(key_name=preview_key_name, values=set(preview_video_ids), expire_time=0.5*3600)
+        redis_helper.add_data_with_set(key_name=preview_key_name, values=tuple(preview_video_ids), expire_time=30*60)
+        log_.info('preview redis update success!')
     # 将此次获取的ROV召回池末位视频id同步刷新到Redis中，方便下次快速定位到召回位置，过期时间为1天
     rov_recall_video = [item['videoId'] for item in rank_result if item['pushFrom'] == 'recall_pool']
     if rov_recall_video:
         redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=rov_recall_video[-1])
+        log_.info('last video redis update success!')
 
     return rank_result

video_rank.py

@@ -27,7 +27,6 @@ def video_rank(data, size):
     flow_recall_rank = sorted(data['flow_pool_recall'], key=lambda k: (k.get('rovScore'), 0), reverse=True)
     # 对各路召回的视频进行去重
     rov_recall_rank, flow_recall_rank = remove_duplicate(rov_recall=rov_recall_rank, flow_recall=flow_recall_rank)
-
     # 从ROV召回池中获取top k
     if len(rov_recall_rank) > 0:
         rank_result = rov_recall_rank[:config_.K]
@@ -46,14 +45,14 @@ def video_rank(data, size):
                 rank_result.append(flow_recall_rank[0])
                 flow_recall_rank.remove(flow_recall_rank[0])
             else:
-                rank_result.append(rov_recall_rank[:size - config_.K - i])
+                rank_result.extend(rov_recall_rank[:size - config_.K - i])
                 return rank_result
         else:
             if rov_recall_rank:
                 rank_result.append(rov_recall_rank[0])
                 rov_recall_rank.remove(rov_recall_rank[0])
             else:
-                rank_result.append(flow_recall_rank[:size - config_.K - i])
+                rank_result.extend(flow_recall_rank[:size - config_.K - i])
                 return rank_result
         i += 1
     return rank_result
@@ -68,21 +67,27 @@ def remove_duplicate(rov_recall, flow_recall):
     :param flow_recall: 流量池-已排序
     :return:
     """
-    flow_recall_remove = []
-    flow_recall_video_ids = [item[0] for item in flow_recall]
+    flow_recall_result = []
+    rov_recall_remove = []
+    flow_recall_video_ids = [item['videoId'] for item in flow_recall]
     # rov_recall topK
     for item in rov_recall[:config_.K]:
-        if item[0] in flow_recall_video_ids:
-            flow_recall_remove.append(item[0])
+        if item['videoId'] in flow_recall_video_ids:
+            flow_recall_video_ids.remove(item['videoId'])
     # other
     for item in rov_recall[config_.K:]:
-        if item[0] in flow_recall_video_ids:
-            rov_recall.remove(item)
+        if item['videoId'] in flow_recall_video_ids:
+            rov_recall_remove.append(item)
+
+    # rov recall remove
+    for item in rov_recall_remove:
+        rov_recall.remove(item)
     # flow recall remove
     for item in flow_recall:
-        if item[0] in flow_recall_remove:
-            flow_recall.remove(item)
-    return rov_recall, flow_recall
+        if item['videoId'] not in flow_recall_video_ids:
+            flow_recall_result.append(item)
+
+    return rov_recall, flow_recall_result
 
 
 def bottom_strategy(size, app_type, ab_code):
@@ -108,6 +113,9 @@ def bottom_strategy(size, app_type, ab_code):
     # 状态过滤
     filter_videos = FilterVideos(app_type=app_type, video_ids=data)
     filtered_data = filter_videos.filter_video_status(video_ids=data)
-    random_data = numpy.random.choice(filtered_data, size, False)
+    if len(filtered_data) > size:
+        random_data = numpy.random.choice(filtered_data, size, False)
+    else:
+        random_data = filtered_data
     bottom_data = [{'videoId': item, 'pushFrom': 'bottom_strategy', 'abCode': ab_code} for item in random_data]
     return bottom_data

video_recall.py

@@ -71,7 +71,7 @@ class PoolRecall(object):
                 rov_pool_recall_result.extend(temp_result)
             else:
                 # 将此次获取的末位视频id同步刷新到Redis中，方便下次快速定位到召回位置，过期时间为1天
-                self.redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=data[-1])
+                self.redis_helper.set_data_to_redis(key_name=last_rov_recall_key, value=data[-1][0])
             idx += get_size
         return rov_pool_recall_result[:size]