Merge branch 'drop_base_20230608' into test

recommend.py

@@ -1600,15 +1600,15 @@ def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
                              no_op_flag=no_op_flag, old_video_index=old_video_index,
                              params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
     # old base (test4, test5)
-    elif ab_code == 60050 or ab_code == 60051:
-        result = video_recommend(request_id=request_id,
-                                 mid=mid, uid=uid, app_type=app_type,
-                                 size=size, top_K=top_K, flow_pool_P=flow_pool_P,
-                                 algo_type=algo_type, client_info=client_info,
-                                 ab_code=ab_code, expire_time=expire_time,
-                                 rule_key=rule_key, data_key=data_key,
-                                 no_op_flag=no_op_flag, old_video_index=old_video_index,
-                                 params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
+    # elif ab_code == 60050 or ab_code == 60051:
+    #     result = video_recommend(request_id=request_id,
+    #                              mid=mid, uid=uid, app_type=app_type,
+    #                              size=size, top_K=top_K, flow_pool_P=flow_pool_P,
+    #                              algo_type=algo_type, client_info=client_info,
+    #                              ab_code=ab_code, expire_time=expire_time,
+    #                              rule_key=rule_key, data_key=data_key,
+    #                              no_op_flag=no_op_flag, old_video_index=old_video_index,
+    #                              params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
     # 60052,60053,60057(test7, test8, test6)
     # simrecal: 60054
     # 3 days , 7days recall: 60005, test1
@@ -1653,8 +1653,8 @@ def video_homepage_recommend(request_id, mid, uid, size, app_type, algo_type,
     update_redis_st = time.time()
     if ab_code == 60047 or  ab_code == 60048 or  ab_code == 60049:
         update_flow_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
-    elif ab_code == 60050 or  ab_code == 60051:
-        update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
+    # elif ab_code == 60050 or  ab_code == 60051:
+    #     update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
     elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 \
             or ab_code == 60056 or ab_code==60057:
         update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
@@ -1731,15 +1731,15 @@ def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_
                                  old_video_index=old_video_index, video_id=video_id,
                                  params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
         # log_.info({
-    elif ab_code == 60050 or ab_code == 60051:
-        result = video_recommend(request_id=request_id,
-                                 mid=mid, uid=uid, app_type=app_type,
-                                 size=size, top_K=top_K, flow_pool_P=flow_pool_P,
-                                 algo_type='', client_info=client_info,
-                                 ab_code=ab_code, expire_time=expire_time,
-                                 rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
-                                 old_video_index=old_video_index, video_id=video_id,
-                                 params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
+    # elif ab_code == 60050 or ab_code == 60051:
+    #     result = video_recommend(request_id=request_id,
+    #                              mid=mid, uid=uid, app_type=app_type,
+    #                              size=size, top_K=top_K, flow_pool_P=flow_pool_P,
+    #                              algo_type='', client_info=client_info,
+    #                              ab_code=ab_code, expire_time=expire_time,
+    #                              rule_key=rule_key, data_key=data_key, no_op_flag=no_op_flag,
+    #                              old_video_index=old_video_index, video_id=video_id,
+    #                              params=params, rule_key_30day=rule_key_30day, shield_config=shield_config)
     elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
             ab_code == 60056 or ab_code==60057:
         result = video_old_recommend(request_id=request_id,
@@ -1782,8 +1782,8 @@ def video_relevant_recommend(request_id, video_id, mid, uid, size, app_type, ab_
     update_redis_st = time.time()
     if ab_code == 60047 or ab_code == 60048 or  ab_code == 60049:
          update_flow_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
-    elif ab_code == 60050 or ab_code == 60051:
-         update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
+    # elif ab_code == 60050 or ab_code == 60051:
+    #      update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)
     elif ab_code == 60052 or ab_code == 60053 or ab_code == 60054 or ab_code == 60055 or \
             ab_code == 60056 or ab_code==60057:
          update_redis_data(result=rank_result, app_type=app_type, mid=mid, top_K=top_K)

utils.py

@@ -665,6 +665,122 @@ class FilterVideos(object):
                     return filtered_viewed_videos
 
 
+
+    def filter_videos_status(self, pool_type='rov', region_code=None, shield_config=None):
+        """视频过滤"""
+        # 预曝光过滤
+        st_pre = time.time()
+        filtered_pre_result = self.filter_video_previewed(self.video_ids)
+        # print("filtered_pre:", (time.time()-st_pre)*1000)
+        # et_pre = time.time()
+        # log_.info({
+        #     'logTimestamp': int(time.time() * 1000),
+        #     'request_id': self.request_id,
+        #     'app_type': self.app_type,
+        #     'mid': self.mid,
+        #     'uid': self.uid,
+        #     'operation': 'preview_filter',
+        #     'request_videos': self.video_ids,
+        #     'preview_filter_result': filtered_pre_result,
+        #     'executeTime': (time.time() - st_pre) * 1000
+        # })
+        if not filtered_pre_result:
+            return None
+
+        # 视频状态过滤采用离线定时过滤方案
+        # 视频状态过滤
+        # st_status = time.time()
+        # filtered_status_result = self.filter_video_status(video_ids=filtered_pre_result)
+        # et_status = time.time()
+        # log_.info('filter by video status: result = {}, execute time = {}ms'.format(
+        #     filtered_status_result, (et_status - st_status) * 1000))
+        # if not filtered_status_result:
+        #     return None
+
+        # 视频已曝光过滤
+        st_viewed = time.time()
+        filtered_viewed_result = self.filter_video_viewed_status(video_ids=filtered_pre_result)
+        # print("filtered_pre:", (time.time() - st_viewed) * 1000)
+        # et_viewed = time.time()
+        # log_.info({
+        #     'logTimestamp': int(time.time() * 1000),
+        #     'pool_type': pool_type,
+        #     'request_id': self.request_id,
+        #     'app_type': self.app_type,
+        #     'mid': self.mid,
+        #     'uid': self.uid,
+        #     'operation': 'view_filter',
+        #     'request_videos': filtered_pre_result,
+        #     'view_filter_result': filtered_viewed_result,
+        #     'executeTime': (time.time() - st_viewed) * 1000
+        # })
+        if not filtered_viewed_result:
+            return None
+        filtered_viewed_videos = [int(video_id) for video_id in filtered_viewed_result]
+        if pool_type == 'flow' or pool_type=='normal':
+            # 流量池视频需过滤屏蔽视频
+            if region_code is None or shield_config is None:
+                return filtered_viewed_videos
+            else:
+                shield_key_name_list = shield_config.get(region_code, None)
+                if shield_key_name_list is not None:
+                    filtered_shield_video_ids = self.filter_shield_video(
+                        video_ids=filtered_viewed_videos, shield_key_name_list=shield_key_name_list
+                    )
+                    log_.info({
+                        'logTimestamp': int(time.time() * 1000),
+                        'pool_type': pool_type,
+                        'request_id': self.request_id,
+                        'app_type': self.app_type,
+                        'mid': self.mid,
+                        'uid': self.uid,
+                        'operation': 'shield_filter',
+                        'request_videos': filtered_viewed_videos,
+                        'shield_filter_result': filtered_shield_video_ids,
+                        'executeTime': (time.time() - st_viewed) * 1000
+                    })
+                    # print("filtered_pre flow:", (time.time() - st_viewed) * 1000)
+                    return filtered_shield_video_ids
+                else:
+                    return filtered_viewed_videos
+        else:
+            return filtered_viewed_videos
+    def filter_video_viewed_status(self, video_ids, types=(1, 6,)):
+        """
+                   调用后端接口过滤用户已观看视频
+                   :param video_ids: 视频id列表 type-list
+                   :param types: 过滤参数 type-tuple, 默认(1, )
+                   1-已观看 2-视频状态 3-是否进入老年人社区 4-话题状态 5-推荐状态 6-白名单过滤 7-涉政视频过滤
+                   :return: filtered_videos
+                   """
+        # 获取对应端的过滤参数types
+        types = config_.FILTER_VIEWED_TYPES_CONFIG.get(self.app_type, None)
+        if types is None:
+            types = config_.FILTER_VIEWED_TYPES_CONFIG.get('other')
+        types = list(types)
+        types.append(2)
+        request_data = {"appType": self.app_type,
+                        "mid": self.mid,
+                        "uid": self.uid,
+                        "types": list(types),
+                        "videoIds": video_ids}
+        # print(request_data)
+        # 调用http接口
+        result = request_post(request_url=config_.VIDEO_FILTER_URL, request_data=request_data, timeout=(0.1, 1))
+
+        # print("result:", result)
+        if result is None:
+            # print("result is None")
+            # log_.info('过滤失败，types: {}'.format(types))
+            return []
+
+        if result['code'] != 0:
+            # log_.info('过滤失败，types: {}'.format(types))
+            return []
+
+        filtered_videos = result['data']
+        return filtered_videos
+
 if __name__ == '__main__':
     user = [
         ('weixin_openid_o0w175fDc8pNnywrYN49E341tKfI', ''),

video_recall.py

@@ -2177,7 +2177,7 @@ class PoolRecall(object):
         #print(video_ids)
         filter_ = FilterVideos(request_id=self.request_id,
                                app_type=self.app_type, mid=self.mid, uid=self.uid, video_ids=video_ids)
-        filtered_viewed_videos = filter_.filter_videos(pool_type='normal')
+        filtered_viewed_videos = filter_.filter_videos_status(pool_type='normal')
         if filtered_viewed_videos is None:
             return recall_result
         print("filtered_viewed_videos:", filtered_viewed_videos)
@@ -2532,7 +2532,7 @@ class PoolRecall(object):
         #print(video_ids)
         filter_ = FilterVideos(request_id=self.request_id,
                                app_type=self.app_type, mid=self.mid, uid=self.uid, video_ids=video_ids)
-        filtered_viewed_videos = filter_.filter_videos(pool_type='normal')
+        filtered_viewed_videos = filter_.filter_videos_status(pool_type='normal')
         if filtered_viewed_videos is None:
             return recall_result
         #print("filtered_viewed_videos:", filtered_viewed_videos)
@@ -2706,7 +2706,7 @@ class PoolRecall(object):
         #print(video_ids)
         filter_ = FilterVideos(request_id=self.request_id,
                                app_type=self.app_type, mid=self.mid, uid=self.uid, video_ids=video_ids)
-        filtered_viewed_videos = filter_.filter_videos(pool_type='normal')
+        filtered_viewed_videos = filter_.filter_videos_status(pool_type='normal')
         if filtered_viewed_videos is None:
             return recall_result
         #print("filtered_viewed_videos:", filtered_viewed_videos)