update time

recommend.py

@@ -381,7 +381,7 @@ def new_video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
     if region_code == '':
         region_code = '-1'
     
-    #print("region_code:", region_code)
+    print("region_code:", region_code)
 
     #size =1000
     pool_recall = PoolRecall(request_id=request_id,
@@ -411,7 +411,7 @@ def new_video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
     all_recall_result = []
     #print(all_recall_result_list)
     result['recallTime'] = (time.time() - start_recall) * 1000
-
+    print("recall time:", result['recallTime'])
     if not all_recall_result_list or len(all_recall_result_list)==0:
         return result
     for recall_item in all_recall_result_list:
@@ -422,6 +422,7 @@ def new_video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
 
     #print("all_recall_result:", all_recall_result)
     #2. duplicate
+    dup_time = time.time()
     recall_dict = {}
     fast_flow_set = set('')
     flow_flow_set = set('')
@@ -470,15 +471,18 @@ def new_video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
         except:
             continue
     #print("recall_dict:", recall_dict)
+    print("recall time:", (time.time()-dup_time)*1000)
     #3. filter video, 先过预曝光
+    filter_time = time.time()
     filter_ = FilterVideos(request_id=request_id,
                            app_type=app_type, mid=mid, uid=uid, video_ids=list(recall_dict.keys()))
 
     #print("filer:", list(recall_dict.keys()))
     #a).expose filter
     #all_recall_list = list(recall_dict.keys())
-    all_recall_list = filter_.filter_videos_new(pool_type='rov', region_code=region_code, shield_config=shield_config)
+    all_recall_list = filter_.filter_videos_new(region_code=region_code, shield_config=shield_config, flow_set=flowFlag_dict.keys())
     #print("filer after:", all_recall_list)
+    print("filter_ time:", (time.time() - filter_time) * 1000)
     #4. sort: old sort: flow 按概率出
     start_rank = time.time()
     #quick_flow_pool_P get from redis
@@ -512,6 +516,7 @@ def new_video_recommend(request_id, mid, uid, size, top_K, flow_pool_P, app_type
 
     result['rankResult'] = rank_result
     result['rankTime'] = (time.time() - start_rank) * 1000
+    print("rank time:", rankTime)
     return result
     # return rank_result, last_rov_recall_key
 

utils.py

@@ -609,7 +609,7 @@ class FilterVideos(object):
             #print(f"m_r res: {video_ids}\nexecute_time: {(time.time() - st_time) * 1000}")
             return video_ids
 
-    def filter_videos_new(self, pool_type='rov', region_code=None, shield_config=None):
+    def filter_videos_new(self, region_code=None, shield_config=None, flow_set=None):
         """视频过滤"""
         # 预曝光过滤
         st_pre = time.time()
@@ -641,31 +641,26 @@ class FilterVideos(object):
             return None
         filtered_viewed_videos = [int(video_id) for video_id in filtered_viewed_result]
         #print("result:", filtered_viewed_videos)
-        if pool_type != 'flow':
+        if flow_set is None:
             return  filtered_viewed_videos
         else:
             # 流量池视频需过滤屏蔽视频
             if region_code is None or shield_config is None:
                 return filtered_viewed_videos
             else:
+                normal_recall_ids = []
+                left_flow_ids = []
+                for vid in filtered_viewed_videos:
+                    if vid in flow_set:
+                        left_flow_ids.append(vid)
+                    else:
+                        normal_recall_ids.append(vid)
                 shield_key_name_list = shield_config.get(region_code, None)
                 if shield_key_name_list is not None:
                     filtered_shield_video_ids = self.filter_shield_video(
-                       video_ids=filtered_viewed_videos, shield_key_name_list=shield_key_name_list
+                       video_ids=left_flow_ids, shield_key_name_list=shield_key_name_list
                     )
-                    # log_.info({
-                    #     'logTimestamp': int(time.time() * 1000),
-                    #     'pool_type': pool_type,
-                    #     'request_id': self.request_id,
-                    #     'app_type': self.app_type,
-                    #     'mid': self.mid,
-                    #     'uid': self.uid,
-                    #     'operation': 'shield_filter',
-                    #     'request_videos': filtered_viewed_videos,
-                    #     'shield_filter_result': filtered_shield_video_ids,
-                    #     'executeTime': (time.time() - st_viewed) * 1000
-                    #  })
-                    return filtered_shield_video_ids
+                    return normal_recall_ids+filtered_shield_video_ids
                 else:
                     return filtered_viewed_videos
 

video_recall.py

@@ -2240,23 +2240,8 @@ class PoolRecall(object):
     def new_flow_pool_recall(self, size=10, flow_pool_id=None):
         """从流量池中获取视频"""
         start_time = time.time()
-        # 获取存在城市分组数据的城市编码列表
-        city_code_list = [code for _, code in config_.CITY_CODE.items()]
-        # 获取provinceCode
-        province_code = self.client_info.get('provinceCode', '-1')
-        # 获取cityCode
-        city_code = self.client_info.get('cityCode', '-1')
-
-        if city_code in city_code_list:
-            # 分城市数据存在时，获取城市分组数据
-            region_code = city_code
-        else:
-            region_code = province_code
-        if region_code == '':
-            region_code = '-1'
-
         flow_pool_key = self.get_pool_redis_key('flow', flow_pool_id=flow_pool_id)
-        # print(flow_pool_key)
+
         flow_pool_recall_result = []
         flow_pool_recall_videos = []
         # 每次获取的视频数
@@ -2302,26 +2287,13 @@ class PoolRecall(object):
                 else:
                     video_mapping[video_id].append(flow_pool)
             # 过滤
-            filter_ = FilterVideos(request_id=self.request_id,
-                                   app_type=self.app_type, mid=self.mid, uid=self.uid, video_ids=video_ids)
-            filtered_result = filter_.filter_videos(pool_type='flow', region_code=region_code, shield_config=self.shield_config)
-            print("flow filter time:", (time.time()-et_get)*1000)
+            # filter_ = FilterVideos(request_id=self.request_id,
+            #                        app_type=self.app_type, mid=self.mid, uid=self.uid, video_ids=video_ids)
+            # filtered_result = filter_.filter_videos(pool_type='flow', region_code=region_code, shield_config=self.shield_config)
+            # print("flow filter time:", (time.time()-et_get)*1000)
             # 检查可分发数
-            if filtered_result:
-                st_check = time.time()
-                ge = gevent.spawn(self.check_video_counts, video_ids=filtered_result, flow_pool_mapping=video_mapping)
-                ge.join()
-                check_result = ge.get()
-                # log_.info({
-                #     'logTimestamp': int(time.time() * 1000),
-                #     'request_id': self.request_id,
-                #     'app_type': self.app_type,
-                #     'mid': self.mid,
-                #     'uid': self.uid,
-                #     'operation': 'check_video_counts',
-                #     'executeTime': (time.time() - st_check) * 1000
-                # })
-
+            if video_ids and len(video_ids)>0:
+                check_result = self.check_video_counts(video_ids=video_ids, flow_pool_mapping=video_mapping)
                 for item in check_result:
                     video_id = int(item[0])
                     flow_pool = item[1]