update videos_filter.py

videos_filter.py

@@ -209,6 +209,65 @@ def filter_flow_pool():
     log_.info("flow pool filter end!")
 
 
+def filter_flow_pool_new():
+    """流量池视频过滤"""
+    log_.info("flow pool filter start ...")
+    app_type_list = [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]
+    for _, app_type in config_.APP_TYPE.items():
+        log_.info('app_type {} videos filter start...'.format(app_type))
+        if app_type in app_type_list:
+            filter_flow_pool_18_19(app_type=app_type)
+        else:
+            for flow_pool_id in [None, config_.QUICK_FLOW_POOL_ID]:
+                log_.info(f"flow_pool_id = {flow_pool_id}")
+                # 拼接redis-key
+                key_name = get_pool_redis_key(pool_type='flow_set', app_type=app_type, flow_pool_id=flow_pool_id)
+                # 获取视频
+                redis_helper = RedisHelper()
+                data = redis_helper.get_data_from_set(key_name=key_name)
+                if data is None:
+                    log_.info(f"flow_pool_id = {flow_pool_id}, data is None")
+                    log_.info(f"app_type = {app_type}, flow_pool_id = {flow_pool_id}, videos filter end!")
+                    continue
+                # videoId与flowPool做mapping
+                video_ids = []
+                mapping = {}
+                for video in data:
+                    video_id, flow_pool = video.split('-')
+                    video_id = int(video_id)
+                    if video_id not in video_ids:
+                        video_ids.append(video_id)
+                        mapping[video_id] = [flow_pool]
+                    else:
+                        mapping[video_id].append(flow_pool)
+                # 过滤
+                if len(video_ids) == 0:
+                    log_.info(f"data size = {len(data)}, video_ids size = {len(video_ids)}, data = {data}")
+                    log_.info(f"app_type = {app_type}, flow_pool_id = {flow_pool_id}, videos filter end!")
+                    continue
+                if app_type == config_.APP_TYPE['APP']:
+                    filtered_result = filter_video_status_app(video_ids=video_ids)
+                else:
+                    filtered_result = filter_video_status(video_ids=video_ids)
+                # 求差集，获取需要过滤掉的视频，并从redis中移除
+                filter_videos = set(video_ids) - set(filtered_result)
+                log_.info("data size = {}, video_ids size = {}, filtered size = {}, filter sizer = {}".format(
+                    len(data), len(video_ids), len(filtered_result), len(filter_videos)))
+                # 移除
+                if len(filter_videos) == 0:
+                    log_.info(f"app_type = {app_type}, flow_pool_id = {flow_pool_id} videos filter end!")
+                    continue
+                remove_videos = ['{}-{}'.format(video_id, flow_pool)
+                                 for video_id in filter_videos
+                                 for flow_pool in mapping[video_id]]
+                redis_helper.remove_value_from_set(key_name=key_name, values=tuple(remove_videos))
+                log_.info(f"app_type = {app_type}, flow_pool_id = {flow_pool_id} videos filter end!")
+
+            log_.info(f"app_type = {app_type} videos filter end!")
+
+    log_.info("flow pool filter end!")
+
+
 def filter_flow_pool_18_19(app_type):
     """流量池视频过滤"""
     log_.info('app_type {} videos filter start...'.format(app_type))
@@ -394,6 +453,12 @@ def get_pool_redis_key(pool_type, app_type=None, flow_pool_id=None):
         else:
             return f"{config_.FLOWPOOL_KEY_NAME_PREFIX}{app_type}"
 
+    elif pool_type == 'flow_set':
+        if flow_pool_id == config_.QUICK_FLOW_POOL_ID:
+            return f"{config_.QUICK_FLOWPOOL_KEY_NAME_PREFIX_SET}{app_type}:{flow_pool_id}"
+        else:
+            return f"{config_.FLOWPOOL_KEY_NAME_PREFIX_SET}{app_type}"
+
     else:
         log_.error('pool type error')
         return None, None
@@ -940,7 +1005,8 @@ def main():
         # appType = 19, ROV召回池视频过滤
         # filter_rov_pool(app_type=config_.APP_TYPE['ZUI_JING_QI'])
         # 流量池视频过滤
-        filter_flow_pool()
+        # filter_flow_pool()
+        filter_flow_pool_new()
         # 兜底视频过滤
         # filter_bottom()
         # 修改过ROV的视频过滤