update region_rule_rank_h dup_to_redis

config.py

@@ -391,7 +391,7 @@ class BaseConfig(object):
             'rule_params': {
                 # 'rule2': {'view_type': 'video-show', 'platform_return_rate': 0.001, 'region_24h_rule_key': 'rule2'},
                 'rule3': {'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                          'region_24h_rule_key': 'rule2'},
+                          'region_24h_rule_key': 'rule2', '24h_rule_key': 'rule2'},
                 'rule4': {'view_type': 'video-show-region', 'platform_return_rate': 0.001,
                           'region_24h_rule_key': 'rule2', '24h_rule_key': 'rule3'},
             },
@@ -409,7 +409,7 @@ class BaseConfig(object):
             'rule_params': {
                 # 'rule2': {'view_type': 'video-show', 'platform_return_rate': 0.001, 'region_24h_rule_key': 'rule2'},
                 'rule3': {'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                          'region_24h_rule_key': 'rule2'},
+                          'region_24h_rule_key': 'rule2', '24h_rule_key': 'rule2'},
             },
             'data_params': {
                 'data1': [APP_TYPE['VLOG'], ],
@@ -427,7 +427,7 @@ class BaseConfig(object):
             'rule_params': {
                 # 'rule2': {'view_type': 'video-show', 'platform_return_rate': 0.001, 'region_24h_rule_key': 'rule2'},
                 'rule3': {'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                          'region_24h_rule_key': 'rule2'},
+                          'region_24h_rule_key': 'rule2', '24h_rule_key': 'rule2'},
             },
             'data_params': {
                 'data1': [APP_TYPE['VLOG'], ],
@@ -440,7 +440,7 @@ class BaseConfig(object):
             'rule_params': {
                 # 'rule2': {'view_type': 'video-show', 'platform_return_rate': 0.001, 'region_24h_rule_key': 'rule2'},
                 'rule3': {'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                          'region_24h_rule_key': 'rule2'},
+                          'region_24h_rule_key': 'rule2', '24h_rule_key': 'rule2'},
             },
             'data_params': {
                 'data1': [APP_TYPE['VLOG'], ],
@@ -453,7 +453,7 @@ class BaseConfig(object):
             'rule_params': {
                 # 'rule2': {'view_type': 'video-show', 'platform_return_rate': 0.001, 'region_24h_rule_key': 'rule2'},
                 'rule3': {'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                          'region_24h_rule_key': 'rule2'},
+                          'region_24h_rule_key': 'rule2', '24h_rule_key': 'rule2'},
             },
             'data_params': {
                 'data1': [APP_TYPE['VLOG'], ],
@@ -466,7 +466,7 @@ class BaseConfig(object):
             'rule_params': {
                 # 'rule2': {'view_type': 'video-show', 'platform_return_rate': 0.001, 'region_24h_rule_key': 'rule2'},
                 'rule3': {'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                          'region_24h_rule_key': 'rule2'},
+                          'region_24h_rule_key': 'rule2', '24h_rule_key': 'rule2'},
             },
             'data_params': {
                 'data1': [APP_TYPE['VLOG'], ],
@@ -484,7 +484,7 @@ class BaseConfig(object):
             'rule_params': {
                 # 'rule2': {'view_type': 'video-show', 'platform_return_rate': 0.001, 'region_24h_rule_key': 'rule2'},
                 'rule3': {'view_type': 'video-show-region', 'platform_return_rate': 0.001,
-                          'region_24h_rule_key': 'rule2'},
+                          'region_24h_rule_key': 'rule2', '24h_rule_key': 'rule2'},
             },
             'data_params': {
                 'data1': [APP_TYPE['VLOG'], ],
@@ -1000,8 +1000,8 @@ class ProductionConfig(BaseConfig):
 
 def set_config():
     # 获取环境变量 ROV_OFFLINE_ENV
-    # env = os.environ.get('ROV_OFFLINE_ENV')
-    env = 'dev'
+    env = os.environ.get('ROV_OFFLINE_ENV')
+    # env = 'dev'
     if env is None:
         # log_.error('ENV ERROR: is None!')
         return

region_rule_rank_h.py

@@ -182,12 +182,14 @@ def video_rank(df, now_date, now_h, rule_key, param, region, app_type, data_key)
         redis_helper.del_keys(key_name=f"{config_.REGION_H_VIDEO_FILER}{region}.{app_type}.{data_key}.{rule_key}")
 
     region_24h_rule_key = param.get('region_24h_rule_key', 'rule1')
+    by_24h_rule_key = param.get('24h_rule_key', None)
     # 与其他召回视频池去重，存入对应的redis
     dup_to_redis(h_video_ids=h_video_ids, now_date=now_date, now_h=now_h, rule_key=rule_key,
-                 region_24h_rule_key=region_24h_rule_key, region=region, app_type=app_type, data_key=data_key)
+                 region_24h_rule_key=region_24h_rule_key, by_24h_rule_key=by_24h_rule_key,
+                 region=region, app_type=app_type, data_key=data_key)
 
 
-def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, region, app_type, data_key):
+def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, by_24h_rule_key, region, app_type, data_key):
     """将地域分组小时级数据与其他召回视频池去重，存入对应的redis"""
     redis_helper = RedisHelper()
     # # ##### 去重更新地域分组天级列表，并另存为redis中
@@ -260,7 +262,7 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, re
     #         redis_helper.add_data_with_zset(key_name=day_dup_key_name, data=day_dup, expire_time=23 * 3600)
 
     # ##### 去重小程序相对24h更新结果，并另存为redis中
-    day_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}{app_type}.{data_key}.rule2." \
+    day_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}{app_type}.{data_key}.{by_24h_rule_key}." \
                    f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
     if redis_helper.key_exists(key_name=day_key_name):
         day_data = redis_helper.get_all_data_from_zset(key_name=day_key_name, with_scores=True)
@@ -289,6 +291,35 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, re
             # 清空线上过滤应用列表
             redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}")
 
+    # ##### 去重小程序相对24h 筛选后剩余数据 更新结果，并另存为redis中
+    if by_24h_rule_key == 'rule3':
+        other_h_24h_recall_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H_OTHER}{app_type}.{data_key}." \
+                                      f"{by_24h_rule_key}.{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+        if redis_helper.key_exists(key_name=other_h_24h_recall_key_name):
+            other_24h_data = redis_helper.get_all_data_from_zset(key_name=other_h_24h_recall_key_name, with_scores=True)
+            log_.info(f'24h other data count = {len(other_24h_data)}')
+
+            # 屏蔽视频过滤
+            other_24h_video_ids = [int(video_id) for video_id, _ in other_24h_data]
+            shield_key_name_list = config_.SHIELD_CONFIG.get(region, None)
+            if shield_key_name_list is not None:
+                other_24h_video_ids = filter_shield_video(video_ids=other_24h_video_ids, shield_key_name_list=shield_key_name_list)
+                log_.info(f"shield filtered_videos count = {len(other_24h_video_ids)}")
+
+            other_24h_dup = {}
+            for video_id, score in other_24h_data:
+                if int(video_id) not in h_video_ids and int(video_id) in other_24h_video_ids:
+                    other_24h_dup[int(video_id)] = score
+                    h_video_ids.append(int(video_id))
+            log_.info(f"other 24h data dup count = {len(other_24h_dup)}")
+            other_24h_dup_key_name = \
+                f"{config_.RECALL_KEY_NAME_PREFIX_DUP3_REGION_24H_H}{region}.{app_type}.{data_key}.{rule_key}." \
+                f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+            if len(other_24h_dup) > 0:
+                redis_helper.add_data_with_zset(key_name=other_24h_dup_key_name, data=other_24h_dup, expire_time=23 * 3600)
+                # 限流视频score调整
+                update_limit_video_score(initial_videos=other_24h_dup, key_name=other_24h_dup_key_name)
+
     # ##### 去重小程序模型更新结果，并另存为redis中
     model_key_name = get_rov_redis_key(now_date=now_date)
     model_data = redis_helper.get_all_data_from_zset(key_name=model_key_name, with_scores=True)