update redis-key format

check_video_limit_distribute.py

@@ -97,17 +97,17 @@ def check_videos_distribute():
 def process_with_region(app_type, data_key, rule_key, region, stop_distribute_video_id_list, now_date, now_h):
     log_.info(f"app_type = {app_type}, data_key = {data_key}, rule_key = {rule_key}, region = {region}")
     # 将已超分发视频加入到地域小时级线上过滤应用列表中
-    redis_helper.add_data_with_set(
-        key_name=f"{config_.REGION_H_VIDEO_FILER}{region}.{app_type}.{data_key}.{rule_key}",
-        values=stop_distribute_video_id_list,
-        expire_time=2 * 3600
-    )
+    # redis_helper.add_data_with_set(
+    #     key_name=f"{config_.REGION_H_VIDEO_FILER}{region}.{app_type}.{data_key}.{rule_key}",
+    #     values=stop_distribute_video_id_list,
+    #     expire_time=2 * 3600
+    # )
     # 将已超分发视频加入到地域分组24h的数据线上过滤应用列表中
-    redis_helper.add_data_with_set(
-        key_name=f"{config_.REGION_H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}",
-        values=stop_distribute_video_id_list,
-        expire_time=2 * 3600
-    )
+    # redis_helper.add_data_with_set(
+    #     key_name=f"{config_.REGION_H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}",
+    #     values=stop_distribute_video_id_list,
+    #     expire_time=2 * 3600
+    # )
     # 将已超分发视频加入到不区分相对24h线上过滤应用列表中
     # redis_helper.add_data_with_set(
     #     key_name=f"{config_.H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}",
@@ -132,8 +132,8 @@ def process_with_region(app_type, data_key, rule_key, region, stop_distribute_vi
         ]
 
     for key_prefix in key_prefix_list:
-        key_name = f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}." \
-                   f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+        key_name = f"{key_prefix}{region}:{app_type}:{data_key}:{rule_key}:" \
+                   f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
         if not redis_helper.key_exists(key_name=key_name):
             if now_h == 0:
                 redis_date = now_date - datetime.timedelta(days=1)
@@ -141,8 +141,8 @@ def process_with_region(app_type, data_key, rule_key, region, stop_distribute_vi
             else:
                 redis_date = now_date
                 redis_h = now_h - 1
-            key_name = f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}." \
-                       f"{datetime.datetime.strftime(redis_date, '%Y%m%d')}.{redis_h}"
+            key_name = f"{key_prefix}{region}:{app_type}:{data_key}:{rule_key}:" \
+                       f"{datetime.datetime.strftime(redis_date, '%Y%m%d')}:{redis_h}"
         redis_helper.remove_value_from_zset(key_name=key_name, value=stop_distribute_video_id_list)
 
     """

config.py

@@ -532,8 +532,9 @@ class BaseConfig(object):
     # 完整影视资源更新结果存放 redis key 前缀，完整格式：'com.weiqu.video.recall.whole.movies.item.{date}.{h}'
     RECALL_KEY_NAME_PREFIX_WHOLE_MOVIES = 'com.weiqu.video.recall.whole.movies.item.'
 
-    # 小程序离线ROV模型结果存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.{date}
-    RECALL_KEY_NAME_PREFIX = 'com.weiqu.video.recall.hot.item.score.'
+    # 小程序离线ROV模型结果存放 redis key前缀，完整格式：recall:item:score:rov:{date}
+    RECALL_KEY_NAME_PREFIX = 'recall:item:score:rov:'
+
     # 小程序小时级更新结果存放 redis key前缀，完整格式：com.weiqu.video.recall.item.score.h.{rule_key}.{date}.{h}
     RECALL_KEY_NAME_PREFIX_BY_H = 'com.weiqu.video.recall.item.score.h.'
     # 小程序相对24h数据更新结果与 小程序小时级更新结果 去重后 存放 redis key前缀，
@@ -555,11 +556,11 @@ class BaseConfig(object):
     RECALL_KEY_NAME_PREFIX_DUP_DAY_PRE = 'com.weiqu.video.recall.hot.item.score.dup.day.pre.'
 
     # 小程序小时级24h数据更新结果存放 redis key前缀，
-    # 完整格式：com.weiqu.video.recall.item.score.apptype.24h.{appType}.{data_key}.{rule_key}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_BY_24H = 'com.weiqu.video.recall.item.score.apptype.24h.'
+    # 完整格式：recall:item:score:apptype:24h:{appType}:{data_key}:{rule_key}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_BY_24H = 'recall:item:score:apptype:24h:'
     # 小程序小时级24h数据 筛选后的剩余数据 更新结果存放 redis key前缀，
-    # 完整格式：com.weiqu.video.recall.item.score.apptype.24h.other.{appType}.{data_key}.{rule_key}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_BY_24H_OTHER = 'com.weiqu.video.recall.item.score.apptype.24h.other.'
+    # 完整格式：recall:item:score:apptype:24h:other:{appType}:{data_key}:{rule_key}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_BY_24H_OTHER = 'recall:item:score:apptype:24h:other:'
     # 小程序离线ROV模型结果与小程序小时级24h更新结果去重后 存放 redis key前缀，
     # 完整格式：com.weiqu.video.recall.hot.item.score.dup.24h.{rule_key}.{date}.{h}
     RECALL_KEY_NAME_PREFIX_DUP_24H = 'com.weiqu.video.recall.hot.item.score.dup.24h.'
@@ -567,26 +568,27 @@ class BaseConfig(object):
     H_VIDEO_FILER_24H = 'com.weiqu.video.filter.apptype.h.item.24h.'
 
     # 小程序地域分组小时级更新结果存放 redis key前缀，
-    # 完整格式：com.weiqu.video.recall.item.score.apptype.region.h.{region}.{appType}.{data_key}.{rule_key}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_REGION_BY_H = 'com.weiqu.video.recall.item.score.apptype.region.h.'
+    # 完整格式：recall:item:score:apptype:region:h:{region}:{appType}:{data_key}:{rule_key}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_REGION_BY_H = 'recall:item:score:apptype:region:h:'
     # 小程序地域分组天级更新结果与小程序地域分组小时级更新结果去重后 存放 redis key前缀，
     # 完整格式：com.weiqu.video.recall.hot.item.score.dup1.region.day.h.{region}.{rule_key}.{date}.{h}
     RECALL_KEY_NAME_PREFIX_DUP1_REGION_DAY_H = 'com.weiqu.video.recall.hot.item.score.dup1.region.day.h.'
     # 小程序地域分组小时级更新24h结果与小程序地域分组小时级更新结果去重后 存放 redis key前缀，
-    # 完整格式：com.weiqu.video.recall.hot.item.score.dup1.apptype.region.24h.h.{region}.{appType}.{data_key}.{rule_key}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H = 'com.weiqu.video.recall.hot.item.score.dup1.apptype.region.24h.h.'
+    # 完整格式：recall:item:score:apptype:region:dup1:region24h:{region}:{appType}:{data_key}:{rule_key}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H = 'recall:item:score:apptype:region:dup1:region24h:'
     # 小程序天级更新结果与 小程序地域分组天级更新结果/小程序地域分组小时级更新结果 去重后 存放 redis key前缀，
     # 完整格式：com.weiqu.video.recall.hot.item.score.dup2.region.day.h.{region}.{rule_key}.{date}.{h}
     RECALL_KEY_NAME_PREFIX_DUP2_REGION_DAY_H = 'com.weiqu.video.recall.hot.item.score.dup2.region.day.h.'
     # 小程序24h更新结果与 小程序地域分组24h更新结果/小程序地域分组小时级更新结果 去重后 存放 redis key前缀，
-    # 完整格式：com.weiqu.video.recall.hot.item.score.dup2.apptype.region.24h.h.{region}.{appType}.{data_key}.{rule_key}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H = 'com.weiqu.video.recall.hot.item.score.dup2.apptype.region.24h.h.'
+    # 完整格式：recall:item:score:apptype:region:dup2:24h:{region}:{appType}:{data_key}:{rule_key}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H = 'recall:item:score:apptype:region:dup2:24h:'
     # 小程序小时级24h数据 筛选后的剩余数据 更新结果 与 小程序24h更新结果/小程序地域分组24h更新结果/小程序地域分组小时级更新结果 去重后 存放 redis key前缀，
-    # 完整格式：com.weiqu.video.recall.hot.item.score.dup3.apptype.region.24h.h.{region}.{appType}.{data_key}.{rule_key}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_DUP3_REGION_24H_H = 'com.weiqu.video.recall.hot.item.score.dup3.apptype.region.24h.h.'
+    # 完整格式：recall:item:score:apptype:region:dup3:24h:{region}:{appType}:{data_key}:{rule_key}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_DUP3_REGION_24H_H = 'recall:item:score:apptype:region:dup3:24h:'
     # 小程序离线ROV模型结果与 小程序天级更新结果/小程序地域分组天级更新结果/小程序地域分组小时级更新结果 去重后 存放 redis key前缀，
-    # 完整格式：com.weiqu.video.recall.hot.item.score.dup.apptype.region.h.{region}.{appType}.{data_key}.{rule_key}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_DUP_REGION_H = 'com.weiqu.video.recall.hot.item.score.dup.apptype.region.h.'
+    # 完整格式：recall:item:score:apptype:region:dup4:rov:{region}:{appType}:{data_key}:{rule_key}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_DUP_REGION_H = 'recall:item:score:apptype:region:dup4:rov:'
+
     # 地域分组小时级视频状态不符合推荐要求的列表 redis key，
     # 完整格式：com.weiqu.video.filter.apptype.region.h.item.{region}.{appType}.{data_key}.{rule_key}
     REGION_H_VIDEO_FILER = 'com.weiqu.video.filter.apptype.region.h.item.'
@@ -597,8 +599,8 @@ class BaseConfig(object):
     RECALL_KEY_NAME_PREFIX_REGION_BY_DAY = 'com.weiqu.video.recall.item.score.region.day.'
 
     # 小程序地域分组小时级更新24h结果存放 redis key前缀，
-    # 完整格式：com.weiqu.video.recall.item.score.apptype.region.24h.{region}.{appType}.{data_key}.{rule_key}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_REGION_BY_24H = 'com.weiqu.video.recall.item.score.apptype.region.24h.'
+    # 完整格式：recall:item:score:apptype:region:24h:{region}:{appType}:{data_key}:{rule_key}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_REGION_BY_24H = 'recall:item:score:apptype:region:24h:'
     # 小程序天级更新结果与 小程序地域分组小时级更新24h结果 去重后 存放 redis key前缀，
     # 完整格式：com.weiqu.video.recall.hot.item.score.dup.region.day.24h.{region}.{rule_key}.{date}.{h}
     RECALL_KEY_NAME_PREFIX_DUP_REGION_DAY_24H = 'com.weiqu.video.recall.hot.item.score.dup.region.day.24h.'
@@ -612,40 +614,40 @@ class BaseConfig(object):
     # 小程序老视频更新结果存放 redis key 前缀，完整格式：'com.weiqu.video.recall.old.item.{date}'
     RECALL_KEY_NAME_PREFIX_OLD_VIDEOS = 'com.weiqu.video.recall.old.item.'
 
-    # app应用 小程序离线ROV模型结果存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.app.{date}
-    RECALL_KEY_NAME_PREFIX_APP = 'com.weiqu.video.recall.hot.item.score.app.'
+    # app应用 小程序离线ROV模型结果存放 redis key前缀，完整格式：recall:item:score:rov:app:{date}
+    RECALL_KEY_NAME_PREFIX_APP = 'recall:item:score:rov:app:'
     # app应用 运营提供的小时级数据存放 redis key前缀，完整格式：com.weiqu.video.app.op.item.score.{date}.{h}
     APP_OP_VIDEOS_KEY_NAME_PREFIX = 'com.weiqu.video.app.op.item.score.'
     # app应用 小时级数据更新最终结果存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.app.{date}.{h}
     APP_FINAL_RECALL_KEY_NAME_PREFIX = 'com.weiqu.video.recall.hot.item.score.app.'
 
-    # appType:[18, 19]小程序离线ROV模型结果存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.{appType}.{date}.{h}
-    RECALL_KEY_NAME_PREFIX_APP_TYPE = 'com.weiqu.video.recall.hot.item.score.'
+    # appType:[18, 19]小程序离线ROV模型结果存放 redis key前缀，完整格式：recall:item:score:{appType}:{date}:{h}
+    RECALL_KEY_NAME_PREFIX_APP_TYPE = 'recall:item:score:'
 
     # appType = 6, ROV召回池redis key前缀，完整格式：com.weiqu.video.recall.hot.apptype.h.item.score.{appType}.{h}
     # RECALL_KEY_NAME_PREFIX_APP_TYPE = 'com.weiqu.video.recall.hot.apptype.h.item.score.'
 
-    # 流量池离线模型结果存放 redis key前缀，完整格式 com.weiqu.video.flowpool.hot.item.score.{appType}
-    FLOWPOOL_KEY_NAME_PREFIX = 'com.weiqu.video.flowpool.hot.item.score.'
-    # 快速曝光流量池数据存放 redis key前缀，完整格式 com.weiqu.video.quick.flowpool.hot.item.score.{appType}.{flowPool_id}
-    QUICK_FLOWPOOL_KEY_NAME_PREFIX = 'com.weiqu.video.quick.flowpool.hot.item.score.'
-    # 快速曝光流量池分发概率 redis key前缀，完整格式 com.weiqu.video.quick.flowpool.distribute_rate.{flowPool_id}
-    QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX = 'com.weiqu.video.quick.flowpool.distribute_rate.'
+    # 流量池离线模型结果存放 redis key前缀，完整格式 flow:pool:item:score:{appType}
+    FLOWPOOL_KEY_NAME_PREFIX = 'flow:pool:item:score:'
+    # 快速曝光流量池数据存放 redis key前缀，完整格式 flow:pool:quick:item:score:{appType}:{flowPool_id}
+    QUICK_FLOWPOOL_KEY_NAME_PREFIX = 'flow:pool:quick:item:score:'
+    # 快速曝光流量池分发概率 redis key前缀，完整格式 flow:pool:quick:distribute:rate:{flowPool_id}
+    QUICK_FLOWPOOL_DISTRIBUTE_RATE_KEY_NAME_PREFIX = 'flow:pool:quick:distribute:rate:'
 
     # 兜底视频redis存储key
-    BOTTOM_KEY_NAME = 'com.weiqu.video.bottom'
+    BOTTOM_KEY_NAME = 'bottom:videos'
     # 兜底视频数量
     BOTTOM_NUM = 1000
     # 首页兜底视频json存储 redis-key
     BOTTOM_JSON_KEY_NAME = 'com.weiqu.video.homepage.bottom.info.json.item'
 
     # 修改ROV的视频 redis key
-    UPDATE_ROV_KEY_NAME = 'com.weiqu.video.update.rov.item.score'
-    UPDATE_ROV_KEY_NAME_APP = 'com.weiqu.video.update.rov.item.score.app'
+    UPDATE_ROV_KEY_NAME = 'update:rov:item:score:applet'
+    UPDATE_ROV_KEY_NAME_APP = 'update:rov:item:score:app'
 
     # 生效中的置顶视频列表 redis key
-    TOP_VIDEO_LIST_KEY_NAME = 'com.weiqu.video.top.item.score.area'
-    TOP_VIDEO_LIST_KEY_NAME_APP = 'com.weiqu.video.top.item.score.area.app'
+    TOP_VIDEO_LIST_KEY_NAME = 'top:item:score:area:applet'
+    TOP_VIDEO_LIST_KEY_NAME_APP = 'top:item:score:area:app'
 
     # rovScore公差
     ROV_SCORE_D = 0.001
@@ -679,23 +681,23 @@ class BaseConfig(object):
         'videos': 'shielded_video_list'
     }
 
-    # 特殊mid更新结果存放 redis key ，完整格式：'com.weiqu.video.special.mid'
-    KEY_NAME_SPECIAL_MID = 'com.weiqu.video.special.mid'
-    # 特殊mid对应指定视频列表更新结果存放 redis key 前缀，完整格式：'com.weiqu.video.special.videos.item.{date}'
-    KEY_NAME_PREFIX_SPECIAL_VIDEOS = 'com.weiqu.video.special.videos.item.'
+    # 特殊mid更新结果存放 redis key ，完整格式：'special:mid'
+    KEY_NAME_SPECIAL_MID = 'special:mid'
+    # 特殊mid对应指定视频列表更新结果存放 redis key 前缀，完整格式：'special:videos:item:{date}'
+    KEY_NAME_PREFIX_SPECIAL_VIDEOS = 'special:videos:item:'
 
-    # 限流视频集合存放 redis key前缀，完整格式：'com.weiqu.video.limit.set.{date}'
-    KEY_NAME_PREFIX_LIMIT_VIDEO_SET = 'com.weiqu.video.limit.set.'
-    # 限流视频最大分发数记录 redis key，完整格式：'com.weiqu.video.limit.item'
-    KEY_NAME_PREFIX_LIMIT_VIDEOS = 'com.weiqu.video.limit.item'
-    # 限流视频分发数记录 redis key前缀，完整格式：'com.weiqu.video.limit.distribute.count.{videoId}'
-    KEY_NAME_PREFIX_LIMIT_VIDEO_DISTRIBUTE_COUNT = 'com.weiqu.video.limit.distribute.count.'
+    # 限流视频集合存放 redis key前缀，完整格式：'limit:video:{date}'
+    KEY_NAME_PREFIX_LIMIT_VIDEO_SET = 'limit:videos:'
+    # 限流视频最大分发数记录 redis key，完整格式：'limit:video:max:distribute:count'
+    KEY_NAME_PREFIX_LIMIT_VIDEOS = 'limit:video:max:distribute:count'
+    # 限流视频分发数记录 redis key前缀，完整格式：'limit:video:distribute:count:{videoId}'
+    KEY_NAME_PREFIX_LIMIT_VIDEO_DISTRIBUTE_COUNT = 'limit:video:distribute:count:'
 
     # 本山祝福视频列表，在北京+重庆+无地域划分表中屏蔽
     PROJECT_BENSHAN_ZHUFU = 'loghubods'
     TABLE_BENSHAN_ZHUFU = 'benshanzhufu_videolist'
     # 本山祝福视频redis存储key
-    BENSHAN_ZHUFU_KEY_NAME = 'com.weiqu.video.benshanzf'
+    BENSHAN_ZHUFU_KEY_NAME = 'benshanzf:videos'
     # 屏蔽视频配置 key:region_code, value:videos key list
     SHIELD_CONFIG = {
         '110000': [BENSHAN_ZHUFU_KEY_NAME, ],

pool_predict.py

@@ -136,7 +136,7 @@ def predict(app_type):
 
         # 快速曝光流量池视频写入redis
         redis_helper = RedisHelper()
-        quick_flow_pool_key_name = f"{config_.QUICK_FLOWPOOL_KEY_NAME_PREFIX}{app_type}.{config_.QUICK_FLOW_POOL_ID}"
+        quick_flow_pool_key_name = f"{config_.QUICK_FLOWPOOL_KEY_NAME_PREFIX}{app_type}:{config_.QUICK_FLOW_POOL_ID}"
         # 如果key已存在，删除key
         if redis_helper.key_exists(quick_flow_pool_key_name):
             redis_helper.del_keys(quick_flow_pool_key_name)
@@ -230,7 +230,7 @@ def predict_18_19(app_type):
             video_id = filtered_videos[i]
             score = video_score[i]
             redis_data[video_id] = score
-        key_name = config_.FLOWPOOL_KEY_NAME_PREFIX + str(app_type)
+        key_name = f'{config_.FLOWPOOL_KEY_NAME_PREFIX}{app_type}'
         redis_helper = RedisHelper()
         # 如果key已存在，删除key
         if redis_helper.key_exists(key_name):
@@ -278,7 +278,7 @@ def predict_19(app_type):
             score = video_score[video_id]
             data[video_id] = score
         log_.info('predict finished!')
-        key_name = config_.FLOWPOOL_KEY_NAME_PREFIX + str(app_type)
+        key_name = f'{config_.FLOWPOOL_KEY_NAME_PREFIX}{app_type}'
         redis_helper = RedisHelper()
         # 如果key已存在，删除key
         if redis_helper.key_exists(key_name):

redis_data_monitor.py

@@ -50,7 +50,7 @@ def region_data_monitor(now_date, now_h):
                     continue
                 no_update_region_list = []
                 for region in region_code_list:
-                    region_key_name = f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}.{now_date}.{now_h}"
+                    region_key_name = f"{key_prefix}{region}:{app_type}:{data_key}:{rule_key}:{now_date}:{now_h}"
                     if not redis_helper.key_exists(key_name=region_key_name):
                         no_update_region_list.append(region)
                 if len(no_update_region_list) == len(region_code_list):

region_rule_rank_h.py

@@ -180,14 +180,14 @@ def video_rank(df, now_date, now_h, rule_key, param, region, app_type, data_key)
         h_recall_result[int(video_id)] = float(score)
         h_video_ids.append(int(video_id))
     h_recall_key_name = \
-        f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H}{region}.{app_type}.{data_key}.{rule_key}." \
-        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+        f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_H}{region}:{app_type}:{data_key}:{rule_key}:" \
+        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
     if len(h_recall_result) > 0:
         redis_helper.add_data_with_zset(key_name=h_recall_key_name, data=h_recall_result, expire_time=23 * 3600)
         # 限流视频score调整
         update_limit_video_score(initial_videos=h_recall_result, key_name=h_recall_key_name)
         # 清空线上过滤应用列表
-        redis_helper.del_keys(key_name=f"{config_.REGION_H_VIDEO_FILER}{region}.{app_type}.{data_key}.{rule_key}")
+        # redis_helper.del_keys(key_name=f"{config_.REGION_H_VIDEO_FILER}{region}.{app_type}.{data_key}.{rule_key}")
 
     region_24h_rule_key = param.get('region_24h_rule_key', 'rule1')
     by_24h_rule_key = param.get('24h_rule_key', None)
@@ -222,8 +222,8 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, by
 
     # ##### 去重更新地域分组小时级24h列表，并另存为redis中
     region_24h_key_name = \
-        f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_24H}{region}.{app_type}.{data_key}.{region_24h_rule_key}." \
-        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+        f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_24H}{region}:{app_type}:{data_key}:{region_24h_rule_key}:" \
+        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
     if redis_helper.key_exists(key_name=region_24h_key_name):
         region_24h_data = redis_helper.get_all_data_from_zset(key_name=region_24h_key_name, with_scores=True)
         # log_.info(f'region 24h data count = {len(region_24h_data)}')
@@ -242,8 +242,8 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, by
                 h_video_ids.append(int(video_id))
         # log_.info(f"region 24h data dup count = {len(region_24h_dup)}")
         region_24h_dup_key_name = \
-            f"{config_.RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H}{region}.{app_type}.{data_key}.{rule_key}." \
-            f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+            f"{config_.RECALL_KEY_NAME_PREFIX_DUP1_REGION_24H_H}{region}:{app_type}:{data_key}:{rule_key}:" \
+            f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
         if len(region_24h_dup) > 0:
             redis_helper.add_data_with_zset(key_name=region_24h_dup_key_name, data=region_24h_dup, expire_time=23 * 3600)
             # 限流视频score调整
@@ -270,8 +270,8 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, by
     #         redis_helper.add_data_with_zset(key_name=day_dup_key_name, data=day_dup, expire_time=23 * 3600)
 
     # ##### 去重小程序相对24h更新结果，并另存为redis中
-    day_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}{app_type}.{data_key}.{by_24h_rule_key}." \
-                   f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+    day_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}{app_type}:{data_key}:{by_24h_rule_key}:" \
+                   f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
     if redis_helper.key_exists(key_name=day_key_name):
         day_data = redis_helper.get_all_data_from_zset(key_name=day_key_name, with_scores=True)
         # log_.info(f'24h data count = {len(day_data)}')
@@ -290,19 +290,19 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, by
                 h_video_ids.append(int(video_id))
         # log_.info(f"24h data dup count = {len(day_dup)}")
         day_dup_key_name = \
-            f"{config_.RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H}{region}.{app_type}.{data_key}.{rule_key}." \
-            f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+            f"{config_.RECALL_KEY_NAME_PREFIX_DUP2_REGION_24H_H}{region}:{app_type}:{data_key}:{rule_key}:" \
+            f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
         if len(day_dup) > 0:
             redis_helper.add_data_with_zset(key_name=day_dup_key_name, data=day_dup, expire_time=23 * 3600)
             # 限流视频score调整
             update_limit_video_score(initial_videos=day_dup, key_name=day_dup_key_name)
             # 清空线上过滤应用列表
-            redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}")
+            # redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}")
 
     # ##### 去重小程序相对24h 筛选后剩余数据 更新结果，并另存为redis中
     if by_24h_rule_key == 'rule3':
-        other_h_24h_recall_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H_OTHER}{app_type}.{data_key}." \
-                                      f"{by_24h_rule_key}.{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+        other_h_24h_recall_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H_OTHER}{app_type}:{data_key}:" \
+                                      f"{by_24h_rule_key}:{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
         if redis_helper.key_exists(key_name=other_h_24h_recall_key_name):
             other_24h_data = redis_helper.get_all_data_from_zset(key_name=other_h_24h_recall_key_name, with_scores=True)
             log_.info(f'24h other data count = {len(other_24h_data)}')
@@ -321,8 +321,8 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, by
                     h_video_ids.append(int(video_id))
             log_.info(f"other 24h data dup count = {len(other_24h_dup)}")
             other_24h_dup_key_name = \
-                f"{config_.RECALL_KEY_NAME_PREFIX_DUP3_REGION_24H_H}{region}.{app_type}.{data_key}.{rule_key}." \
-                f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+                f"{config_.RECALL_KEY_NAME_PREFIX_DUP3_REGION_24H_H}{region}:{app_type}:{data_key}:{rule_key}:" \
+                f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
             if len(other_24h_dup) > 0:
                 redis_helper.add_data_with_zset(key_name=other_24h_dup_key_name, data=other_24h_dup, expire_time=23 * 3600)
                 # 限流视频score调整
@@ -347,8 +347,8 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, by
             h_video_ids.append(int(video_id))
     # log_.info(f"model data dup count = {len(model_data_dup)}")
     model_data_dup_key_name = \
-        f"{config_.RECALL_KEY_NAME_PREFIX_DUP_REGION_H}{region}.{app_type}.{data_key}.{rule_key}." \
-        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+        f"{config_.RECALL_KEY_NAME_PREFIX_DUP_REGION_H}{region}:{app_type}:{data_key}:{rule_key}:" \
+        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
     if len(model_data_dup) > 0:
         redis_helper.add_data_with_zset(key_name=model_data_dup_key_name, data=model_data_dup, expire_time=23 * 3600)
         # 限流视频score调整
@@ -536,7 +536,7 @@ def h_rank_bottom(now_date, now_h, rule_params, region_code_list):
             region_24h_rule_key = rule_param.get('region_24h_rule_key', 'rule1')
             for region in region_code_list:
                 log_.info(f"region = {region}")
-                key_name = f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}.{redis_dt}.{redis_h}"
+                key_name = f"{key_prefix}{region}:{app_type}:{data_key}:{rule_key}:{redis_dt}:{redis_h}"
                 initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
                 if initial_data is None:
                     initial_data = []
@@ -547,16 +547,19 @@ def h_rank_bottom(now_date, now_h, rule_params, region_code_list):
                     h_video_ids.append(int(video_id))
                 # 存入对应的redis
                 final_key_name = \
-                    f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}.{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+                    f"{key_prefix}{region}:{app_type}:{data_key}:{rule_key}:{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
                 if len(final_data) > 0:
                     redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=23 * 3600)
                 # 清空线上过滤应用列表
-                redis_helper.del_keys(
-                    key_name=f"{config_.REGION_H_VIDEO_FILER}{region}.{app_type}.{data_key}.{rule_key}")
+                # redis_helper.del_keys(
+                #     key_name=f"{config_.REGION_H_VIDEO_FILER}{region}.{app_type}.{data_key}.{rule_key}")
+
+                region_24h_rule_key = param.get('region_24h_rule_key', 'rule1')
+                by_24h_rule_key = param.get('24h_rule_key', None)
                 # 与其他召回视频池去重，存入对应的redis
                 dup_to_redis(h_video_ids=h_video_ids, now_date=now_date, now_h=now_h, rule_key=rule_key,
                              region_24h_rule_key=region_24h_rule_key, region=region,
-                             app_type=app_type, data_key=data_key)
+                             app_type=app_type, data_key=data_key, by_24h_rule_key=by_24h_rule_key)
 
 
         # for data_key, data_param in params['data_params'].items():

region_rule_rank_h_by24h.py

@@ -168,12 +168,12 @@ def video_rank(df, now_date, now_h, rule_key, param, region, app_type, data_key)
         h_video_ids.append(int(video_id))
 
     day_recall_key_name = \
-        f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_24H}{region}.{app_type}.{data_key}.{rule_key}." \
-        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+        f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_24H}{region}:{app_type}:{data_key}:{rule_key}:" \
+        f"{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
     if len(day_recall_result) > 0:
         redis_helper.add_data_with_zset(key_name=day_recall_key_name, data=day_recall_result, expire_time=2 * 3600)
         # 清空线上过滤应用列表
-        redis_helper.del_keys(key_name=f"{config_.REGION_H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}")
+        # redis_helper.del_keys(key_name=f"{config_.REGION_H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}")
 
     # 与其他召回视频池去重，存入对应的redis
     # dup_to_redis(h_video_ids=h_video_ids, now_date=now_date, now_h=now_h, rule_key=rule_key, region=region)
@@ -237,25 +237,6 @@ def rank_by_24h(project, table, now_date, now_h, rule_params, region_code_list):
     feature_df = get_feature_data(project=project, table=table, now_date=now_date)
     feature_df['apptype'] = feature_df['apptype'].astype(int)
     # rank
-    # t = [
-    #     gevent.spawn(process_with_app_type, app_type, params, region_code_list, feature_df, now_date, now_h)
-    #     for app_type, params in rule_params.items()
-    # ]
-    # gevent.joinall(t)
-
-    """
-    ps = []
-    for app_type, params in rule_params.items():
-        p = multiprocessing.Process(target=process_with_app_type,
-                                    args=(app_type, params, region_code_list, feature_df, now_date, now_h))
-        ps.append(p)
-    for p in ps:
-        p.daemon = True
-        p.start()
-    for p in ps:
-        p.join()
-    """
-
     pool = multiprocessing.Pool(processes=len(config_.APP_TYPE))
     for app_type, params in rule_params.items():
         pool.apply_async(func=process_with_app_type,
@@ -263,41 +244,6 @@ def rank_by_24h(project, table, now_date, now_h, rule_params, region_code_list):
     pool.close()
     pool.join()
 
-    """
-    for app_type, params in rule_params.items():
-        log_.info(f"app_type = {app_type} start...")
-        data_params_item = params.get('data_params')
-        rule_params_item = params.get('rule_params')
-        for param in params.get('params_list'):
-            log_.info(f"param = {param} start...")
-            data_key = param.get('data')
-            data_param = data_params_item.get(data_key)
-            log_.info(f"data_key = {data_key}, data_param = {data_param}")
-            rule_key = param.get('rule')
-            rule_param = rule_params_item.get(rule_key)
-            log_.info(f"rule_key = {rule_key}, rule_param = {rule_param}")
-            df_list = [feature_df[feature_df['apptype'] == apptype] for apptype in data_param]
-            df_merged = reduce(merge_df, df_list)
-
-            # thread_pool = ThreadPoolExecutor(max_workers=5)
-            # for region in region_code_list:
-            #     t = thread_pool.submit(process_with_region,
-            #                        (region, df_merged, app_type, data_key, rule_key, rule_param, now_date, now_h))
-            # thread_pool.shutdown(wait=True)
-
-            task_list = []
-            for region in region_code_list:
-                t = Thread(target=process_with_region,
-                           args=(region, df_merged, app_type, data_key, rule_key, rule_param, now_date, now_h)
-                           )
-                t.start()
-                task_list.append(t)
-            for t in task_list:
-                t.join()
-            log_.info(f"param = {param} end!")
-        log_.info(f"app_type = {app_type} end!")
-    """
-
     # for app_type, params in rule_params.items():
     #     log_.info(f"app_type = {app_type}")
     #     for data_key, data_param in params['data_params'].items():
@@ -393,7 +339,7 @@ def h_rank_bottom(now_date, now_h, rule_params, region_code_list):
             log_.info(f"data_key = {data_key}, rule_key = {rule_key}")
             for region in region_code_list:
                 log_.info(f"region = {region}")
-                key_name = f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}.{redis_dt}.{redis_h}"
+                key_name = f"{key_prefix}{region}:{app_type}:{data_key}:{rule_key}:{redis_dt}:{redis_h}"
                 initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
                 if initial_data is None:
                     initial_data = []
@@ -404,11 +350,11 @@ def h_rank_bottom(now_date, now_h, rule_params, region_code_list):
                     h_video_ids.append(int(video_id))
                 # 存入对应的redis
                 final_key_name = \
-                    f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}.{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+                    f"{key_prefix}{region}:{app_type}:{data_key}:{rule_key}:{datetime.datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
                 if len(final_data) > 0:
                     redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=2 * 3600)
                 # 清空线上过滤应用列表
-                redis_helper.del_keys(key_name=f"{config_.REGION_H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}")
+                # redis_helper.del_keys(key_name=f"{config_.REGION_H_VIDEO_FILER_24H}{region}.{app_type}.{data_key}.{rule_key}")
 
                 # 与其他召回视频池去重，存入对应的redis
                 # dup_to_redis(h_video_ids=h_video_ids, now_date=now_date, now_h=now_h, rule_key=rule_key, region=region)

rule_rank_h_by_24h.py

@@ -180,12 +180,12 @@ def video_rank_h(df, now_date, now_h, rule_key, param, app_type, data_key):
         day_recall_result[int(video_id)] = float(score)
         day_video_ids.append(int(video_id))
     h_24h_recall_key_name = \
-        f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}{app_type}.{data_key}.{rule_key}.{now_dt}.{now_h}"
+        f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}{app_type}:{data_key}:{rule_key}:{now_dt}:{now_h}"
     if len(day_recall_result) > 0:
         log_.info(f"count = {len(day_recall_result)}")
         redis_helper.add_data_with_zset(key_name=h_24h_recall_key_name, data=day_recall_result, expire_time=2 * 3600)
         # 清空线上过滤应用列表
-        redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{app_type}.{data_key}.{rule_key}")
+        # redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{app_type}.{data_key}.{rule_key}")
 
     if rule_key == 'rule3':
         # 去重筛选结果，保留剩余数据并写入Redis
@@ -203,7 +203,7 @@ def video_rank_h(df, now_date, now_h, rule_key, param, app_type, data_key):
             score = df[df['videoid'] == video_id]['score']
             other_24h_recall_result[int(video_id)] = float(score)
         other_h_24h_recall_key_name = \
-            f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H_OTHER}{app_type}.{data_key}.{rule_key}.{now_dt}.{now_h}"
+            f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H_OTHER}{app_type}:{data_key}:{rule_key}:{now_dt}:{now_h}"
         if len(other_24h_recall_result) > 0:
             log_.info(f"count = {len(other_24h_recall_result)}")
             redis_helper.add_data_with_zset(key_name=other_h_24h_recall_key_name, data=other_24h_recall_result,
@@ -295,7 +295,7 @@ def h_rank_bottom(now_date, now_h, rule_params):
             rule_key = param.get('rule')
             log_.info(f"data_key = {data_key}, rule_key = {rule_key}")
             for key_prefix in key_prefix_list:
-                key_name = f"{key_prefix}{app_type}.{data_key}.{rule_key}.{redis_dt}.{redis_h}"
+                key_name = f"{key_prefix}{app_type}:{data_key}:{rule_key}:{redis_dt}:{redis_h}"
                 initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
                 if initial_data is None:
                     initial_data = []
@@ -304,11 +304,11 @@ def h_rank_bottom(now_date, now_h, rule_params):
                     final_data[video_id] = score
                 # 存入对应的redis
                 final_key_name = \
-                    f"{key_prefix}{app_type}.{data_key}.{rule_key}.{datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+                    f"{key_prefix}{app_type}:{data_key}:{rule_key}:{datetime.strftime(now_date, '%Y%m%d')}:{now_h}"
                 if len(final_data) > 0:
                     redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=2 * 3600)
                 # 清空线上过滤应用列表
-                redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{app_type}.{data_key}.{rule_key}")
+                # redis_helper.del_keys(key_name=f"{config_.H_VIDEO_FILER_24H}{app_type}.{data_key}.{rule_key}")
 
 
 def h_timer_check():

videos_filter.py

@@ -335,7 +335,7 @@ def get_pool_redis_key(pool_type, app_type=None, flow_pool_id=None):
             # 获取当前所在小时
             redis_date = datetime.now().hour
             # 判断热度列表是否更新，未更新则使用前一小时的热度列表
-            key_name = '{}{}.{}'.format(config_.RECALL_KEY_NAME_PREFIX_APP_TYPE, app_type, redis_date)
+            key_name = f'{config_.RECALL_KEY_NAME_PREFIX_APP_TYPE}{app_type}:{redis_date}'
             if redis_helper.key_exists(key_name):
                 return key_name, redis_date
             else:
@@ -343,16 +343,16 @@ def get_pool_redis_key(pool_type, app_type=None, flow_pool_id=None):
                     redis_date = 23
                 else:
                     redis_date = redis_date - 1
-                key_name = '{}{}.{}'.format(config_.RECALL_KEY_NAME_PREFIX_APP_TYPE, app_type, redis_date)
+                key_name = f'{config_.RECALL_KEY_NAME_PREFIX_APP_TYPE}{app_type}:{redis_date}'
 
                 return key_name, redis_date
 
         # appType: [18, 19]
         elif app_type in [config_.APP_TYPE['LAO_HAO_KAN_VIDEO'], config_.APP_TYPE['ZUI_JING_QI']]:
-            key_name_prefix = f'{config_.RECALL_KEY_NAME_PREFIX_APP_TYPE}{app_type}.'
+            key_name_prefix = f'{config_.RECALL_KEY_NAME_PREFIX_APP_TYPE}{app_type}:'
             now_date = datetime.today().strftime('%Y%m%d')
             now_h = datetime.now().hour
-            key_name = f"{key_name_prefix}{now_date}.{now_h}"
+            key_name = f"{key_name_prefix}{now_date}:{now_h}"
             if redis_helper.key_exists(key_name):
                 return key_name, now_h
             else:
@@ -362,7 +362,7 @@ def get_pool_redis_key(pool_type, app_type=None, flow_pool_id=None):
                 else:
                     redis_h = now_h - 1
                     redis_date = now_date
-                key_name = f"{key_name_prefix}{redis_date}.{redis_h}"
+                key_name = f"{key_name_prefix}{redis_date}:{redis_h}"
                 return key_name, redis_h
 
         else:
@@ -377,19 +377,19 @@ def get_pool_redis_key(pool_type, app_type=None, flow_pool_id=None):
                 key_name_prefix = config_.RECALL_KEY_NAME_PREFIX
 
             # 判断热度列表是否更新，未更新则使用前一天的热度列表
-            key_name = key_name_prefix + time.strftime('%Y%m%d')
+            key_name = f"{key_name_prefix}{time.strftime('%Y%m%d')}"
             if redis_helper.key_exists(key_name):
                 redis_date = date.today().strftime('%Y%m%d')
             else:
                 redis_date = (date.today() - timedelta(days=1)).strftime('%Y%m%d')
-                key_name = key_name_prefix + redis_date
+                key_name = f"{key_name_prefix}{redis_date}"
 
             return key_name, redis_date
 
     elif pool_type == 'flow':
         # 流量池
         if flow_pool_id == config_.QUICK_FLOW_POOL_ID:
-            return f"{config_.QUICK_FLOWPOOL_KEY_NAME_PREFIX}{app_type}.{flow_pool_id}"
+            return f"{config_.QUICK_FLOWPOOL_KEY_NAME_PREFIX}{app_type}:{flow_pool_id}"
         else:
             return f"{config_.FLOWPOOL_KEY_NAME_PREFIX}{app_type}"
 
@@ -582,7 +582,7 @@ def filter_process_with_region(app_type, data_key, rule_key, region, now_date, n
     ]
     for i, key_prefix in enumerate(key_prefix_list):
         # 拼接key
-        key_name = f"{key_prefix}{region}.{app_type}.{data_key}.{rule_key}.{now_date}.{now_h}"
+        key_name = f"{key_prefix}{region}:{app_type}:{data_key}:{rule_key}:{now_date}:{now_h}"
         # 获取视频
         data = redis_helper.get_all_data_from_zset(key_name=key_name)
         if data is None:
@@ -603,11 +603,11 @@ def filter_process_with_region(app_type, data_key, rule_key, region, now_date, n
             log_.info("filter end!")
             continue
         redis_helper.remove_value_from_zset(key_name=key_name, value=list(filter_videos))
-        if i == 0:
-            # 将小时级的数据需要过滤的视频加入到线上过滤应用列表中
-            redis_helper.add_data_with_set(key_name=f"{config_.REGION_H_VIDEO_FILER}"
-                                                    f"{region}.{app_type}.{data_key}.{rule_key}",
-                                           values=filter_videos, expire_time=2 * 3600)
+        # if i == 0:
+        #     # 将小时级的数据需要过滤的视频加入到线上过滤应用列表中
+        #     redis_helper.add_data_with_set(key_name=f"{config_.REGION_H_VIDEO_FILER}"
+        #                                             f"{region}.{app_type}.{data_key}.{rule_key}",
+        #                                    values=filter_videos, expire_time=2 * 3600)
         # elif i == 1:
         #     # 将地域分组24h的数据需要过滤的视频加入到线上过滤应用列表中
         #     redis_helper.add_data_with_set(key_name=f"{config_.REGION_H_VIDEO_FILER_24H}"