update

region_rule_rank_h.py

@@ -250,8 +250,7 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, re
         f"{config_.RECALL_KEY_NAME_PREFIX_REGION_BY_24H}{region}.{region_24h_rule_key}." \
         f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
     if redis_helper.key_exists(key_name=region_24h_key_name):
-        region_24h_data = redis_helper.get_data_zset_with_index(
-            key_name=region_24h_key_name, start=0, end=-1, with_scores=True)
+        region_24h_data = redis_helper.get_all_data_from_zset(key_name=region_24h_key_name, with_scores=True)
         log_.info(f'region 24h data count = {len(region_24h_data)}')
         region_24h_dup = {}
         for video_id, score in region_24h_data:
@@ -287,8 +286,7 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, re
     day_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}rule2." \
                    f"{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
     if redis_helper.key_exists(key_name=day_key_name):
-        day_data = redis_helper.get_data_zset_with_index(
-            key_name=day_key_name, start=0, end=-1, with_scores=True)
+        day_data = redis_helper.get_all_data_from_zset(key_name=day_key_name, with_scores=True)
         log_.info(f'24h data count = {len(day_data)}')
         day_dup = {}
         for video_id, score in day_data:
@@ -306,7 +304,7 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region_24h_rule_key, re
 
     # ##### 去重小程序模型更新结果，并另存为redis中
     model_key_name = get_rov_redis_key(now_date=now_date)
-    model_data = redis_helper.get_data_zset_with_index(key_name=model_key_name, start=0, end=-1, with_scores=True)
+    model_data = redis_helper.get_all_data_from_zset(key_name=model_key_name, with_scores=True)
     log_.info(f'model data count = {len(model_data)}')
     model_data_dup = {}
     for video_id, score in model_data:
@@ -374,7 +372,7 @@ def h_rank_bottom(now_date, now_h, rule_key, region_code_list, param):
     for region in region_code_list:
         log_.info(f"region = {region}")
         key_name = f"{key_prefix}{region}.{rule_key}.{redis_dt}.{redis_h}"
-        initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+        initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
         final_data = dict()
         h_video_ids = []
         for video_id, score in initial_data:

region_rule_rank_h_by24h.py

@@ -233,8 +233,7 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region):
     # ##### 去重小程序天级更新结果，并另存为redis中
     day_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_DAY}rule2.{datetime.datetime.strftime(now_date, '%Y%m%d')}"
     if redis_helper.key_exists(key_name=day_key_name):
-        day_data = redis_helper.get_data_zset_with_index(
-            key_name=day_key_name, start=0, end=-1, with_scores=True)
+        day_data = redis_helper.get_all_data_from_zset(key_name=day_key_name, with_scores=True)
         log_.info(f'day data count = {len(day_data)}')
         day_dup = {}
         for video_id, score in day_data:
@@ -250,7 +249,7 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key, region):
 
     # ##### 去重小程序模型更新结果，并另存为redis中
     model_key_name = get_rov_redis_key(now_date=now_date)
-    model_data = redis_helper.get_data_zset_with_index(key_name=model_key_name, start=0, end=-1, with_scores=True)
+    model_data = redis_helper.get_all_data_from_zset(key_name=model_key_name, with_scores=True)
     log_.info(f'model data count = {len(model_data)}')
     model_data_dup = {}
     for video_id, score in model_data:
@@ -282,7 +281,7 @@ def h_rank_bottom(now_date, now_h, rule_key, region_code_list):
     for region in region_code_list:
         log_.info(f"region = {region}")
         key_name = f"{key_prefix}{region}.{rule_key}.{redis_dt}.{redis_h}"
-        initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+        initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
         final_data = dict()
         h_video_ids = []
         for video_id, score in initial_data:

rule_rank_day.py

@@ -113,7 +113,7 @@ def video_rank_day(df, now_date, rule_key, param):
     # 获取rov模型结果
     redis_helper = RedisHelper()
     key_name, key_dt = get_rov_redis_key(now_date=now_date)
-    initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+    initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
     log_.info(f'initial data count = {len(initial_data)}, key_dt = {key_dt}')
 
     # 获取符合进入召回源条件的视频
@@ -194,7 +194,7 @@ def day_rank_bottom(now_date, rule_key):
     now_dt = datetime.strftime(now_date, '%Y%m%d')
     redis_helper = RedisHelper()
     key_name, key_dt = get_rov_redis_key(now_date=now_date)
-    initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+    initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
     final_data = dict()
     for video_id, score in initial_data:
         final_data[video_id] = score

rule_rank_h.py

@@ -145,7 +145,7 @@ def video_rank(df, now_date, now_h, rule_key, param):
     # 获取rov模型结果
     redis_helper = RedisHelper()
     key_name = get_rov_redis_key(now_date=now_date)
-    initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+    initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
     log_.info(f'initial data count = {len(initial_data)}')
 
     # 获取符合进入召回源条件的视频，进入条件：小时级回流>=20 && score>=0.005
@@ -195,8 +195,7 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key):
     # ##### 去重小程序相对24h数据更新结果，并另存为redis中
     rule_24h_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_24H}rule2.{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
     if redis_helper.key_exists(key_name=rule_24h_key_name):
-        rule_24h_data = redis_helper.get_data_zset_with_index(
-            key_name=rule_24h_key_name, start=0, end=-1, with_scores=True)
+        rule_24h_data = redis_helper.get_all_data_from_zset(key_name=rule_24h_key_name, with_scores=True)
         log_.info(f'rule_24h data count = {len(rule_24h_data)}')
         rule_24h_dup = {}
         for video_id, score in rule_24h_data:
@@ -211,7 +210,7 @@ def dup_to_redis(h_video_ids, now_date, now_h, rule_key):
 
     # ##### 去重小程序模型更新结果，并另存为redis中
     model_key_name = get_rov_redis_key(now_date=now_date)
-    model_data = redis_helper.get_data_zset_with_index(key_name=model_key_name, start=0, end=-1, with_scores=True)
+    model_data = redis_helper.get_all_data_from_zset(key_name=model_key_name, with_scores=True)
     log_.info(f'model data count = {len(model_data)}')
     model_data_dup = {}
     for video_id, score in model_data:
@@ -264,7 +263,7 @@ def h_rank_bottom(now_date, now_h, rule_key):
     key_prefix_list = [config_.RECALL_KEY_NAME_PREFIX_BY_H]
     for key_prefix in key_prefix_list:
         key_name = f"{key_prefix}{rule_key}.{redis_dt}.{redis_h}"
-        initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+        initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
         final_data = dict()
         h_video_ids = []
         for video_id, score in initial_data:

rule_rank_h_18_19.py

@@ -162,7 +162,7 @@ def h_rank_bottom(app_type, now_date, now_h):
         redis_dt = datetime.datetime.strftime(now_date, '%Y%m%d')
         redis_h = now_h - 1
     key_name = f"{config_.RECALL_KEY_NAME_PREFIX_APP_TYPE}{app_type}.{redis_dt}.{redis_h}"
-    initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+    initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
     final_data = dict()
     for video_id, score in initial_data:
         final_data[video_id] = score

rule_rank_h_by_24h.py

@@ -132,7 +132,7 @@ def video_rank_h(df, now_date, now_h, rule_key, param):
     # 获取rov模型结果
     redis_helper = RedisHelper()
     key_name = get_rov_redis_key(now_date=now_date)
-    initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+    initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
     log_.info(f'initial data count = {len(initial_data)}')
 
     # 获取符合进入召回源条件的视频
@@ -219,7 +219,7 @@ def h_rank_bottom(now_date, now_h, rule_key):
     key_prefix_list = [config_.RECALL_KEY_NAME_PREFIX_BY_24H, config_.RECALL_KEY_NAME_PREFIX_DUP_24H]
     for key_prefix in key_prefix_list:
         key_name = f"{key_prefix}{rule_key}.{redis_dt}.{redis_h}"
-        initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+        initial_data = redis_helper.get_all_data_from_zset(key_name=key_name, with_scores=True)
         final_data = dict()
         for video_id, score in initial_data:
             final_data[video_id] = score

videos_filter.py

@@ -122,7 +122,7 @@ def filter_rov_pool(app_type=None):
 
     # 获取视频
     redis_helper = RedisHelper()
-    data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+    data = redis_helper.get_all_data_from_zset(key_name=key_name)
     if data is None:
         log_.info("data is None")
         log_.info("rov recall pool filter end!")
@@ -160,7 +160,7 @@ def filter_flow_pool():
             key_name = get_pool_redis_key(pool_type='flow', app_type=app_type)
             # 获取视频
             redis_helper = RedisHelper()
-            data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+            data = redis_helper.get_all_data_from_zset(key_name=key_name)
             if data is None:
                 log_.info("data is None")
                 log_.info("app_type {} videos filter end!".format(app_type))
@@ -208,7 +208,7 @@ def filter_flow_pool_18_19(app_type):
     key_name = get_pool_redis_key(pool_type='flow', app_type=app_type)
     # 获取视频
     redis_helper = RedisHelper()
-    data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+    data = redis_helper.get_all_data_from_zset(key_name=key_name)
     if data is None:
         log_.info("data is None")
         log_.info("app_type {} videos filter end!".format(app_type))
@@ -240,7 +240,7 @@ def filter_bottom():
     log_.info("bottom videos filter start ...")
     # 获取视频
     redis_helper = RedisHelper()
-    data = redis_helper.get_data_zset_with_index(key_name=config_.BOTTOM_KEY_NAME, start=0, end=-1)
+    data = redis_helper.get_all_data_from_zset(key_name=config_.BOTTOM_KEY_NAME)
     if data is None:
         log_.info("data is None")
         log_.info("bottom videos filter end!")
@@ -266,7 +266,7 @@ def filter_rov_updated():
     log_.info("update rov videos filter start ...")
     # 获取视频
     redis_helper = RedisHelper()
-    data = redis_helper.get_data_zset_with_index(key_name=config_.UPDATE_ROV_KEY_NAME, start=0, end=-1)
+    data = redis_helper.get_all_data_from_zset(key_name=config_.UPDATE_ROV_KEY_NAME)
     if data is None:
         log_.info("data is None")
         log_.info("update rov videos filter end!")
@@ -293,7 +293,7 @@ def filter_rov_updated_app():
     log_.info("update rov videos app filter start ...")
     # 获取视频
     redis_helper = RedisHelper()
-    data = redis_helper.get_data_zset_with_index(key_name=config_.UPDATE_ROV_KEY_NAME_APP, start=0, end=-1)
+    data = redis_helper.get_all_data_from_zset(key_name=config_.UPDATE_ROV_KEY_NAME_APP)
     if data is None:
         log_.info("data is None")
         log_.info("update rov videos app filter end!")
@@ -413,7 +413,7 @@ def filter_app_pool():
     # 拼接key
     key_name = f'{config_.APP_FINAL_RECALL_KEY_NAME_PREFIX}{redis_date}.{redis_h}'
     # 获取视频
-    data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+    data = redis_helper.get_all_data_from_zset(key_name=key_name)
     if data is None:
         log_.info("data is None")
         log_.info("app pool filter end!")
@@ -458,7 +458,7 @@ def filter_rov_h():
             key_name = f"{key_prefix}{key}.{now_date}.{now_h}"
             log_.info(f"key_name: {key_name}")
             # 获取视频
-            data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+            data = redis_helper.get_all_data_from_zset(key_name=key_name)
             if data is None:
                 log_.info("data is None")
                 log_.info("filter end!")
@@ -586,7 +586,7 @@ def filter_region_videos():
                 key_name = f"{key_prefix}{region}.{key}.{now_date}.{now_h}"
                 log_.info(f"key_name: {key_name}")
                 # 获取视频
-                data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+                data = redis_helper.get_all_data_from_zset(key_name=key_name)
                 if data is None:
                     log_.info("data is None")
                     log_.info("filter end!")
@@ -644,7 +644,7 @@ def filter_region_videos_by_day():
                 key_name = f"{key_prefix}{region}.{key}.{now_date}"
                 log_.info(f"key_name: {key_name}")
                 # 获取视频
-                data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+                data = redis_helper.get_all_data_from_zset(key_name=key_name)
                 if data is None:
                     log_.info("data is None")
                     log_.info("filter end!")
@@ -686,7 +686,7 @@ def filter_rov_h_24h():
             key_name = f"{key_prefix}{key}.{now_date}.{now_h}"
             log_.info(f"key_name: {key_name}")
             # 获取视频
-            data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+            data = redis_helper.get_all_data_from_zset(key_name=key_name)
             if data is None:
                 log_.info("data is None")
                 log_.info("filter end!")
@@ -738,7 +738,7 @@ def filter_region_videos_24h():
                 key_name = f"{key_prefix}{region}.{key}.{now_date}.{now_h}"
                 log_.info(f"key_name: {key_name}")
                 # 获取视频
-                data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+                data = redis_helper.get_all_data_from_zset(key_name=key_name)
                 if data is None:
                     log_.info("data is None")
                     log_.info("filter end!")
@@ -812,7 +812,7 @@ def filter_whole_movies():
     # 拼接key
     key_name = f'{config_.RECALL_KEY_NAME_PREFIX_WHOLE_MOVIES}{now_date}.{now_h}'
     # 获取视频
-    data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+    data = redis_helper.get_all_data_from_zset(key_name=key_name)
     if data is None:
         log_.info("data is None")
         log_.info("whole movies filter end!")