update

config.py

@@ -74,6 +74,8 @@ class BaseConfig(object):
     RECALL_KEY_NAME_PREFIX = 'com.weiqu.video.recall.hot.item.score.'
     # 小程序小时级更新结果存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.h.{date}.{h}
     RECALL_KEY_NAME_PREFIX_BY_H = 'com.weiqu.video.recall.hot.item.score.h.'
+    # 小程序离线ROV模型结果与小程序小时级更新结果去重后 存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.dup.h.{date}.{h}
+    RECALL_KEY_NAME_PREFIX_DUP_H = 'com.weiqu.video.recall.hot.item.score.dup.h.'
 
     # app应用 小程序离线ROV模型结果存放 redis key前缀，完整格式：com.weiqu.video.recall.hot.item.score.app.{date}
     RECALL_KEY_NAME_PREFIX_APP = 'com.weiqu.video.recall.hot.item.score.app.'

rule_rank_h.py

@@ -112,22 +112,40 @@ def video_rank(df, now_date, now_h):
     h_recall_df = df[(df['lastonehour_return'] >= 20) & (df['score'] >= 0.005)]
     h_recall_videos = h_recall_df['videoid'].to_list()
     log_.info(f'h_recall videos count = {len(h_recall_videos)}')
-
-    # 去重合并
-    final_videos = [int(item) for item in h_recall_videos]
-    temp_videos = [int(video_id) for video_id, _ in initial_data if int(video_id) not in final_videos]
-    final_videos = final_videos + temp_videos
-    log_.info(f'final videos count = {len(final_videos)}')
-
-    # 重新给定score
-    final_data = {}
-    for i, video_id in enumerate(final_videos):
-        score = 100 - i * config_.ROV_SCORE_D
-        final_data[video_id] = score
-
-    # 存入对应的redis
-    final_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
-    redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=24 * 3600)
+    # 写入对应的redis
+    h_video_ids =[]
+    h_recall_result = {}
+    for video_id in h_recall_videos:
+        score = h_recall_df[h_recall_df['videoid'] == video_id]
+        h_recall_result[int(video_id)] = float(score)
+        h_video_ids.append(int(video_id))
+    h_recall_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+    redis_helper.add_data_with_zset(key_name=h_recall_key_name, data=h_recall_result, expire_time=24 * 3600)
+
+    # 去重更新rov模型结果，并另存为redis中
+    initial_data_dup = {}
+    for video_id, score in initial_data:
+        if int(video_id) not in h_video_ids:
+            initial_data_dup[int(video_id)] = score
+    initial_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_DUP_H}{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+    redis_helper.add_data_with_zset(key_name=initial_key_name, data=initial_data_dup, expire_time=24 * 3600)
+
+
+    # # 去重合并
+    # final_videos = [int(item) for item in h_recall_videos]
+    # temp_videos = [int(video_id) for video_id, _ in initial_data if int(video_id) not in final_videos]
+    # final_videos = final_videos + temp_videos
+    # log_.info(f'final videos count = {len(final_videos)}')
+    #
+    # # 重新给定score
+    # final_data = {}
+    # for i, video_id in enumerate(final_videos):
+    #     score = 100 - i * config_.ROV_SCORE_D
+    #     final_data[video_id] = score
+    #
+    # # 存入对应的redis
+    # final_key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{datetime.datetime.strftime(now_date, '%Y%m%d')}.{now_h}"
+    # redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=24 * 3600)
 
 
 def rank_by_h(now_date, now_h):

videos_filter.py

@@ -374,26 +374,30 @@ def filter_rov_h():
     # 获取当前所在小时
     now_h = datetime.now().hour
     log_.info(f'now_date = {now_date}, now_h = {now_h}.')
-    # 拼接key
-    key_name = f"{config_.RECALL_KEY_NAME_PREFIX_BY_H}{now_date}.{now_h}"
-    # 获取视频
-    data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
-    if data is None:
-        log_.info("data is None")
-        log_.info("rov_h pool filter end!")
-        return
-    # 过滤
-    video_ids = [int(video_id) for video_id in data]
-    filtered_result = filter_video_status(video_ids=video_ids)
-    # 求差集，获取需要过滤掉的视频，并从redis中移除
-    filter_videos = set(video_ids) - set(filtered_result)
-    log_.info("video_ids size = {}, filtered size = {}, filter sizer = {}".format(len(video_ids),
-                                                                                  len(filtered_result),
-                                                                                  len(filter_videos)))
-    if len(filter_videos) == 0:
-        log_.info("rov_h pool filter end!")
-        return
-    redis_helper.remove_value_from_zset(key_name=key_name, value=list(filter_videos))
+    # 需过滤两个视频列表
+    key_prefix_list = [config_.RECALL_KEY_NAME_PREFIX_BY_H, config_.RECALL_KEY_NAME_PREFIX_DUP_H]
+    for key_prefix in key_prefix_list:
+        # 拼接key
+        key_name = f"{key_prefix}{now_date}.{now_h}"
+        log_.info(f"key_name: {key_name}")
+        # 获取视频
+        data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+        if data is None:
+            log_.info("data is None")
+            log_.info("filter end!")
+            continue
+        # 过滤
+        video_ids = [int(video_id) for video_id in data]
+        filtered_result = filter_video_status(video_ids=video_ids)
+        # 求差集，获取需要过滤掉的视频，并从redis中移除
+        filter_videos = set(video_ids) - set(filtered_result)
+        log_.info("video_ids size = {}, filtered size = {}, filter sizer = {}".format(len(video_ids),
+                                                                                      len(filtered_result),
+                                                                                      len(filter_videos)))
+        if len(filter_videos) == 0:
+            log_.info("filter end!")
+            continue
+        redis_helper.remove_value_from_zset(key_name=key_name, value=list(filter_videos))
     log_.info("rov_h pool filter end!")