update rank & add app videos filter

app_rank_h.py

@@ -60,15 +60,32 @@ def get_redis_key_date(now_date):
     return key_name
 
 
+def app_rank_bottom(now_date, now_h):
+    """运营未按时更新数据，用rov模型结果作为当前小时的数据"""
+    # 获取rov模型结果
+    redis_helper = RedisHelper()
+    key_name = get_redis_key_date(now_date=now_date)
+    initial_data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1, with_scores=True)
+    final_data = dict()
+    for video_id, score in initial_data:
+        final_data[video_id] = score
+    # 存入对应的redis
+    final_key_name = f"{config_.APP_FINAL_RECALL_KEY_NAME_PREFIX}{dt.strftime(now_date, '%Y%m%d')}.{now_h}"
+    redis_helper.add_data_with_zset(key_name=final_key_name, data=final_data, expire_time=24 * 3600)
+
+
 def app_timer_check():
     now_date = dt.today()
     now_h = dt.now().hour
+    now_min = dt.now().minute
     # 查看当前小时op更新的数据是否已准备好
     op_key_name = f"{config_.APP_OP_VIDEOS_KEY_NAME_PREFIX}{dt.strftime(now_date, '%Y%m%d')}.{now_h}"
     redis_helper = RedisHelper()
     if redis_helper.key_exists(op_key_name):
         # 数据准备好，进行更新
         app_rank_op(now_date=now_date, now_h=now_h)
+    elif now_min > 50:
+        app_rank_bottom(now_date=now_date, now_h=now_h)
     else:
         # 数据没准备好，1分钟后重新检查
         Timer(60, app_timer_check).start()

videos_filter.py

@@ -319,6 +319,37 @@ def get_pool_redis_key(pool_type, app_type=None):
         return None, None
 
 
+def filter_app_pool():
+    """过滤票圈视频APP小时级数据"""
+    log_.info("app pool filter start ...")
+    redis_helper = RedisHelper()
+    # 获取当前日期
+    now_date = date.today().strftime('%Y%m%d')
+    # 获取当前所在小时
+    now_h = datetime.now().hour
+    # 拼接key
+    key_name = f'{config_.APP_FINAL_RECALL_KEY_NAME_PREFIX}{now_date}.{now_h}'
+    # 获取视频
+    data = redis_helper.get_data_zset_with_index(key_name=key_name, start=0, end=-1)
+    if data is None:
+        log_.info("data is None")
+        log_.info("app pool filter end!")
+        return
+    # 过滤
+    video_ids = [int(video_id) for video_id in data]
+    filtered_result = filter_video_status_app(video_ids=video_ids)
+    # 求差集，获取需要过滤掉的视频，并从redis中移除
+    filter_videos = set(video_ids) - set(filtered_result)
+    log_.info("video_ids size = {}, filtered size = {}, filter sizer = {}".format(len(video_ids),
+                                                                                  len(filtered_result),
+                                                                                  len(filter_videos)))
+    if len(filter_videos) == 0:
+        log_.info("app pool filter end!")
+        return
+    redis_helper.remove_value_from_zset(key_name=key_name, value=list(filter_videos))
+    log_.info("app pool filter end!")
+
+
 def main():
     try:
         # ROV召回池视频过滤
@@ -338,6 +369,8 @@ def main():
         filter_relevant_videos()
         # 按位置排序视频过滤
         filter_position_videos()
+        # 过滤票圈视频APP小时级数据
+        filter_app_pool()
     except Exception as e:
         log_.error(traceback.format_exc())
         send_msg_to_feishu(