Merge branch 'feature/scene_cf_recall' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV563.java

@@ -31,6 +31,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
     @Autowired
     private FeatureService featureService;
 
+
     @Value("${similarity.concurrent: true}")
     private boolean similarityConcurrent;
 
@@ -69,6 +70,18 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
         rovRecallRank.addAll(v1);
         setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------scene cf rovn------------------
+        List<Video> sceneCFRovn = extractAndSort(param, SceneCFRovnRecallStrategy.PUSH_FORM);
+        sceneCFRovn = sceneCFRovn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        sceneCFRovn = sceneCFRovn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), sceneCFRovn.size()));
+        rovRecallRank.addAll(sceneCFRovn);
+        setVideo.addAll(sceneCFRovn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------scene cf rosn------------------
+        List<Video> sceneCFRosn = extractAndSort(param, SceneCFRosnRecallStrategy.PUSH_FORM);
+        sceneCFRosn = sceneCFRosn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
+        rovRecallRank.addAll(sceneCFRosn);
+        setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -311,7 +324,6 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
                 userFeatureMap.put(name, String.valueOf(scoreNew));
             }
         }
-
         for (RankItem item : rankItems) {
             Map<String, String> featureMap = new HashMap<>();
             Map<String, Double> featureMapDouble = item.featureMapDouble;
@@ -329,57 +341,32 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
             }
             item.featureMap = featureMap;
         }
-
-
-        // vovh24特征
-        String partition = redisTemplate.opsForValue().get("redis:vid_vovh24pred_time:partition");
-        Map<String, Map<String, String>> vid2VovFeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vovh24pred_time:" + partition + ":");
-        for (RankItem rankItem : rankItems) {
-            if (vid2VovFeatureMap.containsKey(String.valueOf(rankItem.getVideoId()))) {
-                rankItem.getFeatureMap().putAll(vid2VovFeatureMap.get(String.valueOf(rankItem.getVideoId())));
-            }
-        }
-
         // 4 排序模型计算
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
-        if (null != partition && partition.length() > 2) {
-            sceneFeatureMap.put("weightKey", partition.substring(partition.length() - 2));
-        }
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20241107.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240807.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
         // 5 排序公式特征
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
-        double alpha_vov = mergeWeight.getOrDefault("alpha_vov", 0.1);
-        double func = mergeWeight.getOrDefault("func", 1.0);
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
         List<Video> result = new ArrayList<>();
         for (RankItem item : items) {
-            item.getScoresMap().put("alpha_vov", alpha_vov);
-            double score = 0.0;
+            double score;
             double fmRovOrigin = item.getScoreRov();
             item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
             double fmRov = restoreScore(fmRovOrigin);
             item.getScoresMap().put("fmRov", fmRov);
-            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rate_n", "0"));
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double vovScore = item.getVovScore();
-            item.getScoresMap().put("vovScore", vovScore);
-            if (func == 1) {
-                score = fmRov * (1 + hasReturnRovScore) + alpha_vov * vovScore;
-            } else {
-                score = fmRov * (1 + hasReturnRovScore) * (1.0 + alpha_vov * vovScore);
-            }
-
+            double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
+            item.getScoresMap().put("vor", vor);
+            score = fmRov * (0.1 + hasReturnRovScore) * (0.1 + vor);
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
             video.setScoresMap(item.getScoresMap());
             video.setAllFeatureMap(item.getAllFeatureMap());
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getVideoFeature())
-                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+            if (MapUtils.isNotEmpty(feature.getVideoFeature()) && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
                 video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
             }
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+            if (MapUtils.isNotEmpty(feature.getUserFeature())) {
                 video.getMetaFeatureMap().putAll(feature.getUserFeature());
             }
             result.add(video);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -8,8 +8,6 @@ import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.filter.strategy.BlacklistContainer;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -166,7 +164,6 @@ public class RecallService implements ApplicationContextAware {
             case "60105": // 551
             case "60106": // 552
             case "60112": // 562
-            case "60113": // 563
             case "60117": // 567
             case "60114": // 564
             case "60115": // 565
@@ -175,6 +172,12 @@ public class RecallService implements ApplicationContextAware {
                 strategies.addAll(getRegionRecallStrategy(param));
                 strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                 break;
+            case "60113": // 563
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+                strategies.addAll(getRegionRecallStrategy(param));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(SceneCFRovnRecallStrategy.class.getSimpleName()));
+                strategies.add(strategyMap.get(SceneCFRosnRecallStrategy.class.getSimpleName()));
             default:
                 strategies.addAll(getRegionRecallStrategy(param));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/SceneCFRosnRecallStrategy.java

@@ -0,0 +1,111 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class SceneCFRosnRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private RegionFilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String PUSH_FORM = "recall_strategy_scene_cf_rosn";
+    public static final String redisKeyPrefix = "redis:scene_cf_recall:rosn";
+    public static final Set<Long> hotSceneSet = new HashSet<>(Arrays.asList(1008L, 1007L, 1058L, 1074L, 1010L));
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        Long headVid = param.getVideoId();
+        String redisKey = getRedisKey(param);
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            // 1 获取头部vid，请求redis得到vid & score。
+            String redisValue = redisTemplate.opsForValue().get(redisKey);
+            if (null == redisValue || redisValue.isEmpty()) {
+                return videosResult;
+            }
+            Map<Long, Double> vid2Score = getIdScoreMap(headVid, redisValue);
+            List<Long> vids = new ArrayList<>(vid2Score.keySet());
+            FilterParam filterParam = FilterParamFactory.create(param, vids);
+            filterParam.setForceTruncation(10000);
+            filterParam.setConcurrent(true);
+            filterParam.setNotUsePreView(false);
+            FilterResult filterResult = filterService.filter(filterParam);
+            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                filterResult.getVideoIds().forEach(vid -> {
+                    Video video = new Video();
+                    video.setVideoId(vid);
+                    video.setAbCode(param.getAbCode());
+                    video.setRovScore(vid2Score.getOrDefault(vid, 0.0D));
+                    video.setPushFrom(pushFrom());
+                    videosResult.add(video);
+                });
+            }
+
+            // 5 内部日志打印
+
+            // 6 返回结果
+            videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+        } catch (Exception e) {
+            log.error(String.format("json parse is wrong in {}, key={}, error={}", CLASS_NAME, redisKey, e));
+        }
+        return videosResult;
+    }
+
+    private Map<Long, Double> getIdScoreMap(Long headVid, String data) {
+        Map<Long, Double> idScoreMap = new HashMap<>();
+        if (null != data) {
+            String[] pair = data.split("\t");
+            if (2 == pair.length) {
+                List<Long> ids = Arrays.stream(pair[0].split(","))
+                        .map(Long::valueOf).collect(Collectors.toList());
+                List<Double> scores = Arrays.stream(pair[1].split(","))
+                        .map(Double::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty() && ids.size() == scores.size()) {
+                    for (int i = 0; i < ids.size(); ++i) {
+                        Long id = ids.get(i);
+                        if (id.equals(headVid)) {
+                            continue;
+                        }
+                        Double score = scores.get(i);
+                        idScoreMap.put(id, score);
+                    }
+                }
+            }
+        }
+        return idScoreMap;
+    }
+
+    private String getRedisKey(RecallParam param) {
+        Long scene = param.getHotSceneType();
+        Long headVid = param.getVideoId();
+        String sceneTypeStr = "other";
+        if (null != scene && hotSceneSet.contains(scene)) {
+            sceneTypeStr = String.valueOf(scene);
+        }
+        return String.format("%s:%s:%d", redisKeyPrefix, sceneTypeStr, headVid);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/SceneCFRovnRecallStrategy.java

@@ -0,0 +1,112 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class SceneCFRovnRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private RegionFilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String PUSH_FORM = "recall_strategy_scene_cf_rovn";
+    public static final String redisKeyPrefix = "redis:scene_cf_recall:rovn";
+    public static final Set<Long> hotSceneSet = new HashSet<>(Arrays.asList(1008L, 1007L, 1058L, 1074L, 1010L));
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        Long headVid = param.getVideoId();
+        String redisKey = getRedisKey(param);
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            // 1 获取头部vid，请求redis得到vid & score。
+            String redisValue = redisTemplate.opsForValue().get(redisKey);
+            if (null == redisValue || redisValue.isEmpty()) {
+                return videosResult;
+            }
+            Map<Long, Double> vid2Score = getIdScoreMap(headVid, redisValue);
+            List<Long> vids = new ArrayList<>(vid2Score.keySet());
+            FilterParam filterParam = FilterParamFactory.create(param, vids);
+            filterParam.setForceTruncation(10000);
+            filterParam.setConcurrent(true);
+            filterParam.setNotUsePreView(false);
+            FilterResult filterResult = filterService.filter(filterParam);
+            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                filterResult.getVideoIds().forEach(vid -> {
+                    Video video = new Video();
+                    video.setVideoId(vid);
+                    video.setAbCode(param.getAbCode());
+                    video.setRovScore(vid2Score.getOrDefault(vid, 0.0D));
+                    video.setPushFrom(pushFrom());
+                    videosResult.add(video);
+                });
+            }
+
+            // 5 内部日志打印
+
+            // 6 返回结果
+            videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+        } catch (Exception e) {
+            log.error(String.format("json parse is wrong in {}, key={}, error={}", CLASS_NAME, redisKey, e));
+        }
+        return videosResult;
+    }
+
+    private Map<Long, Double> getIdScoreMap(Long headVid, String data) {
+        Map<Long, Double> idScoreMap = new HashMap<>();
+        if (null != data) {
+            String[] pair = data.split("\t");
+            if (2 == pair.length) {
+                List<Long> ids = Arrays.stream(pair[0].split(","))
+                        .map(Long::valueOf).collect(Collectors.toList());
+                List<Double> scores = Arrays.stream(pair[1].split(","))
+                        .map(Double::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty() && ids.size() == scores.size()) {
+                    for (int i = 0; i < ids.size(); ++i) {
+                        Long id = ids.get(i);
+                        if (id.equals(headVid)) {
+                            continue;
+                        }
+                        Double score = scores.get(i);
+                        idScoreMap.put(id, score);
+                    }
+                }
+            }
+        }
+        return idScoreMap;
+    }
+
+
+    private String getRedisKey(RecallParam param) {
+        Long scene = param.getHotSceneType();
+        Long headVid = param.getVideoId();
+        String sceneTypeStr = "other";
+        if (null != scene && hotSceneSet.contains(scene)) {
+            sceneTypeStr = String.valueOf(scene);
+        }
+        return String.format("%s:%s:%d", redisKeyPrefix, sceneTypeStr, headVid);
+    }
+}