Merge branch 'feature/zhangbo_recall' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV563.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.FeatureService;
@@ -11,7 +12,9 @@ import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.math3.util.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.BufferedReader;
@@ -19,6 +22,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Service
@@ -33,6 +39,9 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
     Map<String, double[]> bucketsMap = new HashMap<>();
     Map<String, Double> bucketsLen = new HashMap<>();
 
+    @Value("${similarity.concurrent: false}")
+    private boolean similarityConcurrent;
+
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
@@ -63,7 +72,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         rovRecallRank.addAll(v6);
         setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
         //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_sort.PUSH_FORM);
         v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
         v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
         rovRecallRank.addAll(v1);
@@ -221,14 +230,41 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
 
             String title = videoInfo.getOrDefault("title", "");
             if (!title.isEmpty()) {
-                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
-                        if (!tags.isEmpty()) {
-                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
-                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
-                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
+                if (similarityConcurrent) {
+                    List<Future<Pair<String, Double[]>>> futures = new ArrayList<>();
+                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                            String key = name + "_" + key_time;
+                            String tags = c34567Map.getOrDefault(key, "");
+                            if (!tags.isEmpty()) {
+                                Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
+                                    Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                                    return Pair.create(key, doubles);
+                                });
+                                futures.add(future);
+                            }
+                        }
+                    }
+                    try {
+                        for (Future<Pair<String, Double[]>> future : futures) {
+                            Pair<String, Double[]> pair = future.get(1000, TimeUnit.MILLISECONDS);
+                            featureMap.put(pair.getFirst() + "_matchnum", pair.getSecond()[0]);
+                            featureMap.put(pair.getFirst() + "_maxscore", pair.getSecond()[1]);
+                            featureMap.put(pair.getFirst() + "_avgscore", pair.getSecond()[2]);
+                        }
+                    } catch (Exception e) {
+                        log.error("concurrent similarity error", e);
+                    }
+                } else {
+                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                            String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                            if (!tags.isEmpty()) {
+                                Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                                featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                                featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                                featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
+                            }
                         }
                     }
                 }
@@ -295,34 +331,30 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         // 3 排序
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
 
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_xgb_20240828.conf")
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240807.conf")
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
-        Map<String, Map<String, String>> vid2VovFeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vov_1d3d:");
-        double alpha_vov = mergeWeight.getOrDefault("alpha_vov", 2.0);
+        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
         List<Video> result = new ArrayList<>();
-//        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
-//        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
+        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
 
         for (RankItem item : items) {
             double score = 0.0;
-            double recommend_rate_1d = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("recommend_rate_1d", "0"));
-            double recommend_exp_per_1d = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("recommend_exp_per_1d", "0"));
-            double vorScore =  recommend_rate_1d * recommend_exp_per_1d;
-            item.getScoresMap().put("recommend_rate_1d", recommend_rate_1d);
-            item.getScoresMap().put("recommend_exp_per_1d", recommend_exp_per_1d);
-            item.getScoresMap().put("vorScore", vorScore);
-            item.getScoresMap().put("alpha_vov", alpha_vov);
             double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("rate_n", "0"));
+                    .getOrDefault(hasReturnRovKey, "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double rovRateOrigin = item.getScoreRov();
-            item.getScoresMap().put("rovRateOrigin", rovRateOrigin);
-            double rovRate = restoreScore(rovRateOrigin);
-            item.getScoresMap().put("rovRate", rovRate);
-            score = rovRate * (1 + hasReturnRovScore) * (1.0 + alpha_vov * recommend_rate_1d);
+            double fmRovOrigin = item.getScoreRov();
+            item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
+            double fmRov = restoreScore(fmRovOrigin);
+            item.getScoresMap().put("fmRov", fmRov);
+            if (chooseFunction == 0){
+                score = fmRov * (1 + hasReturnRovScore);
+            }else if (chooseFunction == 1){
+                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
+            }else {
+                score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
+            }
 
             Video video = item.getVideo();
             video.setScore(score);
@@ -346,7 +378,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
     }
 
     private void readBucketFile() {
-        InputStream resourceStream = RankStrategy4RegionMergeModelV563.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        InputStream resourceStream = RankStrategy4RegionMergeModelV552.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
         if (resourceStream != null) {
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
                 Map<String, double[]> bucketsMap = new HashMap<>();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -172,7 +172,7 @@ public class RecallService implements ApplicationContextAware {
             case "60105": // 551
             case "60106": // 552
             case "60112": // 562
-            case "60113": // 563
+
             case "60114": // 564
             case "60115": // 565
             case "60119": // 569
@@ -180,6 +180,11 @@ public class RecallService implements ApplicationContextAware {
                 strategies.addAll(getRegionRecallStrategy(param));
                 strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                 break;
+            case "60113": // 563
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1_sort.class.getSimpleName()));
+                strategies.addAll(getRegionRecallStrategy(param));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                break;
             default:
                 strategies.addAll(getRegionRecallStrategy(param));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV1_sort.java

@@ -40,7 +40,7 @@ public class RegionRealtimeRecallStrategyV1_sort implements RecallStrategy {
         param4Model.put("region_province", provinceCn);
         param4Model.put("sort", "1");
         // 2 通过model拿到召回list
-        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_region_v1.conf");
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_region_v1_vov.conf");
         List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
         List<Pair<Long, Double>> result = results.get(0);
         for (int i = 1; i < results.size(); ++i) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -40,6 +40,7 @@ public final class ScorerUtils {
         ScorerUtils.init("feeds_score_config_20240807.conf");
         ScorerUtils.init("feeds_score_config_xgb_20240828.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v1.conf");
+        ScorerUtils.init4Recall("feeds_recall_config_region_v1_vov.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v2.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v3.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v4.conf");

recommend-server-service/src/main/resources/feeds_recall_config_region_v1_vov.conf

@@ -0,0 +1,8 @@
+scorer-config = {
+    score1-config = {
+        scorer-name = "com.tzld.piaoquan.recommend.server.service.score4recall.strategy.RegionRecallScorerV1"
+        scorer-priority = 99
+        model-path = "alg_recall_file/01_recall_region1hour_vov.txt"
+    }
+
+}