Merge branch 'feature/fitting_vov_v2' of algorithm/recommend-server into master

拟合vovh24(第二版)

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -126,12 +126,6 @@ public abstract class AbstractFilterService {
                 strategies.add(ServiceBeanFactory.getBean(TagStrategy.class));
                 break;
         }
-
-        // VOV过滤实验
-        if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("564")) {
-            strategies.add(ServiceBeanFactory.getBean(VovLowerStrategy.class));
-        }
-
         if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("697")) {
             strategies.add(ServiceBeanFactory.getBean(VideoSourceTypeStrategy.class));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -84,7 +84,7 @@ public class RankRouter {
             case "60113": // 563
                 return rankStrategy4RegionMergeModelV563.rank(param);
             case "60114": // 564
-                return rankStrategy4RegionMergeModelV552.rank(param);
+                return rankStrategy4RegionMergeModelV564.rank(param);
             case "60115": // 565
                 return rankStrategy4RegionMergeModelV565.rank(param);
             case "60116": // 566

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV563.java

@@ -342,7 +342,9 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
 
         // 4 排序模型计算
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
-        sceneFeatureMap.put("weightKey", partition.substring(partition.length() - 2));
+        if (null != partition && partition.length() > 2) {
+            sceneFeatureMap.put("weightKey", partition.substring(partition.length() - 2));
+        }
         List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20241107.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
         // 5 排序公式特征
         Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV564.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.FeatureService;
@@ -8,17 +9,18 @@ import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score.VovH24Weight562Scorer;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.math3.util.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.*;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Service
@@ -27,15 +29,23 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
     @ApolloJsonValue("${rank.score.merge.weightv564:}")
     private Map<String, Double> mergeWeight;
 
+    @ApolloJsonValue("${rank.score.merge.weightv562:}")
+    private Map<String, Double> mergeWeight562;
+
+    @ApolloJsonValue("${rank.score.merge.weightv567:}")
+    private Map<String, Double> mergeWeight567;
+
     @Autowired
     private FeatureService featureService;
 
-    Map<String, double[]> bucketsMap = new HashMap<>();
-    Map<String, Double> bucketsLen = new HashMap<>();
+    @Value("${similarity.concurrent: true}")
+    private boolean similarityConcurrent;
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
+        Map<String, Double> mergeWeight562 = this.mergeWeight562 != null ? this.mergeWeight562 : new HashMap<>(0);
+        Map<String, Double> mergeWeight567 = this.mergeWeight567 != null ? this.mergeWeight567 : new HashMap<>(0);
         //-------------------融-------------------
         //-------------------合-------------------
         //-------------------逻-------------------
@@ -58,24 +68,23 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
         List<Video> rovRecallRank = new ArrayList<>(v0);
         //-------------------return相似召回------------------
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
         v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
         rovRecallRank.addAll(v6);
         setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
         //-------------------新地域召回------------------
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
         v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
         rovRecallRank.addAll(v1);
         setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
-
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        // 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
         List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
 
         // k1:视频、k2:表、k3:特征、v:特征值
@@ -87,7 +96,7 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
         Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
 
 
-        // TODO 2 特征处理
+        // 2 特征处理
         Map<String, Double> userFeatureMapDouble = new HashMap<>();
         String mid = param.getMid();
         Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
@@ -95,8 +104,8 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
         Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
         Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
         Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
-        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags", new HashMap<>());
-        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
         Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
         Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
 
@@ -166,21 +175,21 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
         for (RankItem item : rankItems) {
             Map<String, Double> featureMap = new HashMap<>();
             String vid = item.getVideoId() + "";
-            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp", new HashMap<>());
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
             Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
             Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
-            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
             Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
 
-            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp", new HashMap<>());
-            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share", new HashMap<>());
-            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return", new HashMap<>());
-            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp", new HashMap<>());
-            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share", new HashMap<>());
-            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return", new HashMap<>());
-            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp", new HashMap<>());
-            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share", new HashMap<>());
-            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return", new HashMap<>());
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
 
             List<Tuple4> originData = Arrays.asList(
                     new Tuple4(b1, b2, b3, "b123"),
@@ -201,21 +210,18 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
                     double f3 = ExtractorUtils.calDiv(returns, exp);
                     double f4 = ExtractorUtils.calLog(returns);
                     double f5 = f3 * f4;
-                    double f6 = ExtractorUtils.calDiv(returns, share);
 
                     String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
                     String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
                     String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
                     String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
                     String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
-                    String key6 = tuple4.name + "_" + prefix2 + "_" + "ROS";
 
                     featureMap.put(key1, f1);
                     featureMap.put(key2, f2);
                     featureMap.put(key3, f3);
                     featureMap.put(key4, f4);
                     featureMap.put(key5, f5);
-                    featureMap.put(key6, f6);
                 }
             }
 
@@ -225,14 +231,51 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
 
             String title = videoInfo.getOrDefault("title", "");
             if (!title.isEmpty()) {
-                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
-                        if (!tags.isEmpty()) {
-                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
-                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
-                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
+                if (similarityConcurrent) {
+                    List<Future<Pair<String, Double[]>>> futures = new ArrayList<>();
+                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                            String key = name + "_" + key_time;
+                            String tags = c34567Map.getOrDefault(key, "");
+                            if (!tags.isEmpty()) {
+                                Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
+                                    Double[] doubles = null;
+                                    if (param.getAbExpCodes().contains(word2vecExp)) {
+                                        doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
+                                    } else {
+                                        doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                                    }
+                                    return Pair.create(key, doubles);
+                                });
+                                futures.add(future);
+                            }
+                        }
+                    }
+                    try {
+                        for (Future<Pair<String, Double[]>> future : futures) {
+                            Pair<String, Double[]> pair = future.get(1000, TimeUnit.MILLISECONDS);
+                            featureMap.put(pair.getFirst() + "_matchnum", pair.getSecond()[0]);
+                            featureMap.put(pair.getFirst() + "_maxscore", pair.getSecond()[1]);
+                            featureMap.put(pair.getFirst() + "_avgscore", pair.getSecond()[2]);
+                        }
+                    } catch (Exception e) {
+                        log.error("concurrent similarity error", e);
+                    }
+                } else {
+                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                            String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                            if (!tags.isEmpty()) {
+                                Double[] doubles = null;
+                                if (param.getAbExpCodes().contains(word2vecExp)) {
+                                    doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
+                                } else {
+                                    doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                                }
+                                featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                                featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                                featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
+                            }
                         }
                     }
                 }
@@ -254,7 +297,7 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
                     }
                 }
             }
-            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
             if (!d1.isEmpty()) {
                 featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
                 featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
@@ -296,33 +339,57 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
             item.featureMap = featureMap;
         }
 
-        // TODO 3 排序
-        Map<String, String> sceneFeatureMap = new HashMap<>(0);
 
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240711.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
-        List<Video> result = new ArrayList<>();
-        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
-        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
-        Double rosDefault = mergeWeight.getOrDefault("rosDefault", 1.0);
+        // vovh24特征
+        String partition = redisTemplate.opsForValue().get("redis:vid_vovh24pred_time:partition");
+        Map<String, Map<String, String>> vid2VovFeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vovh24pred_time:" + partition + ":");
+        for (RankItem rankItem : rankItems) {
+            if (vid2VovFeatureMap.containsKey(String.valueOf(rankItem.getVideoId()))) {
+                rankItem.getFeatureMap().putAll(vid2VovFeatureMap.get(String.valueOf(rankItem.getVideoId())));
+            }
+        }
 
+        // 4 排序模型计算
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
+        if (null != partition && partition.length() > 2) {
+            sceneFeatureMap.put("weightKey", partition.substring(partition.length() - 2));
+        }
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20241119.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
+
+        // 562 vovScore
+        Map<String, Map<String, String>> vov562FeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vovhour4rank:");
+        VovH24Weight562Scorer.scoring(mergeWeight562, vov562FeatureMap, items);
+
+        // 5 排序公式特征
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
+        double alpha_vov = mergeWeight.getOrDefault("alpha_vov", 0.2);
+        double alpha_vov_562 = mergeWeight562.getOrDefault("alpha_vov", 1.0);
+        double alpha_vov_567 = mergeWeight567.getOrDefault("alpha_vov", 0.05);
+        double func = mergeWeight.getOrDefault("func", 1.0);
+        List<Video> result = new ArrayList<>();
         for (RankItem item : items) {
+            item.getScoresMap().put("alpha_vov", alpha_vov);
             double score = 0.0;
-            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault(hasReturnRovKey, "0"));
-            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double fmRov = item.getScoreRov();
+            double fmRovOrigin = item.getScoreRov();
+            item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
+            double fmRov = restoreScore(fmRovOrigin);
             item.getScoresMap().put("fmRov", fmRov);
-            if (chooseFunction == 0){
-                score = fmRov * (rosDefault + hasReturnRovScore);
-            }else if (chooseFunction == 1){
-                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
-            }else {
-                score = fmRov * (1 + hasReturnRovScore);
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rate_n", "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double vovScore = item.getVovScore();
+            item.getScoresMap().put("vovScore", vovScore);
+            if (func == 1) {
+                score = fmRov * (1 + hasReturnRovScore) + alpha_vov * vovScore;
+            } else {
+                score = fmRov * (1 + hasReturnRovScore) * (1.0 + alpha_vov * vovScore);
             }
 
+            // 562 && 567
+            double score562 = fmRov * (1 + hasReturnRovScore) * (1.0 + alpha_vov_562 * item.getScoresMap().getOrDefault("vovScore562", 0d));
+            double score567 = fmRov * (1 + hasReturnRovScore) + alpha_vov_567 * item.getScoresMap().getOrDefault("vovScore567", 0d);
+            item.getScoresMap().put("score562", score562);
+            item.getScoresMap().put("score567", score567);
+
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -340,85 +407,6 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
             result.add(video);
         }
         result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-
         return result;
     }
-
-    private Map<String, Map<String, String>> extractVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
-    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
-    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
-    public void readBucketFile() {
-        InputStream resourceStream = RankStrategy4RegionMergeModelV564.class.getClassLoader().getResourceAsStream("20240609_bucket_314.txt");
-        if (resourceStream != null) {
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
-                Map<String, double[]> bucketsMap = new HashMap<>();
-                Map<String, Double> bucketsLen = new HashMap<>();
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    // 替换空格和换行符，过滤空行
-                    line = line.replace(" ", "").replaceAll("\n", "");
-                    if (!line.isEmpty()) {
-                        String[] rList = line.split("\t");
-                        if (rList.length == 3) {
-                            String key = rList[0];
-                            double value1 = Double.parseDouble(rList[1]);
-                            bucketsLen.put(key, value1);
-                            double[] value2 = Arrays.stream(rList[2].split(","))
-                                    .mapToDouble(Double::valueOf)
-                                    .toArray();
-                            bucketsMap.put(key, value2);
-                        }
-                    }
-                }
-                this.bucketsMap = bucketsMap;
-                this.bucketsLen = bucketsLen;
-            } catch (IOException e) {
-                log.error("something is wrong in parse bucket file:" + e);
-            }
-        } else {
-            log.error("no bucket file");
-        }
-
-    }
-
-    static class Tuple4 {
-        public Map<String, String> first;
-        public Map<String, String> second;
-        public Map<String, String> third;
-
-        public String name;
-
-        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name) {
-            this.first = first;
-            this.second = second;
-            this.third = third;
-            this.name = name;
-        }
-
-    }
-
-    static class Tuple2 {
-        public Map<String, String> first;
-
-        public String name;
-
-        public Tuple2(Map<String, String> first, String name) {
-            this.first = first;
-            this.name = name;
-        }
-
-    }
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -39,6 +39,7 @@ public final class ScorerUtils {
         ScorerUtils.init("feeds_score_config_20240806.conf");
         ScorerUtils.init("feeds_score_config_20240807.conf");
         ScorerUtils.init("feeds_score_config_20241107.conf");
+        ScorerUtils.init("feeds_score_config_20241119.conf");
         ScorerUtils.init("feeds_score_config_xgb_20240828.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v1.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v1_vov.conf");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VovH24Weight562Scorer.java

@@ -0,0 +1,120 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+public class VovH24Weight562Scorer {
+    public static void scoring(Map<String, Double> mergeWeight, Map<String, Map<String, String>> vid2VovFeatureMap, final List<RankItem> items) {
+        try {
+            if (MapUtils.isEmpty(mergeWeight) || MapUtils.isEmpty(vid2VovFeatureMap) || CollectionUtils.isEmpty(items)) {
+                return;
+            }
+
+            // 融合权重
+            double vov_thresh = mergeWeight.getOrDefault("vov_thresh", 0.1);
+            double view_thresh = mergeWeight.getOrDefault("view_thresh", 1535.0);
+            double level50_vov = mergeWeight.getOrDefault("level50_vov", 0.123);
+            double level_95_vov = mergeWeight.getOrDefault("level_95_vov", 0.178);
+            double beta_vov = mergeWeight.getOrDefault("beta_vov", 100.0);
+
+            List<Double> weightList = new ArrayList<>(7);
+            weightList.add(mergeWeight.getOrDefault("d2_ago_vov_w", 0.0));
+            weightList.add(mergeWeight.getOrDefault("d1_ago_vov_w", 0.0));
+            weightList.add(mergeWeight.getOrDefault("h48_ago_vov_w", 0.0));
+            weightList.add(mergeWeight.getOrDefault("h24_ago_vov_w", 0.0));
+            weightList.add(mergeWeight.getOrDefault("h3_ago_vov_w", 0.0));
+            weightList.add(mergeWeight.getOrDefault("h2_ago_vov_w", 0.0));
+            weightList.add(mergeWeight.getOrDefault("h1_ago_vov_w", 0.0));
+
+            for (RankItem item : items) {
+                // 获取VoV输入特征
+                double h1_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                        .getOrDefault("h1_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
+                double h2_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                        .getOrDefault("h2_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
+                double h3_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                        .getOrDefault("h3_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
+                double h24_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                        .getOrDefault("h24_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
+                double h48_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                        .getOrDefault("h48_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
+                double d1_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                        .getOrDefault("d1_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
+                double d2_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                        .getOrDefault("d2_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
+                double h1_ago_view = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                        .getOrDefault("h1_ago_view", "-2")); // 如果没有时，默认为多少？？ 需要考虑
+
+                // log feature
+                item.getScoresMap().put("h1_ago_vov", h1_ago_vov);
+                item.getScoresMap().put("h2_ago_vov", h2_ago_vov);
+                item.getScoresMap().put("h3_ago_vov", h3_ago_vov);
+                item.getScoresMap().put("h24_ago_vov", h24_ago_vov);
+                item.getScoresMap().put("h48_ago_vov", h48_ago_vov);
+                item.getScoresMap().put("d1_ago_vov", d1_ago_vov);
+                item.getScoresMap().put("d2_ago_vov", d2_ago_vov);
+                item.getScoresMap().put("h1_ago_view", h1_ago_view);
+
+                List<Double> featureList = new ArrayList<>(7);
+                featureList.add(d2_ago_vov);
+                featureList.add(d1_ago_vov);
+                featureList.add(h48_ago_vov);
+                featureList.add(h24_ago_vov);
+                featureList.add(h3_ago_vov);
+                featureList.add(h2_ago_vov);
+                featureList.add(h1_ago_vov);
+                double vovScore562 = calcScore(featureList, weightList, item, vov_thresh, view_thresh, h1_ago_view, level50_vov, level_95_vov, beta_vov);
+                item.getScoresMap().put("vovScore562", vovScore562);
+            }
+        } catch (Exception e) {
+            log.error(String.format("something is wrong in VovH24Weight562Scorer with {}", e));
+        }
+    }
+
+
+    private static double calcScore(List<Double> featureList, List<Double> weightList, RankItem rankItem,
+                                    double vov_thresh, double view_thresh, double h1_ago_view, double level50_vov, double level_95_vov, double beta_vov) {
+        // 检查 h1_ago_view 条件
+        if (h1_ago_view == -2 || h1_ago_view == -1 || h1_ago_view < view_thresh) {
+            return 0;
+        }
+
+        // 计算有效特征的总权重和得分
+        double score = 0;
+        List<Integer> validIndices = new ArrayList<>();
+        for (int i = 0; i < featureList.size(); i++) {
+            if (featureList.get(i) != -1) {
+                validIndices.add(i);
+            }
+        }
+
+        // 如果没有有效特征，返回 0
+        if (validIndices.isEmpty()) {
+            return 0;
+        }
+
+        // 计算得分，动态调整权重
+        for (int index : validIndices) {
+            double weight = weightList.get(index);
+            score += featureList.get(index) * weight;
+        }
+
+        // 调整vov
+        if (score < vov_thresh) {
+            score = 0;
+        } else {
+            double term1 = 1 / (1 + Math.exp(-1 * beta_vov * (score - level50_vov)));
+            double term2 = 1 + Math.exp(-1 * beta_vov * (level_95_vov - level50_vov));
+            score = term1 * term2;
+        }
+        return score;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VovH24WeightScorer.java

@@ -157,7 +157,7 @@ public class VovH24WeightScorer extends AbstractScorer {
 
         double vovScore = 0.0;
         Map<String, String> featureMap = item.getFeatureMap();
-        String weightKey = sceneFeatureMap.get("weightKey");
+        String weightKey = sceneFeatureMap.getOrDefault("weightKey", "");
         Map<String, Double> weightMap = model.getWeight(weightKey);
         if (MapUtils.isNotEmpty(featureMap) && MapUtils.isNotEmpty(weightMap)) {
             try {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VovH24WeightV2Scorer.java

@@ -0,0 +1,209 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.VovH24WeightModel;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+public class VovH24WeightV2Scorer extends AbstractScorer {
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogRovFMScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+    private static final Set<String> rateFeatureSet = new HashSet<>(Arrays.asList(
+            "1_vovh0",
+            "2_vovh0", "2_vovh1",
+            "3_vovh0", "3_vovh1", "3_vovh2",
+            "4_vovh0", "4_vovh1", "4_vovh3",
+            "7_vovh0", "7_vovh1", "7_vovh6",
+            "13_vovh0", "13_vovh1", "13_vovh12",
+            "25_vovh0", "25_vovh1", "25_vovh24",
+            "1_vovd0",
+            "2_vovd0", "2_vovd1",
+            "3_vovd0", "3_vovd1", "3_vovd2"
+    ));
+
+    private static final Set<String> integerFeatureSet = new HashSet<>(Arrays.asList(
+            "1_vovh0分子", "1_vovh分母",
+            "2_vovh0分子", "2_vovh1分子", "2_vovh分母",
+            "3_vovh0分子", "3_vovh1分子", "3_vovh2分子", "3_vovh分母",
+            "4_vovh0分子", "4_vovh1分子", "4_vovh3分子", "4_vovh分母",
+            "7_vovh0分子", "7_vovh1分子", "7_vovh6分子", "7_vovh分母",
+            "13_vovh0分子", "13_vovh1分子", "13_vovh12分子", "13_vovh分母",
+            "25_vovh0分子", "25_vovh1分子", "25_vovh24分子", "25_vovh分母",
+            "1_vovd0分子", "1_vovd分母",
+            "2_vovd0分子", "2_vovd1分子", "2_vovd分母",
+            "3_vovd0分子", "3_vovd1分子", "3_vovd2分子", "3_vovd分母"
+    ));
+
+    private static final Set<String> numerator567Set = new HashSet<>(Arrays.asList(
+            "1_vovh0分子", "2_vovh1分子", "3_vovh2分子", "4_vovh3分子",
+            "7_vovh6分子", "13_vovh12分子", "25_vovh24分子", "2_vovd1分子"
+    ));
+
+    private static final Set<String> denominator567Set = new HashSet<>(Arrays.asList(
+            "1_vovh分母", "2_vovh分母", "3_vovh分母", "4_vovh分母",
+            "7_vovh分母", "13_vovh分母", "25_vovh分母", "2_vovd分母"
+    ));
+
+
+    public VovH24WeightV2Scorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(VovH24WeightModel.class);
+    }
+
+    @Override
+    public List<RankItem> scoring(ScoreParam param, UserFeature userFeature, List<RankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        VovH24WeightModel model = (VovH24WeightModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("vovh24 scorer time java items size={}, time={} ",
+                result.size(), System.currentTimeMillis() - startTime);
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        VovH24WeightModel model = (VovH24WeightModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (RankItem item : items) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", item, item);
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("[vovh24 scorer time java] items size={}, cost={} ",
+                items.size(), System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final VovH24WeightModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("vovh24 scorer exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // 等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException: ", e);
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {}, ", sceneFeatureMap.size(), e);
+                }
+            }
+        }
+    }
+
+    public double calcScore(final VovH24WeightModel model,
+                            final RankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+        double vovScore = 0.0;
+        Map<String, String> featureMap = item.getFeatureMap();
+        String weightKey = sceneFeatureMap.getOrDefault("weightKey", "");
+        Map<String, Double> weightMap = model.getWeight(weightKey);
+        if (MapUtils.isNotEmpty(featureMap) && MapUtils.isNotEmpty(weightMap)) {
+            try {
+                vovScore += weightMap.getOrDefault("bias", 0d);
+                for (String key : rateFeatureSet) {
+                    double val = Double.parseDouble(featureMap.getOrDefault(key, "0d"));
+                    double weight = weightMap.getOrDefault(key, 0d);
+                    vovScore += val * weight;
+                }
+                for (String key : integerFeatureSet) {
+                    double val = Double.parseDouble(featureMap.getOrDefault(key, "0d"));
+                    double weight = weightMap.getOrDefault(key, 0d);
+                    vovScore += Math.log(val + 1) * weight;
+                }
+                vovScore = Math.max(0, vovScore);
+            } catch (Exception e) {
+                LOGGER.error("vovh24 scorer error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+
+        double vovScore567 = 0.0;
+        double numerator567 = 0D;
+        double denominator567 = 0D;
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                for (String key : numerator567Set) {
+                    numerator567 += Double.parseDouble(featureMap.getOrDefault(key, "0d"));
+                }
+                for (String key : denominator567Set) {
+                    denominator567 += Double.parseDouble(featureMap.getOrDefault(key, "0d"));
+                }
+                vovScore567 = denominator567 != 0.0 ? numerator567 / denominator567 : 0.0;
+            } catch (Exception e) {
+                LOGGER.error("vovh24 567 scorer error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.getScoresMap().put("vovScore", vovScore);
+        item.getScoresMap().put("vovScore567", vovScore567);
+        item.setVovScore(vovScore);
+        return vovScore;
+    }
+}

recommend-server-service/src/main/resources/feeds_score_config_20241119.conf

@@ -0,0 +1,13 @@
+scorer-config = {
+  rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogRovFMScorer"
+    scorer-priority = 96
+    model-path = "zhangbo/model_aka8_new2.txt"
+  }
+
+  vov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VovH24WeightV2Scorer"
+    scorer-priority = 95
+    model-path = "zhangbo/vovh24_hour_weight_v2.json"
+  }
+}