recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/CustomMongoProperties.java

@@ -15,6 +15,10 @@ public class CustomMongoProperties {
     private Integer clusterPort;
     private String secondaryHost;
     private Integer secondaryPort;
+    private String thirdHost;
+    private Integer thirdPort;
+    private String fourthHost;
+    private Integer fourthPort;
 
     private String uri;
     private String database;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/MongoTemplateConfig.java

@@ -35,7 +35,9 @@ public class MongoTemplateConfig {
         builder.applyToClusterSettings(settings -> {
             settings.hosts(Lists.newArrayList(
                     new ServerAddress(properties.getClusterHost(), properties.getClusterPort()),
-                    new ServerAddress(properties.getSecondaryHost(), properties.getSecondaryPort())));
+                    new ServerAddress(properties.getSecondaryHost(), properties.getSecondaryPort()),
+                    new ServerAddress(properties.getThirdHost(), properties.getThirdPort()),
+                    new ServerAddress(properties.getFourthHost(), properties.getFourthPort())));
         });
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterService.java

@@ -1,16 +1,19 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.collect.Lists;
 import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
 import com.tzld.piaoquan.recommend.server.service.filter.strategy.*;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
@@ -24,6 +27,8 @@ import java.util.concurrent.TimeUnit;
 public class FilterService {
 
     private final ExecutorService pool = ThreadPoolFactory.filterPool();
+    @ApolloJsonValue("${filter_exp_config:{}}")
+    private Map<String, Integer> filterExpConfig;
 
     public FilterResult filter(FilterParam param) {
         List<Long> videoIds = viewFilter(param);
@@ -77,14 +82,34 @@ public class FilterService {
         strategies.add(ServiceBeanFactory.getBean(AppletVideoStatusStrategy.class));
         strategies.add(ServiceBeanFactory.getBean(RiskVideoStrategy.class));
 
-        if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("697")) {
-            strategies.add(ServiceBeanFactory.getBean(VideoSourceTypeStrategy.class));
+        // UGC视频过滤
+        Integer ugcFilterWeight = filterExpConfig.get("userupload");
+        if (ugcFilterWeight != null && ugcFilterWeight != 0) {
+            int rand = RandomUtils.nextInt(0, 100);
+            if (ugcFilterWeight >= 100 || rand < ugcFilterWeight) {
+                // log.info("hit ugc filter weight:{} rand:{}", ugcFilterWeight, rand);
+                strategies.add(ServiceBeanFactory.getBean(VideoSourceTypeStrategy.class));
+            }
         }
-        //全面spider视频过滤
-        if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("722")) {
-            strategies.add(ServiceBeanFactory.getBean(GeneralSpiderStrategy.class));
+
+        // 全面spider视频过滤
+        Integer allSpiderFilterWeight = filterExpConfig.get("allspider");
+        if (allSpiderFilterWeight != null && allSpiderFilterWeight != 0) {
+            int rand = RandomUtils.nextInt(0, 100);
+            if (allSpiderFilterWeight >= 100 || rand < allSpiderFilterWeight) {
+                // log.info("hit all spider filter weight:{} rand:{}", allSpiderFilterWeight, rand);
+                strategies.add(ServiceBeanFactory.getBean(GeneralSpiderStrategy.class));
+            }
         }
 
+//        if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("697")) {
+//            strategies.add(ServiceBeanFactory.getBean(VideoSourceTypeStrategy.class));
+//        }
+//        //全面spider视频过滤
+//        if (CollectionUtils.isNotEmpty(param.getAbExpCodes()) && param.getAbExpCodes().contains("722")) {
+//            strategies.add(ServiceBeanFactory.getBean(GeneralSpiderStrategy.class));
+//        }
+
         return strategies;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -31,9 +31,6 @@ public abstract class RankService {
     @ApolloJsonValue("${alg.recall.special.app&vid:{}}")
     protected Map<String, List<Long>> specialAppVid;
 
-    @Value("${word2vec.exp:691}")
-    protected String word2vecExp;
-
     public RankResult rank(RankParam param) {
         if (param == null
                 || param.getRecallResult() == null

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/ExtractorUtils.java

@@ -94,6 +94,12 @@ public class ExtractorUtils {
         }
         return a / b;
     }
+    public static Double smoothDiv(double a, double b, double plus){
+        if (a == 0 || b == 0){
+            return 0D;
+        }
+        return a / (b + plus);
+    }
     public static Double calLog(double a){
         if (a <= 0){
             return 0D;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV536.java

@@ -226,12 +226,7 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
                         String tags = c34567Map.getOrDefault(key, "");
                         if (!tags.isEmpty()) {
                             Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                                Double[] doubles = null;
-                                if (param.getAbExpCodes().contains(word2vecExp)) {
-                                    doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
-                                } else {
-                                    doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                }
+                                Double[] doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
                                 return Pair.create(key, doubles);
                             });
                             futures.add(future);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV562.java

@@ -250,12 +250,7 @@ public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeM
                         String tags = c34567Map.getOrDefault(key, "");
                         if (!tags.isEmpty()) {
                             Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                                Double[] doubles = null;
-                                if (param.getAbExpCodes().contains(word2vecExp)) {
-                                    doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
-                                } else {
-                                    doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                }
+                                Double[] doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
                                 return Pair.create(key, doubles);
                             });
                             futures.add(future);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV563.java

@@ -238,12 +238,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
                         String tags = c34567Map.getOrDefault(key, "");
                         if (!tags.isEmpty()) {
                             Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                                Double[] doubles = null;
-                                if (param.getAbExpCodes().contains(word2vecExp)) {
-                                    doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
-                                } else {
-                                    doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                }
+                                Double[] doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
                                 return Pair.create(key, doubles);
                             });
                             futures.add(future);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV564.java

@@ -334,15 +334,14 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
             item.norFeatureMap = FeatureBucketUtils.bucketFeature("20241209_nor_bucket.txt", featureMapDouble);
         }
         // 4 排序模型计算
-        double fmRovBias = mergeWeight.getOrDefault("fmRovBias", -0.0017);
-        double fmRovWeight = mergeWeight.getOrDefault("fmRovWeight", 1.331);
-        double fmRovSquareWeight = mergeWeight.getOrDefault("fmRovSquareWeight", -6.4597);
-        double fmRovCubeWeight = mergeWeight.getOrDefault("fmRovCubeWeight", 14.393);
+        double fmRovLogBase = mergeWeight.getOrDefault("fmRovLogBase", 3.5);
         double xgbNorScaleType = mergeWeight.getOrDefault("xgbNorScaleType", 0.0);
         double xgbNorBias = mergeWeight.getOrDefault("xgbNorBias", -1.6945);
         double xgbNorWeight = mergeWeight.getOrDefault("xgbNorWeight", 1.8968);
         double xgbNorPowerWeight = mergeWeight.getOrDefault("xgbNorPowerWeight", 1.2216);
         double xgbNorPowerExp = mergeWeight.getOrDefault("xgbNorPowerExp", 1.3217);
+        double vorLogWeight = mergeWeight.getOrDefault("vorLogWeight", 10.0);
+        double vorLogBase = mergeWeight.getOrDefault("vorLogBase", 2.6);
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
         List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20241209.conf").scoring(sceneFeatureMap, userFeatureMap, norUserFeatureMap, rankItems);
         // 5 排序公式特征
@@ -354,14 +353,15 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
             item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
             double fmRov = restoreScore(fmRovOrigin);
             item.getScoresMap().put("fmRov", fmRov);
-            double newFmRov = rovCalibration(fmRovBias, fmRovWeight, fmRovSquareWeight, fmRovCubeWeight, fmRov);
+            double newFmRov = logRov(fmRov, fmRovLogBase);
             double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
             double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
             double newNorXGBScore = norCalibration(xgbNorScaleType, xgbNorBias, xgbNorWeight, xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
             double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
+            double newVor = logVor(vorLogWeight, vor, vorLogBase);
             item.getScoresMap().put("vor", vor);
-            score = newFmRov * (0.1 + newNorXGBScore) * (0.1 + vor);
+            score = newFmRov * (0.1 + newNorXGBScore) * (0.1 + newVor);
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -496,22 +496,6 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
         }
     }
 
-    private double rovCalibration(double bias, double weight, double squareWeight, double cubeWeight, double score) {
-        double newScore = bias + weight * score;
-        if (Math.abs(squareWeight) > 1E-8) {
-            newScore += squareWeight * Math.pow(score, 2);
-        }
-        if (Math.abs(cubeWeight) > 1E-8) {
-            newScore += cubeWeight * Math.pow(score, 3);
-        }
-        if (newScore < 1E-8) {
-            newScore = score;
-        } else if (newScore > 0.9) {
-            newScore = 0.9;
-        }
-        return newScore;
-    }
-
     private double norCalibration(double scaleType, double polyBias, double polyWeight, double powerWeight, double powerExp, double score) {
         if (scaleType < 1) {
             return norPolyCalibration(polyBias, polyWeight, score);
@@ -532,7 +516,25 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
         double newScore = weight * Math.pow(score, exp);
         if (newScore > 100) {
             newScore = 100;
+        } else if (newScore < score) {
+            newScore = score;
         }
         return newScore;
     }
+
+    private double logRov(double x, double base) {
+        if (base > 1) {
+            return Math.log(1 + x) / Math.log(base);
+        } else {
+            return x;
+        }
+    }
+
+    private double logVor(double weight, double x, double base) {
+        if (base > 1) {
+            return weight * Math.log(1 + x / 10) / Math.log(base);
+        } else {
+            return x;
+        }
+    }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV567.java

@@ -39,6 +39,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
     private static final List<String> cfRovList = Collections.singletonList("rovn");
     private static final List<String> videoSimAttrs = Arrays.asList("cate1_list", "cate2", "cate2_list",
             "keywords", "style", "theme", "title", "topic", "user_value");
+    private static final double smoothPlus = 5.0;
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
@@ -72,7 +73,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         //-------------------新地域召回------------------
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 10.0).intValue(), v1.size()));
         rovRecallRank.addAll(v1);
         setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
         //-------------------scene cf rovn------------------
@@ -119,6 +120,9 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
         Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
 
+        // time feature
+        addTimeFeature(userFeatureMapDouble);
+
         if (!c1.isEmpty()) {
             userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
             userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
@@ -215,6 +219,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
                     double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
                     double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
 
+                    double f0 = ExtractorUtils.calLog(exp);
                     double f1 = ExtractorUtils.calDiv(share, exp);
                     double f2 = ExtractorUtils.calLog(share);
                     double f3 = ExtractorUtils.calDiv(returns, exp);
@@ -222,6 +227,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
                     double f5 = f3 * f4;
                     double f6 = ExtractorUtils.calDiv(returns, share);
 
+                    String key0 = tuple4.name + "_" + prefix2 + "_" + "EXP";
                     String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
                     String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
                     String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
@@ -229,6 +235,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
                     String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
                     String key6 = tuple4.name + "_" + prefix2 + "_" + "ROS";
 
+                    featureMap.put(key0, f0);
                     featureMap.put(key1, f1);
                     featureMap.put(key2, f2);
                     featureMap.put(key3, f3);
@@ -303,7 +310,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         // 3 连续值特征分桶
         readBucketFile();
         Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
-        Map<String, String> norUserFeatureMap = FeatureBucketUtils.bucketFeature("20241209_nor_bucket.txt", userFeatureMapDouble);
+        Map<String, String> norUserFeatureMap = FeatureBucketUtils.noBucketFeature(userFeatureMapDouble);
         for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
             String name = entry.getKey();
             Double score = entry.getValue();
@@ -331,16 +338,19 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
                 }
             }
             item.featureMap = featureMap;
-            item.norFeatureMap = FeatureBucketUtils.bucketFeature("20241209_nor_bucket.txt", featureMapDouble);
+            item.norFeatureMap = FeatureBucketUtils.noBucketFeature(featureMapDouble);
         }
         // 4 排序模型计算
+        double fmRovLogBase = mergeWeight.getOrDefault("fmRovLogBase", 2.8);
         double xgbNorScaleType = mergeWeight.getOrDefault("xgbNorScaleType", 0.0);
         double xgbNorBias = mergeWeight.getOrDefault("xgbNorBias", -1.6945);
         double xgbNorWeight = mergeWeight.getOrDefault("xgbNorWeight", 1.8968);
         double xgbNorPowerWeight = mergeWeight.getOrDefault("xgbNorPowerWeight", 1.2216);
         double xgbNorPowerExp = mergeWeight.getOrDefault("xgbNorPowerExp", 1.3217);
+        double vorLogWeight = mergeWeight.getOrDefault("vorLogWeight", 10.0);
+        double vorLogBase = mergeWeight.getOrDefault("vorLogBase", 2.6);
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20241209.conf").scoring(sceneFeatureMap, userFeatureMap, norUserFeatureMap, rankItems);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250208.conf").scoring(sceneFeatureMap, userFeatureMap, norUserFeatureMap, rankItems);
         // 5 排序公式特征
         Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
         List<Video> result = new ArrayList<>();
@@ -350,13 +360,15 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
             item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
             double fmRov = restoreScore(fmRovOrigin);
             item.getScoresMap().put("fmRov", fmRov);
+            double newFmRov = logRov(fmRov, fmRovLogBase);
             double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
             double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
             double newNorXGBScore = norCalibration(xgbNorScaleType, xgbNorBias, xgbNorWeight, xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
             double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
+            double newVor = logVor(vorLogWeight, vor, vorLogBase);
             item.getScoresMap().put("vor", vor);
-            score = fmRov * (0.1 + newNorXGBScore) * (0.1 + vor);
+            score = newFmRov * (0.1 + newNorXGBScore) * (0.1 + newVor);
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -414,17 +426,19 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
             List<String> infoPeriod = tuple3.second;
             Map<String, String> infoMap = tuple3.third;
             for (String period : infoPeriod) {
+                double view = getVideoOneInfo("view_" + period, infoMap);
                 double share = getVideoOneInfo("share_" + period, infoMap);
                 double return_ = getVideoOneInfo("return_" + period, infoMap);
                 double view_hasreturn = getVideoOneInfo("view_hasreturn_" + period, infoMap);
                 double share_hasreturn = getVideoOneInfo("share_hasreturn_" + period, infoMap);
-                double ros = getVideoOneInfo("ros_" + period, infoMap);
-                double rov = getVideoOneInfo("rov_" + period, infoMap);
-                double r_cnt = getVideoOneInfo("r_cnt_" + period, infoMap);
-                double r_rate = getVideoOneInfo("r_rate_" + period, infoMap);
-                double r_cnt4s = getVideoOneInfo("r_cnt4s_" + period, infoMap);
-                double str = getVideoOneInfo("str_" + period, infoMap);
-
+                double ros = ExtractorUtils.smoothDiv(return_, share, smoothPlus);
+                double rov = ExtractorUtils.smoothDiv(return_, view, smoothPlus);
+                double r_cnt = ExtractorUtils.smoothDiv(return_, view_hasreturn, smoothPlus);
+                double r_rate = ExtractorUtils.smoothDiv(view_hasreturn, view, smoothPlus);
+                double r_cnt4s = ExtractorUtils.smoothDiv(return_, share_hasreturn, smoothPlus);
+                double str = ExtractorUtils.smoothDiv(share, view, smoothPlus);
+
+                featureMap.put(infoType + "_" + period + "_" + "view", ExtractorUtils.calLog(view));
                 featureMap.put(infoType + "_" + period + "_" + "share", ExtractorUtils.calLog(share));
                 featureMap.put(infoType + "_" + period + "_" + "return", ExtractorUtils.calLog(return_));
                 featureMap.put(infoType + "_" + period + "_" + "view_hasreturn", ExtractorUtils.calLog(view_hasreturn));
@@ -491,6 +505,16 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         }
     }
 
+    private void addTimeFeature(Map<String, Double> featureMap) {
+        Calendar calendar = Calendar.getInstance();
+        calendar.setTimeInMillis(System.currentTimeMillis());
+
+        int week = calendar.get(Calendar.DAY_OF_WEEK);
+        int hour = calendar.get(Calendar.HOUR_OF_DAY) + 1;
+        featureMap.put("week", week * 1.0);
+        featureMap.put("hour", hour * 1.0);
+    }
+
     private double norCalibration(double scaleType, double polyBias, double polyWeight, double powerWeight, double powerExp, double score) {
         if (scaleType < 1) {
             return norPolyCalibration(polyBias, polyWeight, score);
@@ -511,7 +535,25 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         double newScore = weight * Math.pow(score, exp);
         if (newScore > 100) {
             newScore = 100;
+        } else if (newScore < score) {
+            newScore = score;
         }
         return newScore;
     }
+
+    private double logRov(double x, double base) {
+        if (base > 1) {
+            return Math.log(1 + x) / Math.log(base);
+        } else {
+            return x;
+        }
+    }
+
+    private double logVor(double weight, double x, double base) {
+        if (base > 1) {
+            return weight * Math.log(1 + x / 10) / Math.log(base);
+        } else {
+            return x;
+        }
+    }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV568.java

@@ -77,6 +77,13 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
         rovRecallRank.addAll(sceneCFRosn);
         setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------return相似召回------------------
+        List<Video> returnv2 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        returnv2 = returnv2.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        returnv2 = returnv2.subList(0, Math.min(mergeWeight.getOrDefault("returnv2", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(returnv2);
+        setVideo.addAll(returnv2.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+
 
         //-------------------排-------------------
         //-------------------序-------------------

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -110,7 +110,7 @@ public class RecallService implements ApplicationContextAware {
         strategies.add(strategyMap.get(SceneCFRosnRecallStrategy.class.getSimpleName()));
         Set<String> abExpCodes = param.getAbExpCodes();
         if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568")) {
-            strategies.remove(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+            // strategies.remove(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV7LongTermV1.class.getSimpleName()));
         }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/NorXGBRegressionScorer.java

@@ -147,8 +147,8 @@ public class NorXGBRegressionScorer extends XGBRegressionModelScorer {
         }
         if (pro < 1E-8) {
             pro = 0;
-        } else if (pro > 50) {
-            pro = 50;
+        } else if (pro > 80) {
+            pro = 80;
         }
         item.getScoresMap().put("NorXGBScore", pro);
         return pro;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerPipeline.java

@@ -196,12 +196,13 @@ public class ScorerPipeline {
         }
         List<RankItem> items = rankItems;
 
-        Map<String, String> userFeatMap = userFeatureMap;
+        Map<String, String> userFeatMap;
         for (final AbstractScorer scorer : scorers) {
             if (!scorer.isEnable()) {
                 continue;
             }
-            if (scorer.scorerConfigInfo.getModelPath().equals("zhangbo/model_xgb_for_recsys_nor.tar.gz")) {
+            userFeatMap = userFeatureMap;
+            if (scorer.scorerConfigInfo.getModelPath().contains("_nor.tar.gz")) {
                 userFeatMap = norUserFeatureMap;
             }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -33,6 +33,7 @@ public final class ScorerUtils {
         ScorerUtils.init("feeds_score_config_20240609.conf");
         ScorerUtils.init("feeds_score_config_20240807.conf");
         ScorerUtils.init("feeds_score_config_fm_xgb_20241209.conf");
+        ScorerUtils.init("feeds_score_config_fm_xgb_20250208.conf");
         ScorerUtils.init("feeds_score_config_xgb_20241209.conf");
         ScorerUtils.init("feeds_score_config_xgb_20250109.conf");
         ScorerUtils.init("feeds_score_config_xgb_rov_20241209.conf");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/FeatureBucketUtils.java

@@ -32,6 +32,21 @@ public class FeatureBucketUtils {
         log.info("load {} bucketFile", featureBucketMap.size());
     }
 
+    public static Map<String, String> noBucketFeature(Map<String, Double> srcFeature) {
+        if (MapUtils.isEmpty(srcFeature)) {
+            return new HashMap<>();
+        }
+        Map<String, String> featureMap = new HashMap<>(srcFeature.size());
+        for (Map.Entry<String, Double> entry : srcFeature.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            if (score > 1E-8) {
+                featureMap.put(name, String.valueOf(score));
+            }
+        }
+        return featureMap;
+    }
+
     public static Map<String, String> bucketFeature(String bucketFile, Map<String, Double> srcFeature) {
         if (MapUtils.isEmpty(srcFeature)) {
             return new HashMap<>();

recommend-server-service/src/main/resources/application-pre.yml

@@ -77,6 +77,10 @@ spring:
       clusterPort: 3717
       secondaryHost: s-bp137073555e7bc4.mongodb.rds.aliyuncs.com
       secondaryPort: 3717
+      thirdHost: s-bp1b01b2507c9fa4.mongodb.rds.aliyuncs.com
+      thirdPort: 3717
+      fourthHost: s-bp103402c272fa24.mongodb.rds.aliyuncs.com
+      fourthPort: 3717
       database: longvideo
       username: lv
       password: lv2018

recommend-server-service/src/main/resources/application-prod.yml

@@ -77,6 +77,10 @@ spring:
       clusterPort: 3717
       secondaryHost: s-bp137073555e7bc4.mongodb.rds.aliyuncs.com
       secondaryPort: 3717
+      thirdHost: s-bp1b01b2507c9fa4.mongodb.rds.aliyuncs.com
+      thirdPort: 3717
+      fourthHost: s-bp103402c272fa24.mongodb.rds.aliyuncs.com
+      fourthPort: 3717
       database: longvideo
       username: lv
       password: lv2018

recommend-server-service/src/main/resources/feeds_score_config_fm_xgb_20250208.conf

@@ -0,0 +1,277 @@
+scorer-config = {
+  rov-score-config = {
+     scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogRovFMScorer"
+     scorer-priority = 96
+     model-path = "zhangbo/model_aka8_new2.txt"
+  }
+  nor-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.NorXGBRegressionScorer"
+    scorer-priority = 97
+    model-path = "zhangbo/model_xgb_for_recsys_v2_nor.tar.gz"
+    param = {
+      localDir = "xgboost/recsys_v2_nor"
+      features = [
+      "b111213_1d_ROS",
+      "b111213_1d_ROV*log(return)",
+      "b111213_2h_ROS",
+      "b111213_2h_ROV*log(return)",
+      "b111213_3d_ROS",
+      "b111213_3d_ROV*log(return)",
+      "b111213_7d_ROS",
+      "b111213_7d_ROV*log(return)",
+      "b123_1d_ROS",
+      "b123_1d_ROV*log(return)",
+      "b123_2h_ROS",
+      "b123_2h_ROV*log(return)",
+      "b123_3d_ROS",
+      "b123_3d_ROV*log(return)",
+      "b123_7d_ROS",
+      "b123_7d_ROV*log(return)",
+      "b167_1d_ROS",
+      "b167_1d_ROV*log(return)",
+      "b167_2h_ROS",
+      "b167_2h_ROV*log(return)",
+      "b167_3d_ROS",
+      "b167_3d_ROV*log(return)",
+      "b167_7d_ROS",
+      "b167_7d_ROV*log(return)",
+      "b171819_1d_ROS",
+      "b171819_1d_ROV*log(return)",
+      "b171819_2h_ROS",
+      "b171819_2h_ROV*log(return)",
+      "b171819_3d_ROS",
+      "b171819_3d_ROV*log(return)",
+      "b171819_7d_ROS",
+      "b171819_7d_ROV*log(return)",
+      "b20_24h_r_cnt",
+      "b20_24h_r_cnt4s",
+      "b20_24h_ros",
+      "b20_24h_share_hasreturn",
+      "b20_2h_r_cnt",
+      "b20_2h_r_cnt4s",
+      "b20_2h_ros",
+      "b20_2h_share_hasreturn",
+      "b20_7d_r_cnt",
+      "b20_7d_r_cnt4s",
+      "b20_7d_ros",
+      "b20_7d_share_hasreturn",
+      "b21_24h_r_cnt",
+      "b21_24h_r_cnt4s",
+      "b21_24h_ros",
+      "b21_24h_share_hasreturn",
+      "b21_2h_r_cnt",
+      "b21_2h_r_cnt4s",
+      "b21_2h_ros",
+      "b21_2h_share_hasreturn",
+      "b21_7d_r_cnt",
+      "b21_7d_r_cnt4s",
+      "b21_7d_ros",
+      "b21_7d_share_hasreturn",
+      "b22_24h_r_cnt",
+      "b22_24h_r_cnt4s",
+      "b22_24h_ros",
+      "b22_24h_share_hasreturn",
+      "b22_2h_r_cnt",
+      "b22_2h_r_cnt4s",
+      "b22_2h_ros",
+      "b22_2h_share_hasreturn",
+      "b22_7d_r_cnt",
+      "b22_7d_r_cnt4s",
+      "b22_7d_ros",
+      "b22_7d_share_hasreturn",
+      "b23_14d_r_cnt",
+      "b23_14d_r_cnt4s",
+      "b23_14d_ros",
+      "b23_14d_share_hasreturn",
+      "b23_30d_r_cnt",
+      "b23_30d_r_cnt4s",
+      "b23_30d_ros",
+      "b23_30d_share_hasreturn",
+      "b24_14d_r_cnt",
+      "b24_14d_r_cnt4s",
+      "b24_14d_ros",
+      "b24_14d_share_hasreturn",
+      "b24_30d_r_cnt",
+      "b24_30d_r_cnt4s",
+      "b24_30d_ros",
+      "b24_30d_share_hasreturn",
+      "b25_14d_r_cnt",
+      "b25_14d_r_cnt4s",
+      "b25_14d_ros",
+      "b25_14d_share_hasreturn",
+      "b25_30d_r_cnt",
+      "b25_30d_r_cnt4s",
+      "b25_30d_ros",
+      "b25_30d_share_hasreturn",
+      "b26_35d_r_cnt",
+      "b26_35d_r_cnt4s",
+      "b26_35d_ros",
+      "b26_35d_share_hasreturn",
+      "b26_365d_r_cnt",
+      "b26_365d_r_cnt4s",
+      "b26_365d_ros",
+      "b26_365d_share_hasreturn",
+      "b26_7d_r_cnt",
+      "b26_7d_r_cnt4s",
+      "b26_7d_ros",
+      "b26_7d_share_hasreturn",
+      "b26_90d_r_cnt",
+      "b26_90d_r_cnt4s",
+      "b26_90d_ros",
+      "b26_90d_share_hasreturn",
+      "b27_35d_r_cnt",
+      "b27_35d_r_cnt4s",
+      "b27_35d_ros",
+      "b27_35d_share_hasreturn",
+      "b27_365d_r_cnt",
+      "b27_365d_r_cnt4s",
+      "b27_365d_ros",
+      "b27_365d_share_hasreturn",
+      "b27_7d_r_cnt",
+      "b27_7d_r_cnt4s",
+      "b27_7d_ros",
+      "b27_7d_share_hasreturn",
+      "b27_90d_r_cnt",
+      "b27_90d_r_cnt4s",
+      "b27_90d_ros",
+      "b27_90d_share_hasreturn",
+      "b28_24h_r_cnt",
+      "b28_24h_r_cnt4s",
+      "b28_24h_ros",
+      "b28_24h_share_hasreturn",
+      "b28_2h_r_cnt",
+      "b28_2h_r_cnt4s",
+      "b28_2h_ros",
+      "b28_2h_share_hasreturn",
+      "b28_7d_r_cnt",
+      "b28_7d_r_cnt4s",
+      "b28_7d_ros",
+      "b28_7d_share_hasreturn",
+      "b8910_1d_ROS",
+      "b8910_1d_ROV*log(return)",
+      "b8910_2h_ROS",
+      "b8910_2h_ROV*log(return)",
+      "b8910_3d_ROS",
+      "b8910_3d_ROV*log(return)",
+      "b8910_7d_ROS",
+      "b8910_7d_ROV*log(return)",
+      "bit_rate",
+      "c3_feature_tags_1d_avgscore",
+      "c3_feature_tags_1d_matchnum",
+      "c3_feature_tags_1d_maxscore",
+      "c3_feature_tags_3d_avgscore",
+      "c3_feature_tags_3d_matchnum",
+      "c3_feature_tags_3d_maxscore",
+      "c3_feature_tags_7d_avgscore",
+      "c3_feature_tags_7d_matchnum",
+      "c3_feature_tags_7d_maxscore",
+      "c4_feature_tags_1d_avgscore",
+      "c4_feature_tags_1d_matchnum",
+      "c4_feature_tags_1d_maxscore",
+      "c4_feature_tags_3d_avgscore",
+      "c4_feature_tags_3d_matchnum",
+      "c4_feature_tags_3d_maxscore",
+      "c4_feature_tags_7d_avgscore",
+      "c4_feature_tags_7d_matchnum",
+      "c4_feature_tags_7d_maxscore",
+      "c5_feature_tags_1d_avgscore",
+      "c5_feature_tags_1d_matchnum",
+      "c5_feature_tags_1d_maxscore",
+      "c5_feature_tags_3d_avgscore",
+      "c5_feature_tags_3d_matchnum",
+      "c5_feature_tags_3d_maxscore",
+      "c5_feature_tags_7d_avgscore",
+      "c5_feature_tags_7d_matchnum",
+      "c5_feature_tags_7d_maxscore",
+      "c6_feature_tags_1d_avgscore",
+      "c6_feature_tags_1d_matchnum",
+      "c6_feature_tags_1d_maxscore",
+      "c6_feature_tags_3d_avgscore",
+      "c6_feature_tags_3d_matchnum",
+      "c6_feature_tags_3d_maxscore",
+      "c6_feature_tags_7d_avgscore",
+      "c6_feature_tags_7d_matchnum",
+      "c6_feature_tags_7d_maxscore",
+      "c7_feature_tags_1d_avgscore",
+      "c7_feature_tags_1d_matchnum",
+      "c7_feature_tags_1d_maxscore",
+      "c7_feature_tags_3d_avgscore",
+      "c7_feature_tags_3d_matchnum",
+      "c7_feature_tags_3d_maxscore",
+      "c7_feature_tags_7d_avgscore",
+      "c7_feature_tags_7d_matchnum",
+      "c7_feature_tags_7d_maxscore",
+      "c8_feature_return_num",
+      "c8_feature_return_rank",
+      "c8_feature_return_score",
+      "c8_feature_share_num",
+      "c8_feature_share_rank",
+      "c8_feature_share_score",
+      "c9_feature_return_num",
+      "c9_feature_return_rank",
+      "c9_feature_return_score",
+      "c9_feature_share_num",
+      "c9_feature_share_rank",
+      "c9_feature_share_score",
+      "return_uv_12h",
+      "return_uv_1d",
+      "return_uv_3d",
+      "return_uv_7d",
+      "share_pv_12h",
+      "share_pv_1d",
+      "share_pv_3d",
+      "share_pv_7d",
+      "total_time",
+      "video_sim_cate1_list",
+      "video_sim_cate2",
+      "video_sim_cate2_list",
+      "video_sim_keywords",
+      "video_sim_style",
+      "video_sim_theme",
+      "video_sim_title",
+      "video_sim_topic",
+      "video_sim_user_value",
+      "b111213_1d_EXP",
+      "b111213_2h_EXP",
+      "b111213_3d_EXP",
+      "b111213_7d_EXP",
+      "b123_1d_EXP",
+      "b123_2h_EXP",
+      "b123_3d_EXP",
+      "b123_7d_EXP",
+      "b167_1d_EXP",
+      "b167_2h_EXP",
+      "b167_3d_EXP",
+      "b167_7d_EXP",
+      "b171819_1d_EXP",
+      "b171819_2h_EXP",
+      "b171819_3d_EXP",
+      "b171819_7d_EXP",
+      "b8910_1d_EXP",
+      "b8910_2h_EXP",
+      "b8910_3d_EXP",
+      "b8910_7d_EXP",
+      "b27_35d_view",
+      "b27_365d_view",
+      "b27_7d_view",
+      "b27_90d_view",
+      "b29_24h_view",
+      "b29_2h_view",
+      "b29_7d_view",
+      "b29_24h_r_cnt",
+      "b29_24h_r_cnt4s",
+      "b29_24h_ros",
+      "b29_24h_share_hasreturn",
+      "b29_2h_r_cnt",
+      "b29_2h_r_cnt4s",
+      "b29_2h_ros",
+      "b29_2h_share_hasreturn",
+      "b29_7d_r_cnt",
+      "b29_7d_r_cnt4s",
+      "b29_7d_ros",
+      "b29_7d_share_hasreturn",
+      "hour"
+      ]
+    }
+  }
+}