Merge branch 'feature/zhangbo_rank' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV1.java

@@ -4,18 +4,31 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -29,14 +42,61 @@ import java.util.stream.Collectors;
 
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV1 extends RankService {
-//    @ApolloJsonValue("${video.model.weightv3:}")
-//    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${rank.score.merge.weightv1:}")
+    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -50,7 +110,13 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
@@ -61,43 +127,27 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
 
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
 
         // 1 模型分
         List<String> rtFeaPart = new ArrayList<>();
@@ -115,7 +165,7 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -142,14 +192,47 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
+        double a = mergeWeight.getOrDefault("a", 1.0);
+        double b = mergeWeight.getOrDefault("b", 1.0);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
         for (RankItem item : items){
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -161,6 +244,31 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         return result;
     }
 
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -179,7 +287,6 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         }
         return data;
     }
-
     public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
                                       List<String> datehours, String key){
         List<Double> views = new LinkedList<>();
@@ -191,9 +298,8 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         }
         return views;
     }
-
     public List<RankItem> model(List<Video> videos, RankParam param,
-                             List<String> rtFeaPart){
+                                List<String> rtFeaPart){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -282,7 +388,8 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 }
                 try{
                     vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
                     Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
                     while (iteratorIn.hasNext()) {
                         Map.Entry<String, String> entry = iteratorIn.next();
@@ -407,7 +514,6 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
 //        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -437,5 +543,92 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
 
         return sceneFeatureMap;
     }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV2.java

@@ -2,19 +2,33 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 
 import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -28,14 +42,61 @@ import java.util.stream.Collectors;
 
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV2 extends RankService {
-//    @ApolloJsonValue("${video.model.weightv3:}")
-//    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${rank.score.merge.weightv2:}")
+    private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -49,7 +110,13 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
@@ -60,43 +127,27 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
 
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
 
         // 1 模型分
         List<String> rtFeaPart = new ArrayList<>();
@@ -114,7 +165,7 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -141,15 +192,47 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
+        double a = mergeWeight.getOrDefault("a", 1.0);
+        double b = mergeWeight.getOrDefault("b", 1.0);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
         for (RankItem item : items){
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
-                    Math.log(1 + item.scoresMap.getOrDefault("allreturnsScore", 0.0));
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -160,7 +243,31 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
-
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -190,9 +297,8 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         }
         return views;
     }
-
     public List<RankItem> model(List<Video> videos, RankParam param,
-                             List<String> rtFeaPart){
+                                List<String> rtFeaPart){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -281,7 +387,8 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
                 }
                 try{
                     vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
                     Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
                     while (iteratorIn.hasNext()) {
                         Map.Entry<String, String> entry = iteratorIn.next();
@@ -406,7 +513,6 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
 //        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -436,5 +542,92 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
 
         return sceneFeatureMap;
     }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV4.java

@@ -4,18 +4,31 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -29,14 +42,61 @@ import java.util.stream.Collectors;
 
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV4 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weight:}")
+    @ApolloJsonValue("${rank.score.merge.weightv4:}")
     private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -50,7 +110,13 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
@@ -61,43 +127,27 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
-
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
 
         // 1 模型分
         List<String> rtFeaPart = new ArrayList<>();
@@ -115,7 +165,7 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -142,24 +192,47 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
             // rov的趋势
             double trendScore = calTrendScore(view2return);
             item.scoresMap.put("trendScore", trendScore);
 
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
-        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
+        double a = mergeWeight.getOrDefault("a", 1.0);
+        double b = mergeWeight.getOrDefault("b", 1.0);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
         for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
-                    + alpha * trendScore
-                    ;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -170,22 +243,31 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
     public double calTrendScore(List<Double> data){
-        int cnt = 0;
         double sum = 0.0;
         int size = data.size();
         for (int i=0; i<size-4; ++i){
             sum += data.get(i) - data.get(i+4);
-            cnt ++;
         }
         if (sum * 10 > 0.6){
             sum = 0.6;
         }else{
             sum = sum * 10;
         }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
         return sum;
     }
-
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -215,9 +297,8 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         }
         return views;
     }
-
     public List<RankItem> model(List<Video> videos, RankParam param,
-                             List<String> rtFeaPart){
+                                List<String> rtFeaPart){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -306,7 +387,8 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
                 }
                 try{
                     vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
                     Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
                     while (iteratorIn.hasNext()) {
                         Map.Entry<String, String> entry = iteratorIn.next();
@@ -431,7 +513,6 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
 //        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -461,5 +542,92 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
 
         return sceneFeatureMap;
     }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV5.java

@@ -4,18 +4,31 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -29,14 +42,61 @@ import java.util.stream.Collectors;
 
 /**
  * @author zhangbo
- * @desc 地域召回融合
+ * @desc 地域召回融合 流量池汤姆森
  */
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV5 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weight:}")
+    @ApolloJsonValue("${rank.score.merge.weightv5:}")
     private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
         while(iterator.hasNext()){
@@ -50,7 +110,13 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
     }
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
+        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
@@ -61,43 +127,27 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         this.duplicate(setVideo, v2);
         this.duplicate(setVideo, v3);
         this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
         this.duplicate(setVideo, v5);
         this.duplicate(setVideo, v6);
-
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+        //-------------------节日扶持召回 融合+去重-------------------
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
-//        List<String> videoIdKeys = rovRecallRank.stream()
-//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-//                .collect(Collectors.toList());
-//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
-//                JSONUtils.toJson(videoScores));
-//        if (CollectionUtils.isNotEmpty(videoScores)
-//                && videoScores.size() == rovRecallRank.size()) {
-//            for (int i = 0; i < videoScores.size(); i++) {
-//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-//            }
-//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        }
 
         // 1 模型分
         List<String> rtFeaPart = new ArrayList<>();
@@ -115,7 +165,7 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
         for (int i=0; i<24; ++i){
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
@@ -142,9 +192,14 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
+            // 平台回流
+            Double preturnsScore = calScoreWeight(returns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
             // rov的趋势
             double trendScore = calTrendScore(view2return);
             item.scoresMap.put("trendScore", trendScore);
@@ -152,22 +207,31 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
             // 新视频提取
             double newVideoScore = calNewVideoScore(itemBasicMap);
             item.scoresMap.put("newVideoScore", newVideoScore);
-
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
-        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
-        double beta = this.mergeWeight.getOrDefault("beta", 1.0);
+        double a = mergeWeight.getOrDefault("a", 1.0);
+        double b = mergeWeight.getOrDefault("b", 1.0);
+        double c = mergeWeight.getOrDefault("c", 0.0002);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
         for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
-                    + alpha * trendScore
-                    + beta * newVideoScore
-                    ;
+            double strScore = item.getScoreStr();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5){
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+            }
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -180,10 +244,10 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
     }
     public double calNewVideoScore(Map<String, String> itemBasicMap){
         double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 8){
+        if (existenceDays > 5){
             return 0.0;
         }
-        double score = 1.0 / (existenceDays + 5.0);
+        double score = 1.0 / (existenceDays + 10.0);
         return score;
     }
     public double calTrendScore(List<Double> data){
@@ -203,7 +267,6 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         }
         return sum;
     }
-
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -233,9 +296,8 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         }
         return views;
     }
-
     public List<RankItem> model(List<Video> videos, RankParam param,
-                             List<String> rtFeaPart){
+                                List<String> rtFeaPart){
         List<RankItem> result = new ArrayList<>();
         if (videos.isEmpty()){
             return result;
@@ -450,7 +512,6 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
 //        log.info(obj.toString());
         return rovRecallScore;
     }
-
     private Map<String, String> getSceneFeature(RankParam param) {
         Map<String, String> sceneFeatureMap = new HashMap<>();
         String provinceCn = param.getProvince();
@@ -480,5 +541,92 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
 
         return sceneFeatureMap;
     }
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -156,13 +156,13 @@ public class RecallService implements ApplicationContextAware {
                 case "60103": // 增加地域1小时扩量，通过配置实现
                 case "60105": // 通过更改param中的配置实现使用不同数据源 data66 rule68 + 有排序模块
                 case "60120": // 576
+                    strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                    break;
                 case "60121": // 536
                 case "60122": // 537
                 case "60124": // 546
                 case "60125": // 547
-                    strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
-                    strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
-                    break;
                 case "60123": // 541
                 case "60126": // 548
                     strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/RegionRecallScorerV1.java

@@ -30,7 +30,7 @@ public class RegionRecallScorerV1 extends AbstractScorer4Recall {
             lists = model.kv.getOrDefault("中国", new ArrayList<>());
         }
 
-        return lists.subList(0, Math.min(100, lists.size()));
+        return lists.subList(0, Math.min(200, lists.size()));
     }