Merge branch 'feature/sunxiaoyi_recall' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -28,6 +28,8 @@ public class RankRouter {
     private RankStrategy4RegionMergeModelV5 rankStrategy4RegionMergeModelV5;
     @Autowired
     private RankStrategy4RegionMergeModelV6 rankStrategy4RegionMergeModelV6;
+    @Autowired
+    private FestivalStrategy4RankModel festivalStrategy4RankModel;
 
     @Autowired
     private RankStrategyFlowThompsonModel rankStrategyFlowThompsonModel;
@@ -74,7 +76,7 @@ public class RankRouter {
                 return rankStrategy4RegionMergeModelV6.rank(param);
             case "60130":
                 // 先走默认排序，后续需要优化祝福类的视频排序
-                return rankService.rank(param);
+                return festivalStrategy4RankModel.rank(param);
             default:
                 break;
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -104,7 +104,7 @@ public class RankService {
         return mergeAndSort(param, rovRecallRank, flowPoolRank);
     }
 
-    public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank){
+    public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
         return;
     }
 
@@ -162,7 +162,6 @@ public class RankService {
                 || param.getAbCode().equals("60117")
                 || param.getAbCode().equals("60118")
                 || param.getAbCode().equals("60119")
-                || param.getAbCode().equals("60130")
         ) {
             // 地域召回要做截取，再做融合排序
             removeDuplicate(rovRecallRank);
@@ -180,9 +179,6 @@ public class RankService {
                 rovRecallRank.addAll(extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM));
                 rovRecallRank.addAll(extractAndSort(param, TopGoodPerformanceVideoRecallStrategy.PUSH_FORM));
             }
-            if (param.getAbCode().equals("60130")) {
-                rovRecallRank.addAll(extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM));
-            }
             removeDuplicate(rovRecallRank);
 
             // 融合排序

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/RankExtractorItemTags.java

@@ -0,0 +1,51 @@
+package com.tzld.piaoquan.recommend.server.service.rank.extractor;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import org.springframework.data.redis.core.RedisTemplate;
+
+import java.util.*;
+
+public class RankExtractorItemTags {
+    public RedisTemplate<String, String> redisTemplate;
+    public RankExtractorItemTags(RedisTemplate<String, String> redisTemplate){
+        this.redisTemplate = redisTemplate;
+    }
+
+    public void processor(List<Video> rovVideos, List<Video> flowVideos){
+        List<Long> videoIds = new ArrayList<>();
+        for (Video v : rovVideos) {
+            videoIds.add(v.getVideoId());
+        }
+        for (Video v : flowVideos) {
+            videoIds.add(v.getVideoId());
+        }
+        Map<Long, List<String>> videoTagDict = getVideoTags(redisTemplate, videoIds);
+        for (Video v : rovVideos) {
+            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        }
+        for (Video v : flowVideos) {
+            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        }
+    }
+
+    public static Map<Long, List<String>> getVideoTags(RedisTemplate<String, String> redisHelper, List<Long> videoIds) {
+        String REDIS_PREFIX = "alg_recsys_video_tags_";
+        List<String> redisKeys = new ArrayList<>();
+        for (Long videoId : videoIds) {
+            redisKeys.add(REDIS_PREFIX + String.valueOf(videoId));
+        }
+        List<String> videoTags = redisHelper.opsForValue().multiGet(redisKeys);
+        Map<Long, List<String>> videoTagDict = new HashMap<>();
+        if (videoTags != null) {
+            for (int i = 0; i < videoTags.size(); i++) {
+                String tagsStr = videoTags.get(i);
+                List<String> tags = new ArrayList<>();
+                if (tagsStr != null && !tagsStr.isEmpty()) {
+                    String[] tagsArray = tagsStr.split(",");
+                    tags = new ArrayList<>(Arrays.asList(tagsArray));
+                }
+                videoTagDict.put(videoIds.get(i), tags);
+            }
+        }
+        return videoTagDict;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorBoost.java

@@ -0,0 +1,44 @@
+package com.tzld.piaoquan.recommend.server.service.rank.processor;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.math.BigDecimal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 提权处理器
+ *
+ * @author sunxy
+ */
+public class RankProcessorBoost {
+
+    public static void boostByTag(List<Video> rovList, Map<String, Map<String, String>> rulesMap) {
+        if (CollectionUtils.isEmpty(rovList) || rulesMap == null || rulesMap.isEmpty()) {
+            return;
+        }
+        Map<String, Double> densityRules = new HashMap<>();
+        for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+            String key = entry.getKey();
+            Map<String, String> value = entry.getValue();
+            if (value.containsKey("boost")) {
+                densityRules.put(key, Double.valueOf(value.get("boost")));
+            }
+        }
+
+
+        for (Map.Entry<String, Double> entry : densityRules.entrySet()) {
+            rovList.stream().filter(video -> video.getTags().contains(entry.getKey()))
+                    .forEach(video -> video.setSortScore(
+                            BigDecimal.valueOf(video.getSortScore())
+                                    .multiply(BigDecimal.valueOf(entry.getValue()))
+                                    .doubleValue()));
+        }
+        rovList.sort((o1, o2) -> Double.compare(o2.getSortScore(), o1.getSortScore()));
+
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorInsert.java

@@ -0,0 +1,94 @@
+package com.tzld.piaoquan.recommend.server.service.rank.processor;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.util.MathUtil;
+import com.tzld.piaoquan.recommend.server.util.WeightRandom;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author sunxy
+ */
+public class RankProcessorInsert {
+
+    public static void insertByTag(RankParam param, List<Video> rovList,
+                                   Map<String, Map<String, String>> rulesMap) {
+        if (CollectionUtils.isEmpty(rovList) || rulesMap == null || rulesMap.isEmpty()) {
+            return;
+        }
+        String insertRules = null, tagName = null;
+        for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+            Map<String, String> value = entry.getValue();
+            if (value.containsKey("insert")) {
+                insertRules = value.get("insert");
+                tagName = entry.getKey();
+                break;
+            }
+        }
+        if (StringUtils.isBlank(insertRules) || StringUtils.isBlank(tagName)) {
+            return;
+        }
+        final String finalTagName = tagName;
+
+        // 判断是否前几个已经有该tag的视频了
+        boolean isExistResultTagVideo = rovList.stream()
+                .limit(param.getSize())
+                .anyMatch(video -> video.getTags().contains(finalTagName));
+        if (isExistResultTagVideo) {
+            return;
+        }
+
+        // 获取需要插入的视频
+        Video insertTagVideo = null;
+        int tagVideoIndex = -1;
+        for (int i = 0; i < rovList.size(); i++) {
+            if (rovList.get(i).getTags().contains(finalTagName)) {
+                insertTagVideo = rovList.get(i);
+                tagVideoIndex = i;
+                break;
+            }
+        }
+        if (insertTagVideo == null) {
+            return;
+        }
+
+        // 获取插入的权重
+        String[] insertWeight = insertRules.split(",");
+        List<Double> insertWeightList = Arrays.stream(insertWeight)
+                .map(Double::valueOf).collect(Collectors.toList());
+
+        if (CollectionUtils.size(insertWeightList) != 5) {
+            return;
+        }
+        Double showProbability = insertWeightList.get(0);
+        double randomDouble = MathUtil.nextDouble(0, 1);
+        if (randomDouble > showProbability) {
+            // 没有命中强插规则
+            return;
+        }
+        // 遍历权重，获取插入位置
+        List<WeightRandom.ItemWithWeight<Integer>> items = new ArrayList<>(insertWeightList.size());
+        for (int i = 1; i < insertWeightList.size(); i++) {
+            WeightRandom.ItemWithWeight<Integer> itemWithWeight = new WeightRandom.ItemWithWeight<>(i, insertWeightList.get(i));
+            items.add(itemWithWeight);
+        }
+        WeightRandom<Integer> integerWeightRandom = new WeightRandom<>(items);
+        Integer insertIndex = integerWeightRandom.choose();
+        if (insertIndex == null) {
+            return;
+        }
+        // 删除原视频
+        rovList.remove(tagVideoIndex);
+        // 插入该视频到指定位置
+        rovList.add(insertIndex - 1, insertTagVideo);
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorTagFilter.java

@@ -1,46 +1,60 @@
 package com.tzld.piaoquan.recommend.server.service.rank.processor;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import org.apache.commons.lang3.tuple.Pair;
 
 import java.text.SimpleDateFormat;
 import java.util.*;
-
 public class RankProcessorTagFilter {
 
-    public static void tagFitlter(List<Video> rov, List<Video> flow, Map<String, Map<String, String>> rule) {
+    public static void processor(List<Video> rov, List<Video> flow, Map<String, Map<String, String>> rule) {
 
-        Map<String, List<String>> tag2Dates = new HashMap<>();
-//        Map<String, List<String>> tag2Apps = new HashMap<>();
-        Map<String, Integer> tag2Hour = new HashMap<>();
+        Map<String, Double> tag2Rate = new HashMap<>();
+        Map<String, List<Pair<String, String>>> tag2Dates = new HashMap<>();
         for (Map.Entry<String, Map<String, String>> entry : rule.entrySet()){
             String key = entry.getKey();
             Map<String, String> value = entry.getValue();
-            if (value.containsKey("date")){
-                tag2Dates.put(key, Arrays.asList(value.get("date").split(",")));
+            if (value.containsKey("filter_rate")){
+                tag2Rate.put(key, Double.valueOf(value.get("filter_rate")));
+            }
+            if (value.containsKey("filter_date")){
+                List<Pair<String, String>> tmpList = new ArrayList<>();
+                for (String tmp : value.get("filter_date").split(",")){
+                    String start = tmp.split("-")[0];
+                    String end = tmp.split("-")[1];
+                    tmpList.add(Pair.of(start, end));
+                }
+                tag2Dates.put(key, tmpList);
             }
-//            if (value.containsKey("app")){
-//                tag2Apps.put(key, Arrays.asList(value.get("app").split(",")));
-//            }
-            if (value.containsKey("hour")){
-                tag2Hour.put(key, Integer.valueOf(value.get("hour")));
+        }
+
+        // 通过过滤概率获取本轮过滤tag集合
+        Set<String> filterTags = new HashSet<>();
+        for (Map.Entry<String, Double> entry : tag2Rate.entrySet()) {
+            String key = entry.getKey();
+            Double value = entry.getValue();
+            if (Math.random() <= value){
+                filterTags.add(key);
             }
         }
 
+        // 获取系统时间
         Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        Integer hour = Integer.valueOf(new SimpleDateFormat("HH").format(calendar.getTime()));
+        String dateHour = new SimpleDateFormat("yyyyMMddHH").format(calendar.getTime());
 
+        //执行过滤
         Iterator<Video> iterator = rov.iterator();
         while (iterator.hasNext()) {
             Video video = iterator.next();
             List<String> tags = video.getTags();
             boolean filter = false;
             for (String tag : tags){
-                if (tag2Dates.containsKey(tag) && !tag2Dates.get(tag).isEmpty() && !tag2Dates.get(tag).contains(date)){
-                    // 如果有日期存在，但日期没有命中，则不过滤，跳过即可。
+                if (!filterTags.contains(tag)){
                     continue;
                 }
-                if (tag2Hour.containsKey(tag) && hour >= tag2Hour.get(tag)){
+                boolean flag = ifFiter(tag2Dates, tag, dateHour);
+                if (flag){
                     filter = true;
+                    break;
                 }
             }
             if (filter){
@@ -54,12 +68,13 @@ public class RankProcessorTagFilter {
             List<String> tags = video.getTags();
             boolean filter = false;
             for (String tag : tags){
-                if (tag2Dates.containsKey(tag) && !tag2Dates.get(tag).isEmpty() && !tag2Dates.get(tag).contains(date)){
-                    // 如果有日期存在，但日期没有命中，则不过滤，跳过即可。
+                if (!filterTags.contains(tag)){
                     continue;
                 }
-                if (tag2Hour.containsKey(tag) && hour >= tag2Hour.get(tag)){
+                boolean flag = ifFiter(tag2Dates, tag, dateHour);
+                if (flag){
                     filter = true;
+                    break;
                 }
             }
             if (filter){
@@ -68,5 +83,16 @@ public class RankProcessorTagFilter {
         }
     }
 
+    public static boolean ifFiter(Map<String, List<Pair<String, String>>> tag2Dates, String tag, String dateHour){
+        if (!tag2Dates.containsKey(tag) || tag2Dates.get(tag).isEmpty()){
+            return false;
+        }
+        for (Pair<String, String> d: tag2Dates.get(tag)){
+            if (dateHour.compareTo(d.getLeft()) >= 0 && dateHour.compareTo(d.getRight()) < 0){
+                return true;
+            }
+        }
+        return false;
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/FestivalStrategy4RankModel.java

@@ -0,0 +1,75 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author sunxy
+ */
+@Service
+@Slf4j
+public class FestivalStrategy4RankModel extends RankService {
+
+    @Resource
+    private RankStrategy4Density rankStrategy4Density;
+
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        List<Video> rovRecallRank = new ArrayList<>();
+        rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        List<Video> festivalRecallVideoList = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        // 截断
+        if (CollectionUtils.isNotEmpty(festivalRecallVideoList)) {
+            rovRecallRank.addAll(festivalRecallVideoList.stream().limit(30).collect(Collectors.toList()));
+        }
+        // 补充不分地域小时数据
+        rovRecallRank.addAll(extractAndSort(param, RegionHWithoutDupRecallStrategy.PUSH_FORM));
+        // merge sim recall 和 return recall
+        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
+        // 地域召回要做截取，再做融合排序
+        removeDuplicate(rovRecallRank);
+
+        // 融合排序
+        List<String> videoIdKeys = rovRecallRank.stream()
+                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+                .collect(Collectors.toList());
+        List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
+//            log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                    JSONUtils.toJson(videoScores));
+        if (CollectionUtils.isNotEmpty(videoScores)
+                && videoScores.size() == rovRecallRank.size()) {
+            for (int i = 0; i < videoScores.size(); i++) {
+                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+            }
+            Collections.sort(rovRecallRank,
+                    Comparator.comparingDouble(o -> -(o.getSortScore() * 0.1 * (o.getRovScore() / 100))));
+        }
+
+        return rovRecallRank;
+    }
+
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank,
+                                   List<Video> flowPoolRank) {
+        return rankStrategy4Density.mergeAndSort(param, rovRecallRank, flowPoolRank);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java

@@ -1,17 +1,17 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 
-import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
@@ -21,132 +21,107 @@ import java.util.*;
 import java.util.stream.Collectors;
 
 /**
- * @author zhangbo
- * @desc 带密度控制的后处理 排序实验
+ * @author zhangbo sunxiaoyi
+ * @desc 后处理规则 + roc池flow池的合并
+ * 后处理参考文档：https://w42nne6hzg.feishu.cn/wiki/MYaGwCnF1iTFXUkSddAcA6CanFe
+ * roc池flow池的合并 文档： 暂无
  */
 @Service
 @Slf4j
 public class RankStrategy4Density extends RankService {
     @ApolloJsonValue("${RankStrategy4DensityFilter:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules;
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
 
-        if (CollectionUtils.isEmpty(rovRecallRank)) {
-            if (param.getSize() < flowPoolRank.size()) {
-                return new RankResult(flowPoolRank.subList(0, param.getSize()));
+         //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
             } else {
-                return new RankResult(flowPoolRank);
-            }
-        }
-        // 1 读取多样性密度控制规则------------------
-//        String appType = String.valueOf(param.getAppType());
-//        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
-//        Map<String, Integer> densityRules = new HashMap<>();
-//        if (ruleStr != null){
-//            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
-//                    new TypeToken<Map<String, Map<String, Object>>>() {},
-//                    Collections.emptyMap());
-//            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
-//                String k = entry.getKey();
-//                if (!entry.getValue().containsKey(appType)){
-//                    continue;
-//                }
-//                JSONObject jb = (JSONObject) entry.getValue().get(appType);
-//                try{
-//                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
-//                        densityRules.put(k, jb.getInteger("density"));
-//                    }
-//                }catch (Exception e){
-//                    log.error("parse densityRules is wrong:", e);
-//                }
-//            }
-//        }
-        Map<String, Integer> densityRules = new HashMap<>();
-        String abCode = param.getAbCode();
-        if (this.filterRules != null && this.filterRules.containsKey(abCode)){
-            Map<String, Map<String, String>> rule = this.filterRules.get(abCode);
-            for (Map.Entry<String, Map<String, String>> entry : rule.entrySet()){
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")){
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
+                return new RankResult(flowVideos);
             }
         }
 
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
 
-        // 2 读取video的tags------------------
-        List<Long> videoIds = new ArrayList<>();
-        for (Video v : rovRecallRank) {
-            videoIds.add(v.getVideoId());
-        }
-        for (Video v : flowPoolRank) {
-            videoIds.add(v.getVideoId());
-        }
-        Map<Long, List<String>> videoTagDict = RankExtractorFeature.getVideoTags(this.redisTemplate, videoIds);
-        for (Video v : rovRecallRank) {
-            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
         }
-        for (Video v : flowPoolRank) {
-            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
         }
 
-        // 3 读取过滤规则，根据tag和过滤规则进行过滤------------------
-        if (this.filterRules != null && this.filterRules.containsKey(abCode)){
-            Map<String, Map<String, String>> rule = this.filterRules.get(abCode);
-            RankProcessorTagFilter.tagFitlter(rovRecallRank, flowPoolRank, rule);
-        }
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
 
-        // 4 流量池按比例强插---------------------
+        //7 流量池按比例强插
         List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovRecallRank.size(); i++) {
-            result.add(rovRecallRank.get(i));
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
         }
-
         double flowPoolP = getFlowPoolP(param);
         int flowPoolIndex = 0;
         int rovPoolIndex = param.getTopK();
-
         for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
             double rand = RandomUtils.nextDouble(0, 1);
             log.info("rand={}, flowPoolP={}", rand, flowPoolP);
             if (rand < flowPoolP) {
-                if (flowPoolIndex < flowPoolRank.size()) {
-                    result.add(flowPoolRank.get(flowPoolIndex++));
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
                 } else {
                     break;
                 }
             } else {
-                if (rovPoolIndex < rovRecallRank.size()) {
-                    result.add(rovRecallRank.get(rovPoolIndex++));
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
                 } else {
                     break;
                 }
             }
         }
-        if (rovPoolIndex >= rovRecallRank.size()) {
-            for (int i = flowPoolIndex; i < flowPoolRank.size() && result.size() < param.getSize(); i++) {
-                result.add(flowPoolRank.get(i));
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
             }
         }
-        if (flowPoolIndex >= flowPoolRank.size()) {
-            for (int i = rovPoolIndex; i < rovRecallRank.size() && result.size() < param.getSize(); i++) {
-                result.add(rovRecallRank.get(i));
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
             }
         }
 
-        // 3 进行密度控制------------------
-        Set<Long> videosSet = result.stream().map(r-> r.getVideoId()).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovRecallRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowPoolRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDnsity = RankProcessorDensity.mergeDensityControl(result,
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
                 rovRecallRankNew, flowPoolRankNew, densityRules);
-//        log.info("zhangbo22 old={}, new={}", JSONUtils.toJson(result),
-//                    JSONUtils.toJson(resultWithDnsity));
-        return new RankResult(resultWithDnsity);
+
+        return new RankResult(resultWithDensity);
     }
 
+}
+
+
 
 
 //    public Video getTestVideo(Long id, String s){
@@ -156,6 +131,26 @@ public class RankStrategy4Density extends RankService {
 //        return a1;
 //    }
 
-
-
-}
+// 1 读取多样性密度控制规则------------------
+//        String appType = String.valueOf(param.getAppType());
+//        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
+//        Map<String, Integer> densityRules = new HashMap<>();
+//        if (ruleStr != null){
+//            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
+//                    new TypeToken<Map<String, Map<String, Object>>>() {},
+//                    Collections.emptyMap());
+//            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
+//                String k = entry.getKey();
+//                if (!entry.getValue().containsKey(appType)){
+//                    continue;
+//                }
+//                JSONObject jb = (JSONObject) entry.getValue().get(appType);
+//                try{
+//                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
+//                        densityRules.put(k, jb.getInteger("density"));
+//                    }
+//                }catch (Exception e){
+//                    log.error("parse densityRules is wrong:", e);
+//                }
+//            }
+//        }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/DynamicGaussianFunction.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+
+/**
+ * 动态高斯函数
+ *
+ * @author sunxy
+ */
+public class DynamicGaussianFunction {
+    private static final double SIGMA = 24.0; // 标准差，可以根据需要调整这个值
+
+    public static double calculateValue(LocalDateTime testTime, LocalDateTime startTime, LocalDateTime endTime,
+                                        LocalDateTime peakTime) {
+        long testTimestamp = testTime.toEpochSecond(ZoneOffset.UTC) * 1000;
+        // 检查时间是否在指定范围内
+        if (testTimestamp < startTime.toEpochSecond(ZoneOffset.UTC) * 1000 ||
+                testTimestamp > endTime.toEpochSecond(ZoneOffset.UTC) * 1000) {
+            return 0;
+        }
+
+        // 计算 t_0，即从开始时间到峰值时间的小时数
+        double t_0 = (peakTime.toEpochSecond(ZoneOffset.UTC) - startTime.toEpochSecond(ZoneOffset.UTC)) / 3600.0;
+
+        // 计算时间差（以小时为单位）
+        double t = (testTimestamp / 1000.0 - startTime.toEpochSecond(ZoneOffset.UTC)) / 3600.0;
+
+        // 计算高斯函数值
+        return 100 * Math.exp(-Math.pow(t - t_0, 2) / (2 * Math.pow(SIGMA, 2)));
+    }
+
+    public static void main(String[] args) {
+        // 示例：计算2024-02-03 09:00的值
+        LocalDateTime startTime = LocalDateTime.of(2024, 1, 29, 0, 0);
+        LocalDateTime endTime = LocalDateTime.of(2024, 2, 3, 23, 59);
+        LocalDateTime peakTime = LocalDateTime.of(2024, 2, 3, 9, 0);
+        LocalDateTime testTime = LocalDateTime.of(2024, 1, 31, 22, 0);
+
+        double value = calculateValue(testTime, startTime, endTime, peakTime);
+        System.out.println("Function value at " + testTime + ": " + value);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/FestivalRecallScore.java

@@ -3,23 +3,30 @@ package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerConfigInfo;
 import com.tzld.piaoquan.recommend.server.service.score4recall.AbstractScorer4Recall;
 import com.tzld.piaoquan.recommend.server.service.score4recall.model4recall.Model4RecallKeyValue;
+import com.tzld.piaoquan.recommend.server.util.ListMerger;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
+import java.util.stream.Collectors;
 
 
 public class FestivalRecallScore extends AbstractScorer4Recall {
 
-    private static final Map<String, List<String>> FESTIVAL_TIME_MAP = new HashMap<String, List<String>>() {
+    private static final Map<String, String> DAILY_BLESSING_TIME_MAP = new HashMap<String, String>() {
+        {
+            put("晚安", "daily 21:00-24:00");
+            put("晚上好", "daily 18:00-20:00");
+            put("下午好", "daily 15:00-16:00");
+            put("中午好 ", "daily 11:00-13:00");
+            put("早上好", "daily 00:00-08:00");
+        }
+    };
+
+    private static final Map<String, List<String>> YEARLY_FESTIVAL_TIME_MAP = new HashMap<String, List<String>>() {
         {
-            put("晚安", Arrays.asList("daily 21:00-24:00"));
-            put("晚上好", Arrays.asList("daily 18:00-20:00"));
-            put("下午好", Arrays.asList("daily 15:00-16:00"));
-            put("中午好 ", Arrays.asList("daily 11:00-13:00"));
-            put("早上好", Arrays.asList("daily 00:00-08:00"));
             put("圣诞节", Arrays.asList("2024-12-20 00:00~2024-12-25 08:00", "2025-12-20 00:00~2025-12-25 08:00", "2026-12-20 " +
                     "00:00~2026-12-25 08:00"));
             put("平安夜", Arrays.asList("2024-12-19 00:00~2024-12-24 08:00", "2025-12-19 00:00~2025-12-24 08:00", "2026-12-19 " +
@@ -134,33 +141,86 @@ public class FestivalRecallScore extends AbstractScorer4Recall {
     }
 
     @Override
-    public List<Pair<Long, Double>> recall(Map<String, String> params){
+    public List<Pair<Long, Double>> recall(Map<String, String> params) {
         // 节假日、时效性，判断
         Model4RecallKeyValue model = (Model4RecallKeyValue) this.getModel();
         if (model == null || model.kv == null) {
             return new ArrayList<>();
         }
-        List<Pair<Long, Double>> result = new ArrayList<>();
         LocalDateTime now = LocalDateTime.now();
-        for (Map.Entry<String, List<String>> entry : FESTIVAL_TIME_MAP.entrySet()) {
+        // 节日祝福-每年
+        List<Pair<Long, Double>> yearResult = new ArrayList<>();
+        for (Map.Entry<String, List<String>> entry : YEARLY_FESTIVAL_TIME_MAP.entrySet()) {
             String festival = entry.getKey();
             List<String> timeRangeList = entry.getValue();
             if (isFestivalTime(now, timeRangeList)) {
+                Pair<LocalDateTime, LocalDateTime> startTimeAndEndTime = getStartTimeAndEndTime(timeRangeList.get(0));
+                if (startTimeAndEndTime == null) {
+                    continue;
+                }
+                // 节日峰值设置为结束时间的当天的9点
+                double weight = DynamicGaussianFunction.calculateValue(LocalDateTime.now(), startTimeAndEndTime.getLeft(),
+                        startTimeAndEndTime.getRight(), startTimeAndEndTime.getRight().withHour(9));
+
+                List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault(festival, new ArrayList<>());
+                if (festivalLists.isEmpty()) {
+                    continue;
+                }
+                festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), weight))
+                        .limit(Math.min(50, festivalLists.size()))
+                        .collect(Collectors.toList());
+                yearResult.addAll(festivalLists);
+            }
+        }
+        List<Pair<Long, Double>> dayResult = new ArrayList<>();
+        // 每日祝福-每天固定时间段
+        for (Map.Entry<String, String> entry : DAILY_BLESSING_TIME_MAP.entrySet()) {
+            String festival = entry.getKey();
+            String timeRange = entry.getValue();
+            if (isFestivalTime(now, Collections.singletonList(timeRange))) {
                 List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault(festival, new ArrayList<>());
                 if (festivalLists.isEmpty()) {
                     continue;
                 }
-                festivalLists = festivalLists.subList(0, Math.min(100, festivalLists.size()));
-                result.addAll(festivalLists);
+                festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), 0.0))
+                        .limit(Math.min(50, festivalLists.size()))
+                        .collect(Collectors.toList());
+                dayResult.addAll(festivalLists);
             }
         }
-        // 固定获取常规祝福类的小程序
+        // 常规祝福类的小程序-任意时间
+        List<Pair<Long, Double>> anyResult = new ArrayList<>();
         List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault("祝福", new ArrayList<>());
         if (!festivalLists.isEmpty()) {
-            festivalLists = festivalLists.subList(0, Math.min(100, festivalLists.size()));
-            result.addAll(festivalLists);
+            festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), 0.0))
+                    .limit(Math.min(50, festivalLists.size()))
+                    .collect(Collectors.toList());
+            anyResult.addAll(festivalLists);
+        }
+        return ListMerger.mergeLists(yearResult, dayResult, anyResult);
+    }
+
+    public Pair<LocalDateTime, LocalDateTime> getStartTimeAndEndTime(String timeRangeList) {
+        if (timeRangeList == null || timeRangeList.isEmpty()) {
+            return null;
+        }
+        // 时间格式 2024-12-20 00:00~2024-12-25 08:00
+        if (StringUtils.startsWith(timeRangeList, "daily")) {
+            // 判断是否是 daily 开头
+            return null;
+        } else {
+            String[] split = StringUtils.split(timeRangeList, "~");
+            if (split.length != 2) {
+                return null;
+            }
+            String startTime = split[0];
+            String endTime = split[1];
+            // 解析 startTime endTime
+            DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
+            LocalDateTime startLocalDateTime = LocalDateTime.parse(startTime, dateTimeFormatter);
+            LocalDateTime endLocalDateTime = LocalDateTime.parse(endTime, dateTimeFormatter);
+            return Pair.of(startLocalDateTime, endLocalDateTime);
         }
-        return result;
     }
 
     public boolean isFestivalTime(LocalDateTime now, List<String> timeRangeList) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/ListMerger.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 合并多个List
+ *
+ * @author sunxy
+ */
+public class ListMerger {
+
+    public static <T> List<T> mergeLists(List<T> list1, List<T> list2, List<T> list3) {
+        List<T> result = new ArrayList<>();
+        int maxSize = Math.max(list1.size(), Math.max(list2.size(), list3.size()));
+
+        for (int i = 0; i < maxSize; i++) {
+            if (i < list1.size()) {
+                result.add(list1.get(i));
+            }
+            if (i < list2.size()) {
+                result.add(list2.get(i));
+            }
+            if (i < list3.size()) {
+                result.add(list3.get(i));
+            }
+        }
+
+        return result;
+    }
+
+    public static void main(String[] args) {
+        // 示例
+        List<Integer> list1 = Lists.newArrayList(1, 4, 7);
+        List<Integer> list2 = Lists.newArrayList(2, 5, 8, 10, 11);
+        List<Integer> list3 = Lists.newArrayList(3, 6, 9);
+
+        List<Integer> mergedList = mergeLists(list1, list2, list3);
+        System.out.println(mergedList);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/MathUtil.java

@@ -0,0 +1,17 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import java.util.Random;
+
+/**
+ * @author sunxy
+ */
+public class MathUtil {
+
+    /**
+     * 生成max到min范围的浮点数
+     * */
+    public static double nextDouble(final double min, final double max) {
+        return min + ((max - min) * new Random().nextDouble());
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/WeightRandom.java

@@ -0,0 +1,119 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * 基于权重随机算法选择
+ *
+ * @author sunxy
+ */
+public class WeightRandom<T> {
+
+    private final List<T> items = new ArrayList<>();
+    private double[] weights;
+
+    public WeightRandom(List<ItemWithWeight<T>> itemsWithWeight) {
+        this.calWeights(itemsWithWeight);
+    }
+
+    /**
+     * 计算权重，初始化或者重新定义权重时使用
+     */
+    private void calWeights(List<ItemWithWeight<T>> itemsWithWeight) {
+        items.clear();
+
+        // 计算权重总和
+        double originWeightSum = 0;
+        for (ItemWithWeight<T> itemWithWeight : itemsWithWeight) {
+            double weight = itemWithWeight.getWeight();
+            if (weight <= 0) {
+                continue;
+            }
+
+            items.add(itemWithWeight.getItem());
+            if (Double.isInfinite(weight)) {
+                weight = 10000.0D;
+            }
+            if (Double.isNaN(weight)) {
+                weight = 1.0D;
+            }
+            originWeightSum += weight;
+        }
+
+        // 计算每个item的实际权重比例
+        double[] actualWeightRatios = new double[items.size()];
+        int index = 0;
+        for (ItemWithWeight<T> itemWithWeight : itemsWithWeight) {
+            double weight = itemWithWeight.getWeight();
+            if (weight <= 0) {
+                continue;
+            }
+            actualWeightRatios[index++] = weight / originWeightSum;
+        }
+
+        // 计算每个item的权重范围
+        // 权重范围起始位置
+        weights = new double[items.size()];
+        double weightRangeStartPos = 0;
+        for (int i = 0; i < index; i++) {
+            weights[i] = weightRangeStartPos + actualWeightRatios[i];
+            weightRangeStartPos += actualWeightRatios[i];
+        }
+    }
+
+    /**
+     * 基于权重随机算法选择
+     */
+    public T choose() {
+        if (items.isEmpty()) {
+            return null;
+        }
+        double random = ThreadLocalRandom.current().nextDouble();
+        int index = Arrays.binarySearch(weights, random);
+        if (index < 0) {
+            index = -index - 1;
+        } else {
+            return items.get(index);
+        }
+
+        if (index < weights.length && random < weights[index]) {
+            return items.get(index);
+        }
+
+        // 通常不会走到这里，为了保证能得到正确的返回，这里随便返回一个
+        return items.get(0);
+    }
+
+    public static class ItemWithWeight<T> {
+        T item;
+        double weight;
+
+        public ItemWithWeight() {
+        }
+
+        public ItemWithWeight(T item, double weight) {
+            this.item = item;
+            this.weight = weight;
+        }
+
+        public T getItem() {
+            return item;
+        }
+
+        public void setItem(T item) {
+            this.item = item;
+        }
+
+        public double getWeight() {
+            return weight;
+        }
+
+        public void setWeight(double weight) {
+            this.weight = weight;
+        }
+    }
+
+}