Merge remote-tracking branch 'origin/master' into vlog_merge_refactor_smz

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.recommend.server.service.filter;
 
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
 import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.service.PreViewedService;
 import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
@@ -241,7 +242,7 @@ public abstract class AbstractFilterService {
         if (CollectionUtils.isEmpty(videoIds)) {
             return Collections.emptyList();
         }
-        List<Long> result = new ArrayList<>();
+        List<Long> result = Lists.newArrayList(param.getVideoIds());
         for (int i = 0; i < videoIds.size(); ++i) {
             result.retainAll(videoIds.get(i));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -75,7 +75,8 @@ public class RankRouter {
             case "60126": // 548
                 return rankStrategy4RegionMergeModelV6.rank(param);
             case "60130":
-                // 先走默认排序，后续需要优化祝福类的视频排序
+            case "60131":
+            case "60132":
                 return festivalStrategy4RankModel.rank(param);
             default:
                 break;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/FestivalStrategy4RankModel.java

@@ -35,13 +35,23 @@ public class FestivalStrategy4RankModel extends RankService {
         rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
         rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
         rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        removeDuplicate(rovRecallRank);
+        String abCode = param.getAbCode();
+        int sizeReturn = param.getSize();
+        if (abCode != null && this.regionRecallReturnSize != null &&
+                this.regionRecallReturnSize.containsKey(abCode) &&
+                this.regionRecallReturnSize.get(abCode) != null) {
+            sizeReturn = this.regionRecallReturnSize.get(abCode).getOrDefault("result_size", sizeReturn);
+        }
+        rovRecallRank = rovRecallRank.size() <= sizeReturn
+                ? rovRecallRank
+                : rovRecallRank.subList(0, sizeReturn);
+
         List<Video> festivalRecallVideoList = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
         // 截断
         if (CollectionUtils.isNotEmpty(festivalRecallVideoList)) {
             rovRecallRank.addAll(festivalRecallVideoList.stream().limit(30).collect(Collectors.toList()));
         }
-        // 补充不分地域小时数据
-        rovRecallRank.addAll(extractAndSort(param, RegionHWithoutDupRecallStrategy.PUSH_FORM));
         // merge sim recall 和 return recall
         rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
         rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV3.java

@@ -10,10 +10,16 @@ import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
@@ -22,6 +28,7 @@ import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -42,6 +49,8 @@ import java.util.stream.Collectors;
 public class RankStrategy4RegionMergeModelV3 extends RankService {
     @ApolloJsonValue("${rank.score.merge.weight:}")
     private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilter:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
@@ -535,4 +544,92 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
         return sceneFeatureMap;
     }
 
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV6.java

@@ -6,15 +6,23 @@ import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -35,6 +43,8 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 public class RankStrategy4RegionMergeModelV6 extends RankService {
     @ApolloJsonValue("${rank.score.merge.weight:}")
     private Map<String, Double> mergeWeight;
+    @ApolloJsonValue("${RankStrategy4DensityFilter:}")
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
@@ -483,4 +493,92 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         return sceneFeatureMap;
     }
 
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
+
+        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowVideos);
+            }
+        }
+
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
+        }
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        }
+
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+
+        //7 流量池按比例强插
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
+        }
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
+            }
+        }
+
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+
+        return new RankResult(resultWithDensity);
+    }
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -177,6 +177,8 @@ public class RecallService implements ApplicationContextAware {
                     strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                     break;
                 case "60130":
+                case "60131":
+                case "60132":
                     strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/FestivalRecallScore.java

@@ -158,9 +158,9 @@ public class FestivalRecallScore extends AbstractScorer4Recall {
                 if (startTimeAndEndTime == null) {
                     continue;
                 }
-                // 节日峰值设置为结束时间的当天的9点
+                // 节日峰值设置为结束时间的当天的7点
                 double weight = DynamicGaussianFunction.calculateValue(LocalDateTime.now(), startTimeAndEndTime.getLeft(),
-                        startTimeAndEndTime.getRight(), startTimeAndEndTime.getRight().withHour(9));
+                        startTimeAndEndTime.getRight(), startTimeAndEndTime.getRight().withHour(7));
 
                 List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault(festival, new ArrayList<>());
                 if (festivalLists.isEmpty()) {