feat:对二级品类降权

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/RankExtractorForPostProcessor.java

@@ -1,47 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.extractor;
-
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.FeatureService;
-import com.tzld.piaoquan.recommend.server.service.ServiceBeanFactory;
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class RankExtractorForPostProcessor {
-
-    public static void fullMergeCate2FromFeature(List<Video> rovList, List<Video> flowVideos) {
-
-        FeatureService featureService = ServiceBeanFactory.getBean(FeatureService.class);
-
-        List<String> vidStrList = new ArrayList<>(rovList.size() + flowVideos.size());
-        rovList.stream().map(v -> String.valueOf(v.getVideoId())).forEach(vidStrList::add);
-        flowVideos.stream().map(v -> String.valueOf(v.getVideoId())).forEach(vidStrList::add);
-
-        Map<String, Map<String, Map<String, String>>> featureMap = featureService.getVideoBaseInfo("", vidStrList);
-
-        parseAndSetMergeCate2(rovList, featureMap);
-        parseAndSetMergeCate2(flowVideos, featureMap);
-    }
-
-    private static void parseAndSetMergeCate2(List<Video> videos, Map<String, Map<String, Map<String, String>>> featureMap) {
-        for (Video video : videos) {
-            String vidStr = String.valueOf(video.getVideoId());
-            if (!featureMap.containsKey(vidStr)) {
-                continue;
-            }
-
-            Map<String, Map<String, String>> vidFeature = featureMap.get(vidStr);
-            if (!vidFeature.containsKey("alg_vid_feature_basic_info")) {
-                continue;
-            }
-
-            Map<String, String> basicInfoMap = vidFeature.get("alg_vid_feature_basic_info");
-            String mergeCate2 = basicInfoMap.getOrDefault("merge_second_level_cate", "");
-            if (StringUtils.isNotBlank(mergeCate2)) {
-                video.getMergeCateList().add(mergeCate2);
-            }
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankMergeCate2PostProcessor.java

@@ -1,99 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.processor;
-
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.util.DateUtils;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-
-import java.time.LocalDateTime;
-import java.util.List;
-import java.util.Map;
-
-public class RankMergeCate2PostProcessor {
-
-    public static void mergeCate2PostProcessor(RankParam param, List<Video> rovList, Map<String, Map<String, String>> rulesMap) {
-        if (CollectionUtils.isEmpty(rovList) || MapUtils.isEmpty(rulesMap)) {
-            return;
-        }
-
-        // 解析配置，判断当前是否需要对品类进行降权
-        String mergeCate2 = null;
-        double reduceCoefficient = 1d;
-        for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-            Map<String, String> configMap = entry.getValue();
-            boolean currentHourIsNeedReduce = currentHourIsNeedReduce(configMap);
-            boolean currentDateIsNeedReduce = currentDateIsNeedReduce(configMap);
-            if (currentHourIsNeedReduce || currentDateIsNeedReduce) {
-                mergeCate2 = entry.getKey();
-                reduceCoefficient = Double.parseDouble(configMap.getOrDefault("reduce_coefficient", "1"));
-                break;
-            }
-        }
-
-        if (StringUtils.isEmpty(mergeCate2)) {
-            return;
-        }
-
-        for (Video video : rovList) {
-            if (CollectionUtils.isEmpty(video.getMergeCateList())) {
-                continue;
-            }
-
-            if (!video.getMergeCateList().contains(mergeCate2)) {
-                continue;
-            }
-            double sortScore = video.getSortScore();
-            double newScore = sortScore * reduceCoefficient;
-
-            video.setSortScore(newScore);
-            video.setScore(newScore);
-            if (MapUtils.isNotEmpty(video.getScoresMap())) {
-                video.getScoresMap().put("originSortScore", sortScore);
-                video.getScoresMap().put("reduceCoefficient", reduceCoefficient);
-            }
-        }
-
-        // 根据分数重新排序
-        rovList.sort((o1, o2) -> Double.compare(o2.getSortScore(), o1.getSortScore()));
-
-    }
-
-    /**
-     * 当前小时是否需要降权
-     */
-    private static boolean currentHourIsNeedReduce(Map<String, String> configMap) {
-        if (!configMap.containsKey("reduce_hour")) {
-            return false;
-        }
-
-        String currentHourStr = String.valueOf(LocalDateTime.now().getHour());
-        String[] split = configMap.get("reduce_hour").split(",");
-        for (String s : split) {
-            if (StringUtils.equals(s, currentHourStr)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
-    /**
-     * 当前时间段是否需要降权
-     */
-    private static boolean currentDateIsNeedReduce(Map<String, String> configMap) {
-        if (!configMap.containsKey("reduce_date")) {
-            return false;
-        }
-
-        String[] reduceDates = configMap.get("reduce_date").split(",");
-        for (String s : reduceDates) {
-            if (DateUtils.ifTimeRangeInNow(s)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelBasic.java

@@ -8,7 +8,6 @@ import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorForPostProcessor;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.*;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
@@ -76,9 +75,6 @@ public abstract class RankStrategy4RegionMergeModelBasic extends RankService {
         if (rulesMap != null && !rulesMap.isEmpty()) {
             RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
             extractorItemTags.processor(rovVideos, flowVideos);
-
-            RankExtractorForPostProcessor.fullMergeCate2FromFeature(rovVideos, flowVideos);
-
         }
         // 6 合并结果时间卡控
         if (rulesMap != null && !rulesMap.isEmpty()) {
@@ -91,9 +87,6 @@ public abstract class RankStrategy4RegionMergeModelBasic extends RankService {
         // 5 rov池强插功能
         RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
 
-        // 品类降权功能
-        RankMergeCate2PostProcessor.mergeCate2PostProcessor(param, rovVideos, rulesMap);
-
         // 7 流量池按比例强插
         List<Video> result = new ArrayList<>();
         for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV536.java

@@ -10,13 +10,15 @@ import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.DateUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.math3.util.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.time.LocalDateTime;
 import java.util.*;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
@@ -28,12 +30,17 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
     @ApolloJsonValue("${rank.score.merge.weightv536:}")
     private Map<String, Double> mergeWeight;
 
+    @ApolloJsonValue("${rank.score.reduce.config.v536:}")
+    private Map<String, List<Map<String, String>>> reduceConfigMap;
+
     @Autowired
     private FeatureService featureService;
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
+        Map<String, List<Map<String, String>>> reduceConfigMap = this.reduceConfigMap != null ? this.reduceConfigMap : new HashMap<>(0);
+
         //-------------------融-------------------
         //-------------------合-------------------
         //-------------------逻-------------------
@@ -66,6 +73,18 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
         v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
         rovRecallRank.addAll(v1);
         setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------scene cf rovn------------------
+        List<Video> sceneCFRovn = extractAndSort(param, SceneCFRovnRecallStrategy.PUSH_FORM);
+        sceneCFRovn = sceneCFRovn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        sceneCFRovn = sceneCFRovn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), sceneCFRovn.size()));
+        rovRecallRank.addAll(sceneCFRovn);
+        setVideo.addAll(sceneCFRovn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------scene cf rosn------------------
+        List<Video> sceneCFRosn = extractAndSort(param, SceneCFRosnRecallStrategy.PUSH_FORM);
+        sceneCFRosn = sceneCFRosn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
+        rovRecallRank.addAll(sceneCFRosn);
+        setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -313,16 +332,25 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
             item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
             double fmRov = restoreScore(fmRovOrigin);
             item.getScoresMap().put("fmRov", fmRov);
+
             double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+
             double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
             item.getScoresMap().put("vor", vor);
-            score = fmRov * (0.1 + hasReturnRovScore) * (0.1 + vor);
+
+            String vidStr = String.valueOf(item.getVideo().getVideoId());
+            String mergeCate2 = this.parseMergeCate2(vidStr, featureOriginVideo);
+            double reduceCoefficient = this.parseReduceCoefficient(mergeCate2, reduceConfigMap);
+            item.getScoresMap().put("reduceCoefficient", reduceCoefficient);
+
+            score = fmRov * (0.1 + hasReturnRovScore) * (0.1 + vor) * reduceCoefficient;
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
             video.setScoresMap(item.getScoresMap());
             video.setAllFeatureMap(item.getAllFeatureMap());
+            video.getMergeCateList().add(mergeCate2);
             if (MapUtils.isNotEmpty(feature.getVideoFeature()) && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
                 video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
             }
@@ -334,4 +362,81 @@ public class RankStrategy4RegionMergeModelV536 extends RankStrategy4RegionMergeM
         result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
+
+
+    public double parseReduceCoefficient(String mergeCate2, Map<String, List<Map<String, String>>> reduceConfigMap) {
+        if (StringUtils.isBlank(mergeCate2) || MapUtils.isEmpty(reduceConfigMap)) {
+            return 1d;
+        }
+
+        List<Map<String, String>> configMaps = reduceConfigMap.getOrDefault(mergeCate2, new ArrayList<>());
+        for (Map<String, String> configMap : configMaps) {
+            boolean currentHourIsNeedReduce = currentHourIsNeedReduce(configMap);
+            boolean currentDateIsNeedReduce = currentDateIsNeedReduce(configMap);
+            if (currentHourIsNeedReduce || currentDateIsNeedReduce) {
+                return Double.parseDouble(configMap.getOrDefault("reduce_coefficient", "1"));
+            }
+        }
+
+        return 1d;
+    }
+
+
+    /**
+     * 当前小时是否需要降权
+     */
+    private boolean currentHourIsNeedReduce(Map<String, String> configMap) {
+        if (!configMap.containsKey("reduce_hour")) {
+            return false;
+        }
+        try {
+
+            int currentHour = LocalDateTime.now().getHour();
+            String[] reduceHours = configMap.get("reduce_hour").split(",");
+            for (String hourRange : reduceHours) {
+                String[] split = hourRange.split("-");
+                int h1 = Integer.parseInt(split[0]);
+                int h2 = (split.length == 2) ? Integer.parseInt(split[1]) : h1;
+
+                if (currentHour >= h1 && currentHour <= h2) {
+                    return true;
+                }
+            }
+        } catch (Exception ignore) {
+        }
+        return false;
+    }
+
+    /**
+     * 当前时间段是否需要降权
+     */
+    private boolean currentDateIsNeedReduce(Map<String, String> configMap) {
+        if (!configMap.containsKey("reduce_date")) {
+            return false;
+        }
+
+        String[] reduceDates = configMap.get("reduce_date").split(",");
+        for (String s : reduceDates) {
+            if (DateUtils.ifTimeRangeInNow(s)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+
+    private String parseMergeCate2(String vidStr, Map<String, Map<String, Map<String, String>>> featureMap) {
+        if (!featureMap.containsKey(vidStr)) {
+            return "";
+        }
+
+        Map<String, Map<String, String>> vidFeature = featureMap.get(vidStr);
+        if (!vidFeature.containsKey("alg_vid_feature_basic_info")) {
+            return "";
+        }
+
+        Map<String, String> basicInfoMap = vidFeature.get("alg_vid_feature_basic_info");
+        return basicInfoMap.getOrDefault("merge_second_level_cate", "");
+    }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV563.java

@@ -14,7 +14,6 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.math3.util.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.util.*;