Merge branch 'master' into feature/zhangbo_rank

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/merger/MergeUtils.java

@@ -119,12 +119,7 @@ public class MergeUtils {
                                           int freeRecNum,
                                           int expId) {
         // 先定义按score排序优选的独立额
-        PriorityQueue<Pair<String, Integer>> mergePriorityQueue = new PriorityQueue<Pair<String, Integer>>(freeRecNum, new Comparator<Pair<String, Integer>>() {
-            @Override
-            public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
-                return rankerItemsListMap.get(o1.getLeft()).getRight().get(o1.getRight()).compareTo(rankerItemsListMap.get(o2.getLeft()).getRight().get(o2.getRight()));
-            }
-        });
+        PriorityQueue<Pair<String, Integer>> mergePriorityQueue = new PriorityQueue<Pair<String, Integer>>(freeRecNum, Comparator.comparing(o -> rankerItemsListMap.get(o.getLeft()).getRight().get(o.getRight())));
 
         // 大于最小mergenum 同时小于maxMergenum，在队列中add
         for (Pair<MergeRule, List<RankItem>> entry : rankerItemsListMap.values()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/TopRecommendPipeline.java

@@ -91,13 +91,34 @@ public class TopRecommendPipeline {
         return videos;
     }
 
+    private List<Video> rankItem2Video(List<RankItem> rankItems) {
+        List<Video> videos = new ArrayList<>();
+        for (RankItem item : rankItems) {
+            Video video = new Video();
+            video.setVideoId(Long.parseLong(item.getId()));
+            video.setPushFrom(item.getQueue());
+            video.setScore(item.getScore());
+            video.setSortScore(item.getScore());
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+
+            Map<String, List<String>> pushFromIndex = new HashMap<>();
+            pushFromIndex.put(item.getQueue(), item.getCandidateInfoList().stream()
+                    .map(CandidateInfo::getCandidateQueueName).collect(Collectors.toList()));
+            video.setPushFromIndex(pushFromIndex);
+            videos.add(video);
+        }
+        videos.sort(Comparator.comparing(Video::getScore).reversed());
+        return videos;
+    }
+
     public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key){
+                                      List<String> datehours, String key) {
         List<Double> views = new LinkedList<>();
         Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours){
+        for (String dh : datehours) {
             views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
             );
         }
         return views;
@@ -158,14 +179,18 @@ public class TopRecommendPipeline {
         timeLogMap.put("recalling-cost", stopwatch.elapsed().toMillis() + "");
         timeLogMap.put("recalling-size", items == null ? "0" : items.size() + "");
 
-        // Step 4: Advance Scoring
-//        timestamp = System.currentTimeMillis();
-//        ScorerPipeline scorerPipeline = getScorerPipeline(requestData);
-//        items = scorerPipeline.scoring(requestData, userInfo, requestIndex, items);
         if (CollectionUtils.isEmpty(items)) {
             return new ArrayList<>();
         }
 
+        // Step 4: Advance Scoring
+        stopwatch.reset().start();
+        videoScoredByFeature(items);
+        if (logPrint) {
+            log.info("traceId = {}, cost = {}, items = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(items));
+        }
+
         stopwatch.reset().start();
         // Step 5: Merger
         MergeUtils.distributeItemsToMultiQueues(topQueue, items);
@@ -188,18 +213,7 @@ public class TopRecommendPipeline {
 //        MergeUtils.diversityRerank(mergeItems, SimilarityUtils.getIsSameUserTagOrCategoryFunc(), recallNum, 6, 2);
 
         // Step 6: Global Rank & subList
-        // TODO 前置和后置处理逻辑 hardcode，后续优化
-        stopwatch.reset().start();
-        List<RankItem> rovRecallRankNewScore = rankByScore(mergeItems, requestData);
-
-        timeLogMap.put("rankByScore-cost", stopwatch.elapsed().toMillis() + "");
-        timeLogMap.put("rankByScore-size", rovRecallRankNewScore.size() + "");
-
-        if (logPrint) {
-            log.info("traceId = {}, cost = {}, rovRecallRankNewScore = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(rovRecallRankNewScore));
-        }
-        return rovRecallRankNewScore;
+        return mergeItems;
     }
 
     public Double calScoreWeightNoTimeDecay(List<Double> data) {
@@ -212,7 +226,7 @@ public class TopRecommendPipeline {
         return down > 1E-8 ? up / down : 0.0;
     }
 
-    private List<Video> rankItem2Video(List<RankItem> items) {
+    private void videoScoredByFeature(List<RankItem> items) {
         // 1 模型分
         List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
         List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
@@ -280,7 +294,6 @@ public class TopRecommendPipeline {
 
         }
         // 3 融合公式
-        List<Video> result = new ArrayList<>();
         double a = mergeWeight.getOrDefault("a", 0.1);
         double b = mergeWeight.getOrDefault("b", 0.0);
         double c = mergeWeight.getOrDefault("c", 0.000001);
@@ -300,7 +313,7 @@ public class TopRecommendPipeline {
             double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
             double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
             double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
+            double score;
             if (ifAdd < 0.5) {
                 score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
                         (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
@@ -313,22 +326,9 @@ public class TopRecommendPipeline {
             if (allreturnsScore > h) {
                 score += (f * share2allreturnScore + g * view2allreturnScore);
             }
-            Video video = new Video();
-            video.setVideoId(Long.parseLong(item.getId()));
-            video.setPushFrom(item.getQueue());
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-
-            Map<String, List<String>> pushFromIndex = new HashMap<>();
-            pushFromIndex.put(item.getQueue(), item.getCandidateInfoList().stream()
-                    .map(CandidateInfo::getCandidateQueueName).collect(Collectors.toList()));
-            video.setPushFromIndex(pushFromIndex);
-            result.add(video);
+            // 设置计算好的分数
+            item.setScore(score);
         }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
     }
 
     public double calNewVideoScore(Map<String, String> itemBasicMap) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global1hHotCandidate.java

@@ -27,24 +27,24 @@ public class Global1hHotCandidate extends IndexCandidateQueue {
 
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.8)).intValue();
-//        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-//        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
 
 
         QueueName queueName = new QueueName(ItemType, "rov")
                 .addMatch("type", "global1h");
         currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
 
-//        queueName = new QueueName(ItemType, "str")
-//                .addMatch("type", "global1h");
-//        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
 
-//        queueName = new QueueName(ItemType, "short_rov")
-//                .addMatch("type", "global1h");
-//        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
+        queueName = new QueueName(ItemType, "short_rov")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
 
         queueName = new QueueName(ItemType, "short_ros")
                 .addMatch("type", "global1h");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region1hHotCandidate.java

@@ -27,11 +27,11 @@ public class Region1hHotCandidate extends IndexCandidateQueue {
         Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.6)).intValue();
-//        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-//        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
 
 
         QueueName queueName = new QueueName(ItemType, "short_rov")
@@ -47,11 +47,11 @@ public class Region1hHotCandidate extends IndexCandidateQueue {
         currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
                 getStrategyQueueInfo().getQueueName());
 
-//        queueName = new QueueName(ItemType, "short_ros")
-//                .addMatch("type", "region1h")
-//                .addMatch("region", region);
-//        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
-//                getStrategyQueueInfo().getQueueName());
+        queueName = new QueueName(ItemType, "short_ros")
+                .addMatch("type", "region1h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
+                getStrategyQueueInfo().getQueueName());
 
         queueName = new QueueName(ItemType, "combination")
                 .addMatch("type", "region1h")
@@ -59,11 +59,11 @@ public class Region1hHotCandidate extends IndexCandidateQueue {
         currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum,
                 getStrategyQueueInfo().getQueueName());
 
-//        queueName = new QueueName(ItemType, "str")
-//                .addMatch("type", "region1h")
-//                .addMatch("region", region);
-//        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
-//                getStrategyQueueInfo().getQueueName());
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "region1h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
+                getStrategyQueueInfo().getQueueName());
 
         candidates.putAll(simplifiedCandidates);
         return currRecall;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region24hHotCandidate.java

@@ -28,10 +28,10 @@ public class Region24hHotCandidate extends IndexCandidateQueue {
 
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.6)).intValue();
+        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
         int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
-//        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
 
         // index key  video:queue:type=region24h:region=北京:ordering=rov
         QueueName queueName = new QueueName(ItemType, "rov")
@@ -54,11 +54,11 @@ public class Region24hHotCandidate extends IndexCandidateQueue {
                 getStrategyQueueInfo().getQueueName());
 
         // index key  video:queue:type=region24h:region=北京:ordering=str
-//        queueName = new QueueName(ItemType, "str")
-//                .addMatch("type", "region24h")
-//                .addMatch("region", region);
-//        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
-//                getStrategyQueueInfo().getQueueName());
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "region24h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
+                getStrategyQueueInfo().getQueueName());
 
         candidates.putAll(simplifiedCandidates);
         return currRecall;

recommend-server-service/src/main/resources/merge_config.conf

@@ -40,14 +40,12 @@ rule-config = {
     merge-rule = {
       global-hot-queue = {
         recall-percentage = 0.5
-        min-merge-num = 100
-        max-merge-num = 200
+        max-merge-num = 400
         priority = 1
       }
       region-hot-queue = {
         recall-percentage = 0.5
-        min-merge-num = 100
-        max-merge-num = 200
+        max-merge-num = 400
         priority = 1
       }
     }
@@ -57,19 +55,16 @@ rule-config = {
   global-hot-queue = {
     merge-rule = {
       global1h-index = {
-        recall-percentage = 0.7
-        min-merge-num = 140
-        max-merge-num = 280
+        recall-percentage = 0.3
+        max-merge-num = 200
       }
       global3h-index = {
-        recall-percentage = 0.1
-        min-merge-num = 20
-        max-merge-num = 40
+        recall-percentage = 0.3
+        max-merge-num = 200
       }
       global24h-index = {
-        recall-percentage = 0.2
-        min-merge-num = 40
-        max-merge-num = 80
+        recall-percentage = 0.3
+        max-merge-num = 200
       }
     }
   }
@@ -78,23 +73,19 @@ rule-config = {
   region-hot-queue = {
     merge-rule = {
       region1h-index = {
-        recall-percentage = 0.6
-        min-merge-num = 120
-        max-merge-num = 240
+        recall-percentage = 0.3
+        max-merge-num = 200
 
       }
       region3h-index = {
-        recall-percentage = 0.1
-        min-merge-num = 20
-        max-merge-num = 40
+        recall-percentage = 0.3
+        max-merge-num = 200
       }
       region24h-index = {
         recall-percentage = 0.3
-        min-merge-num = 60
-        max-merge-num = 120
+        max-merge-num = 200
       }
     }
   }
 
-}
-
+}