Merge branch 'feature/sunxy/20240326/Mod610Sort' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/TopRecommendPipeline.java

@@ -1,7 +1,6 @@
 package com.tzld.piaoquan.recommend.server.implement;
 
 
-import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.base.Stopwatch;
 import com.google.common.reflect.TypeToken;
@@ -186,9 +185,18 @@ public class TopRecommendPipeline {
         return rovRecallRankNewScore;
     }
 
+    public Double calScoreWeightNoTimeDecay(List<Double> data) {
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i = 0; i < data.size(); ++i) {
+            up += 1.0 * data.get(i);
+            down += 1.0;
+        }
+        return down > 1E-8 ? up / down : 0.0;
+    }
+
     private List<Video> rankItem2Video(List<RankItem> items) {
         // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
         List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
         List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
         Calendar calendar = Calendar.getInstance();
@@ -207,7 +215,7 @@ public class TopRecommendPipeline {
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
         }
-        for (RankItem item : items){
+        for (RankItem item : items) {
             Map<String, String> itemBasicMap = item.getItemBasicFeature();
             Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
             List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
@@ -217,13 +225,13 @@ public class TopRecommendPipeline {
             List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
 
             List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
+            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
             List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
+            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
             List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
+            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
             List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
+            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
             item.scoresMap.put("share2returnScore", share2returnScore);
             item.scoresMap.put("view2returnScore", view2returnScore);
             item.scoresMap.put("view2playScore", view2playScore);
@@ -231,18 +239,18 @@ public class TopRecommendPipeline {
 
             // 全部回流的rov和ros
             List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeight(share2allreturn);
+            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
             List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeight(view2allreturn);
+            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
             item.scoresMap.put("share2allreturnScore", share2allreturnScore);
             item.scoresMap.put("view2allreturnScore", view2allreturnScore);
 
             // 全部回流
-            Double allreturnsScore = calScoreWeight(allreturns);
+            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
 
             // 平台回流
-            Double preturnsScore = calScoreWeight(preturns);
+            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
             item.scoresMap.put("preturnsScore", preturnsScore);
 
             // rov的趋势

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global1hHotCandidate.java

@@ -27,9 +27,10 @@ public class Global1hHotCandidate extends IndexCandidateQueue {
 
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
         int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
 
 
@@ -37,6 +38,10 @@ public class Global1hHotCandidate extends IndexCandidateQueue {
                 .addMatch("type", "global1h");
         currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
 
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
+
         queueName = new QueueName(ItemType, "short_rov")
                 .addMatch("type", "global1h");
         currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global24hHotCandidate.java

@@ -26,10 +26,10 @@ public class Global24hHotCandidate extends IndexCandidateQueue {
         Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
-
+        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
 
         QueueName queueName = new QueueName(ItemType, "rov")
                 .addMatch("type", "global24h");
@@ -43,6 +43,10 @@ public class Global24hHotCandidate extends IndexCandidateQueue {
                 .addMatch("type", "global24h");
         currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum, getStrategyQueueInfo().getQueueName());
 
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "global24h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
+
         candidates.putAll(simplifiedCandidates);
         return currRecall;
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global3hHotCandidate.java

@@ -27,9 +27,10 @@ public class Global3hHotCandidate extends IndexCandidateQueue {
 
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
 
 
         QueueName queueName = new QueueName(ItemType, "rov")
@@ -44,6 +45,9 @@ public class Global3hHotCandidate extends IndexCandidateQueue {
                 .addMatch("type", "global3h");
         currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum, getStrategyQueueInfo().getQueueName());
 
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "global3h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
 
         candidates.putAll(simplifiedCandidates);
         return currRecall;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region1hHotCandidate.java

@@ -27,10 +27,11 @@ public class Region1hHotCandidate extends IndexCandidateQueue {
         Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
         int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
 
 
         QueueName queueName = new QueueName(ItemType, "short_rov")
@@ -58,6 +59,12 @@ public class Region1hHotCandidate extends IndexCandidateQueue {
         currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum,
                 getStrategyQueueInfo().getQueueName());
 
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "region1h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
+                getStrategyQueueInfo().getQueueName());
+
         candidates.putAll(simplifiedCandidates);
         return currRecall;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region24hHotCandidate.java

@@ -28,9 +28,10 @@ public class Region24hHotCandidate extends IndexCandidateQueue {
 
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
+        int combinationNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
         int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.4)).intValue();
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
 
         // index key  video:queue:type=region24h:region=北京:ordering=rov
         QueueName queueName = new QueueName(ItemType, "rov")
@@ -52,6 +53,13 @@ public class Region24hHotCandidate extends IndexCandidateQueue {
         currRecall += addCandidateKey(simplifiedCandidates, queueName, combinationNum,
                 getStrategyQueueInfo().getQueueName());
 
+        // index key  video:queue:type=region24h:region=北京:ordering=str
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "region24h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
+                getStrategyQueueInfo().getQueueName());
+
         candidates.putAll(simplifiedCandidates);
         return currRecall;
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region3hHotCandidate.java

@@ -27,9 +27,10 @@ public class Region3hHotCandidate extends IndexCandidateQueue {
         Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
 
         BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
-        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
-        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.3)).intValue();
+        int rosNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
         int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.2)).intValue();
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
 
 
         // index key video:queue:type=region3h:region=北京:ordering=short_rov
@@ -53,6 +54,12 @@ public class Region3hHotCandidate extends IndexCandidateQueue {
         currRecall += addCandidateKey(simplifiedCandidates, queueName, rosNum,
                 getStrategyQueueInfo().getQueueName());
 
+        // index key video:queue:type=region3h:region=北京:ordering=str
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "region3h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
+                getStrategyQueueInfo().getQueueName());
 
         candidates.putAll(simplifiedCandidates);
         return currRecall;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/VideoRecommendService.java

@@ -45,7 +45,7 @@ public class VideoRecommendService {
     @Resource
     private FlowPoolRecommendPipeline flowPoolRecommendPipeline;
     @Resource
-    private RankStrategy4RegionMergeModelV547 rankStrategy4RegionMergeModelV2;
+    private RankStrategy4RegionMergeModelV547 rankStrategy4RegionMergeModelV547;
     @Autowired
     @Qualifier("redisTemplate")
     private RedisTemplate<String, String> redisTemplate;
@@ -139,7 +139,7 @@ public class VideoRecommendService {
 
         stopwatch.reset().start();
         RankParam rankParam = recommendService.convertToRankParam(param, null);
-        RankResult rankResult = rankStrategy4RegionMergeModelV2.mergeAndSort(rankParam, topRecommendVideoList,
+        RankResult rankResult = rankStrategy4RegionMergeModelV547.mergeAndSort(rankParam, topRecommendVideoList,
                 flowPoolRecommendVideoList);
         if (newLogPrint) {
             log.info("traceId = {}, cost = {}, final rankResult [{}]", request.getRequestId(),