Merge branch 'feature/sunxy/202403/changeRecallBaseSortWay' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/TopRecommendPipeline.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.recommend.server.implement;
 
 
 import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.base.Stopwatch;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
@@ -46,6 +47,8 @@ public class TopRecommendPipeline {
 
     @Value("${recommend.recall.num:500}")
     private int recallNum;
+    @ApolloJsonValue("${rank.score.merge.weightv547:}")
+    private Map<String, Double> mergeWeight;
     @Resource
     private RedisSmartClient client;
     @Resource
@@ -55,6 +58,7 @@ public class TopRecommendPipeline {
     @PostConstruct
     public void init() {
         queueProvider = new RedisBackedQueue(client, 15 * 60 * 1000L);
+        mergeWeight = mergeWeight == null ? new HashMap<>() : mergeWeight;
     }
 
     public List<Video> feeds(final RecommendRequest requestData,
@@ -183,21 +187,23 @@ public class TopRecommendPipeline {
     }
 
     private List<Video> rankItem2Video(List<RankItem> items) {
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
         List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
         List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
         Calendar calendar = Calendar.getInstance();
         String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
         String hour = new SimpleDateFormat("HH").format(calendar.getTime());
         String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(1) != null){
+        if (rtFeaPartKeyResult != null) {
+            if (rtFeaPartKeyResult.get(1) != null) {
                 rtFeaPart1h = rtFeaPartKeyResult.get(1);
             }
         }
         // 2 统计分
         String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
-        for (int i=0; i<24; ++i){
+        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
+        for (int i = 0; i < 24; ++i) {
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
         }
@@ -207,12 +213,12 @@ public class TopRecommendPipeline {
             List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
             List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
             List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
             List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
 
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
             Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
             Double view2returnScore = calScoreWeight(view2return);
             List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
             Double view2playScore = calScoreWeight(view2play);
@@ -223,15 +229,65 @@ public class TopRecommendPipeline {
             item.scoresMap.put("view2playScore", view2playScore);
             item.scoresMap.put("play2shareScore", play2shareScore);
 
+            // 全部回流的rov和ros
+            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
+            Double share2allreturnScore = calScoreWeight(share2allreturn);
+            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
+            Double view2allreturnScore = calScoreWeight(view2allreturn);
+            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
+            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
+
+            // 全部回流
             Double allreturnsScore = calScoreWeight(allreturns);
             item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // 平台回流
+            Double preturnsScore = calScoreWeight(preturns);
+            item.scoresMap.put("preturnsScore", preturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
         }
         // 3 融合公式
         List<Video> result = new ArrayList<>();
-        for (RankItem item : items){
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
-                    Math.log(1 + item.scoresMap.getOrDefault("allreturnsScore", 0.0));
+        double a = mergeWeight.getOrDefault("a", 0.1);
+        double b = mergeWeight.getOrDefault("b", 0.0);
+        double c = mergeWeight.getOrDefault("c", 0.000001);
+        double d = mergeWeight.getOrDefault("d", 1.0);
+        double e = mergeWeight.getOrDefault("e", 1.0);
+        double f = mergeWeight.getOrDefault("f", 0.8);
+        double g = mergeWeight.getOrDefault("g", 2.0);
+        double h = mergeWeight.getOrDefault("h", 240.0);
+        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
+        for (RankItem item : items) {
+            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double strScore = item.getScoreStr();
+            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
+            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
+            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+            double score = 0.0;
+            if (ifAdd < 0.5) {
+                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
+            } else {
+                score = a * strScore + b * rosScore + c * preturnsScore +
+                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
+
+            }
+            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
+            if (allreturnsScore > h) {
+                score += (f * share2allreturnScore + g * view2allreturnScore);
+            }
             Video video = new Video();
             video.setVideoId(Long.parseLong(item.getId()));
             video.setPushFrom(item.getQueue());
@@ -245,6 +301,33 @@ public class TopRecommendPipeline {
         return result;
     }
 
+    public double calNewVideoScore(Map<String, String> itemBasicMap) {
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5) {
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+
+    public double calTrendScore(List<Double> data) {
+        double sum = 0.0;
+        int size = data.size();
+        for (int i = 0; i < size - 4; ++i) {
+            sum += data.get(i) - data.get(i + 4);
+        }
+        if (sum * 10 > 0.6) {
+            sum = 0.6;
+        } else {
+            sum = sum * 10;
+        }
+        if (sum > 0) {
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
+
     private void duplicate(List<RankItem> items) {
         Set<String> ids = new HashSet<>();
         List<RankItem> result = new ArrayList<>();