Merge branch 'feature/zhangbo_rank' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV5.java

@@ -29,6 +29,7 @@ import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
@@ -149,98 +150,111 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i=0; i<24; ++i){
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items){
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeight(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeight(returns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 1.0);
-        double b = mergeWeight.getOrDefault("b", 1.0);
-        double c = mergeWeight.getOrDefault("c", 0.0002);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
-        for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5){
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-            }else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+        List<String> videoIdKeys = rovRecallRank.stream()
+                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+                .collect(Collectors.toList());
+        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+        if (CollectionUtils.isNotEmpty(videoScores)
+                && videoScores.size() == rovRecallRank.size()) {
+            for (int i = 0; i < videoScores.size(); i++) {
+                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
             }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
+            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
         }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
+        return rovRecallRank;
+
+//        // 1 模型分
+//        List<String> rtFeaPart = new ArrayList<>();
+//        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+//        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+//        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+//        Calendar calendar = Calendar.getInstance();
+//        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+//        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+//        String rtFeaPart1h = date + hour;
+//        if (rtFeaPartKeyResult != null){
+//            if (rtFeaPartKeyResult.get(1) != null){
+//                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+//            }
+//        }
+//        // 2 统计分
+//        String cur = rtFeaPart1h;
+//        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
+//        for (int i=0; i<24; ++i){
+//            datehours.add(cur);
+//            cur = ExtractorUtils.subtractHours(cur, 1);
+//        }
+//        for (RankItem item : items){
+//            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+//            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+//            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+//            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+//            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+//            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+//            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+//
+//            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+//            Double share2returnScore = calScoreWeight(share2return);
+//            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+//            Double view2returnScore = calScoreWeight(view2return);
+//            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+//            Double view2playScore = calScoreWeight(view2play);
+//            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+//            Double play2shareScore = calScoreWeight(play2share);
+//            item.scoresMap.put("share2returnScore", share2returnScore);
+//            item.scoresMap.put("view2returnScore", view2returnScore);
+//            item.scoresMap.put("view2playScore", view2playScore);
+//            item.scoresMap.put("play2shareScore", play2shareScore);
+//
+//            // 全部回流
+//            Double allreturnsScore = calScoreWeight(allreturns);
+//            item.scoresMap.put("allreturnsScore", allreturnsScore);
+//
+//            // 平台回流
+//            Double preturnsScore = calScoreWeight(returns);
+//            item.scoresMap.put("preturnsScore", preturnsScore);
+//
+//            // rov的趋势
+//            double trendScore = calTrendScore(view2return);
+//            item.scoresMap.put("trendScore", trendScore);
+//
+//            // 新视频提取
+//            double newVideoScore = calNewVideoScore(itemBasicMap);
+//            item.scoresMap.put("newVideoScore", newVideoScore);
+//        }
+//        // 3 融合公式
+//        List<Video> result = new ArrayList<>();
+//        double a = mergeWeight.getOrDefault("a", 1.0);
+//        double b = mergeWeight.getOrDefault("b", 1.0);
+//        double c = mergeWeight.getOrDefault("c", 0.0002);
+//        double d = mergeWeight.getOrDefault("d", 1.0);
+//        double e = mergeWeight.getOrDefault("e", 1.0);
+//        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
+//        for (RankItem item : items){
+//            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
+//                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+//            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
+//                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+//            double strScore = item.getScoreStr();
+//            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
+//            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
+//            double score = 0.0;
+//            if (ifAdd < 0.5){
+//                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
+//                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+//            }else {
+//                score = a * strScore + b * rosScore + c * preturnsScore +
+//                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
+//            }
+//            Video video = item.getVideo();
+//            video.setScore(score);
+//            video.setSortScore(score);
+//            video.setScoreStr(item.getScoreStr());
+//            video.setScoresMap(item.getScoresMap());
+//            result.add(video);
+//        }
+//        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        return result;
     }
     public double calNewVideoScore(Map<String, String> itemBasicMap){
         double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));