FIX：排序分计算

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/TopRecommendPipeline.java

@@ -12,24 +12,22 @@ import com.tzld.piaoquan.recommend.server.framework.merger.MergeUtils;
 import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueue;
 import com.tzld.piaoquan.recommend.server.framework.recaller.BaseRecaller;
 import com.tzld.piaoquan.recommend.server.framework.recaller.provider.RedisBackedQueue;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
 import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.PostConstruct;
@@ -52,7 +50,12 @@ public class TopRecommendPipeline {
     @Resource
     private RedisSmartClient client;
     @Resource
-    public RedisTemplate<String, String> redisTemplate;
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Qualifier("featureRedisTemplate")
+    @Autowired
+    private RedisTemplate<String, String> featureRedisTemplate;
+
     private RedisBackedQueue queueProvider;
 
     @PostConstruct
@@ -185,15 +188,15 @@ public class TopRecommendPipeline {
 
         // Step 4: Advance Scoring
         stopwatch.reset().start();
-        videoScoredByFeature(items);
+        List<RankItem> rankItemList = videoScoredByFeature(items, requestData);
         if (logPrint) {
-            log.info("traceId = {}, cost = {}, items = {}", requestData.getRequestId(),
-                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(items));
+            log.info("traceId = {}, cost = {}, rankItemList = {}", requestData.getRequestId(),
+                    stopwatch.elapsed().toMillis(), JSONUtils.toJson(rankItemList));
         }
 
         stopwatch.reset().start();
         // Step 5: Merger
-        MergeUtils.distributeItemsToMultiQueues(topQueue, items);
+        MergeUtils.distributeItemsToMultiQueues(topQueue, rankItemList);
         topQueue.merge(recallNum * 3, userInfo, requestData, requestIndex, 0);
 
         // 多样性融合
@@ -226,8 +229,9 @@ public class TopRecommendPipeline {
         return down > 1E-8 ? up / down : 0.0;
     }
 
-    private void videoScoredByFeature(List<RankItem> items) {
+    private List<RankItem> videoScoredByFeature(List<RankItem> items, RecommendRequest recommendRequest) {
         // 1 模型分
+        List<RankItem> rankItemList = model(items, recommendRequest);
         List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
         List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
         Calendar calendar = Calendar.getInstance();
@@ -246,7 +250,7 @@ public class TopRecommendPipeline {
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
         }
-        for (RankItem item : items) {
+        for (RankItem item : rankItemList) {
             Map<String, String> itemBasicMap = item.getItemBasicFeature();
             Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
             List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
@@ -303,7 +307,7 @@ public class TopRecommendPipeline {
         double g = mergeWeight.getOrDefault("g", 2.0);
         double h = mergeWeight.getOrDefault("h", 240.0);
         double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
+        for (RankItem item : rankItemList) {
             double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
                     item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
             double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
@@ -329,79 +333,29 @@ public class TopRecommendPipeline {
             // 设置计算好的分数
             item.setScore(score);
         }
+        return rankItemList;
     }
 
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
+    private List<RankItem> model(List<RankItem> items, RecommendRequest param) {
+        if (items.isEmpty()) {
+            return items;
         }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    private void duplicate(List<RankItem> items) {
-        Set<String> ids = new HashSet<>();
-        List<RankItem> result = new ArrayList<>();
-        for (RankItem item : items) {
-            if (ids.contains(item.getId())) {
-                continue;
-            }
-            ids.add(item.getId());
-            result.add(item);
-        }
-        items.clear();
-        items.addAll(result);
-    }
-
-    public List<RankItem> rankByScore(List<RankItem> rankItems, RecommendRequest param){
-        List<RankItem> result = new ArrayList<>();
-        if (rankItems.isEmpty()){
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
 
         // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
 
         // 1: user特征处理
         Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()){
+        if (param.getMid() != null && !param.getMid().isEmpty()) {
             String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null){
-                try{
+            String userFeatureStr = featureRedisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null) {
+                try {
                     userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {},
+                            new TypeToken<Map<String, String>>() {
+                            },
                             userFeatureMap);
-                }catch (Exception e){
+                } catch (Exception e) {
                     log.error(String.format("parse user json is wrong in {} with {}", this.getClass().getSimpleName(), e));
                 }
             }
@@ -418,188 +372,7 @@ public class TopRecommendPipeline {
                 iterator.remove();
             }
         }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<String> videoIds = CommonCollectionUtils.toListDistinct(rankItems, RankItem::getId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null){
-            for (int i=0; i<videoFeatures.size(); ++i){
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null){
-                    continue;
-                }
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMap);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                }catch (Exception e){
-                    log.error(String.format("parse video json is wrong in {} with {}", this.getClass().getSimpleName(), e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(0) != null){
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
 
-
-        if (videoRtFeatures != null){
-            int j = 0;
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}",
-                            this.getClass().getSimpleName(), e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}",
-                            this.getClass().getSimpleName(), e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF_FEED)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getUserFeatureMap(RecommendRequest param, List<RankItem> rankItems) {
-        Map<String, String> userFeatureMap = new HashMap<>(64);
-        if (param.getMid() != null && !param.getMid().isEmpty()){
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null){
-                try{
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {},
-                            userFeatureMap);
-                }catch (Exception e){
-                    log.error(String.format("parse user json is wrong in {} with {}", this.getClass().getSimpleName(), e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
         Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
                 new HashSet<String>(Arrays.asList(
                         "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
@@ -622,21 +395,22 @@ public class TopRecommendPipeline {
                 "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
         ));
 
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(rankItems, RankItem::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+        List<String> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
                 .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null){
-            for (int i=0; i<videoFeatures.size(); ++i){
+        List<String> videoFeatures = featureRedisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null) {
+            for (int i = 0; i < videoFeatures.size(); ++i) {
                 String vF = videoFeatures.get(i);
                 Map<String, String> vfMap = new HashMap<>();
-                if (vF == null){
+                if (vF == null) {
                     continue;
                 }
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                try {
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
+                    }, vfMap);
                     Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
+                    items.get(i).setItemBasicFeature(vfMapCopy);
                     Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
                     while (iteratorIn.hasNext()) {
                         Map.Entry<String, String> entry = iteratorIn.next();
@@ -652,8 +426,8 @@ public class TopRecommendPipeline {
                                     "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
                     );
                     f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                }catch (Exception e){
+                    items.get(i).setFeatureMap(f4);
+                } catch (Exception e) {
                     log.error(String.format("parse video json is wrong in {} with {}", this.getClass().getSimpleName(), e));
                 }
             }
@@ -666,37 +440,34 @@ public class TopRecommendPipeline {
         String hour = new SimpleDateFormat("HH").format(calendar.getTime());
         String rtFeaPart1day = date + hour;
         String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(0) != null){
+        if (rtFeaPartKeyResult != null) {
+            if (rtFeaPartKeyResult.get(0) != null) {
                 rtFeaPart1day = rtFeaPartKeyResult.get(0);
             }
-            if (rtFeaPartKeyResult.get(1) != null){
+            if (rtFeaPartKeyResult.get(1) != null) {
                 rtFeaPart1h = rtFeaPartKeyResult.get(1);
             }
         }
 
-        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
                 .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
                 .collect(Collectors.toList());
         videoRtKeys1.addAll(videoRtKeys2);
         List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
 
 
-        if (videoRtFeatures != null){
+        if (videoRtFeatures != null) {
             int j = 0;
-            for (RankItem item: rankItems){
+            for (RankItem item : items) {
+                String vF = videoRtFeatures.get(j);
                 ++j;
-                if (j >= rankItems.size()) {
+                if (vF == null) {
                     continue;
                 }
                 Map<String, String> vfMap = new HashMap<>();
                 Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
                 try {
-                    String vF = videoRtFeatures.get(j);
-                    if (vF == null) {
-                        continue;
-                    }
                     vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
                     }, vfMap);
                     for (Map.Entry<String, String> entry : vfMap.entrySet()) {
@@ -712,25 +483,22 @@ public class TopRecommendPipeline {
                         }
                         vfMapNew.put(entry.getKey(), tmp);
                     }
-                }catch (Exception e){
+                } catch (Exception e) {
                     log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}",
                             this.getClass().getSimpleName(), e));
                 }
                 Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
                 item.getFeatureMap().putAll(f8);
             }
-            for (RankItem item: rankItems){
+            for (RankItem item : items) {
+                String vF = videoRtFeatures.get(j);
                 ++j;
-                if (j >= rankItems.size()) {
+                if (vF == null) {
                     continue;
                 }
                 Map<String, String> vfMap = new HashMap<>();
                 Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
                 try {
-                    String vF = videoRtFeatures.get(j);
-                    if (vF == null) {
-                        continue;
-                    }
                     vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
                     }, vfMap);
 
@@ -742,13 +510,13 @@ public class TopRecommendPipeline {
                         String[] var1 = value.split(",");
                         Map<String, Double> tmp = new HashMap<>();
                         for (String var2 : var1) {
-                            String [] var3 = var2.split(":");
+                            String[] var3 = var2.split(":");
                             tmp.put(var3[0], Double.valueOf(var3[1]));
                         }
                         vfMapNew.put(entry.getKey(), tmp);
                     }
                     item.setItemRealTimeFeature(vfMapNew);
-                }catch (Exception e){
+                } catch (Exception e) {
                     log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}",
                             this.getClass().getSimpleName(), e));
                 }
@@ -758,7 +526,50 @@ public class TopRecommendPipeline {
         }
 
 
-        return userFeatureMap;
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, items);
+        return rovRecallScore;
+    }
+
+    public double calNewVideoScore(Map<String, String> itemBasicMap) {
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 5) {
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+
+    public double calTrendScore(List<Double> data) {
+        double sum = 0.0;
+        int size = data.size();
+        for (int i = 0; i < size - 4; ++i) {
+            sum += data.get(i) - data.get(i + 4);
+        }
+        if (sum * 10 > 0.6) {
+            sum = 0.6;
+        } else {
+            sum = sum * 10;
+        }
+        if (sum > 0) {
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
+
+    private void duplicate(List<RankItem> items) {
+        Set<String> ids = new HashSet<>();
+        List<RankItem> result = new ArrayList<>();
+        for (RankItem item : items) {
+            if (ids.contains(item.getId())) {
+                continue;
+            }
+            ids.add(item.getId());
+            result.add(item);
+        }
+        items.clear();
+        items.addAll(result);
     }
 
     private Map<String, String> getSceneFeature(RecommendRequest param) {