多样性实验

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -10,6 +10,9 @@ import java.util.Map;
 
 @Data
 public class RankItem implements Comparable<RankItem> {
+
+    // featureMap中保存所有的特征
+    public Map<String, Object> featureMap = new HashMap<>();
     public long videoId;
     private double score; // 记录最终的score
     private Video video;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -17,7 +17,7 @@ import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.RankRouter;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallService;
@@ -68,7 +68,7 @@ public class RecommendService {
     @Autowired
     private RecallService recallService;
     @Autowired
-    private RankService rankService;
+    private RankRouter rankRouter;
     @Autowired
     private PreViewedService preViewedService;
     @Autowired
@@ -164,7 +164,7 @@ public class RecommendService {
         rankParam.setSize(request.getSize());
         rankParam.setSpecialRecommend(true);
 
-        RankResult rankResult = rankService.rank(rankParam);
+        RankResult rankResult = rankRouter.rank(rankParam);
         log.info("rankParam, rankResult {}", JSONUtils.toJson(rankParam), JSONUtils.toJson(rankResult));
 
         if (rankResult == null || CollectionUtils.isEmpty(rankResult.getVideos())) {
@@ -348,7 +348,7 @@ public class RecommendService {
                 stopwatch.elapsed(TimeUnit.MILLISECONDS));
         stopwatch.reset().start();
 
-        RankResult rankResult = rankService.rank(convertToRankParam(param, recallResult));
+        RankResult rankResult = rankRouter.rank(convertToRankParam(param, recallResult));
         log.info("rankResult={}, videoRecommend rank cost={}", rankResult,
                 stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -0,0 +1,27 @@
+package com.tzld.piaoquan.recommend.server.service.rank;
+
+import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4Density;
+import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4RankModel;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+@Service
+@Slf4j
+public class RankRouter {
+    private RankService rankService;
+    public RankResult rank(RankParam param) {
+        String abCode = param.getAbCode();
+        switch (abCode){
+            case "70010":
+                rankService = new RankStrategy4RankModel();
+                break;
+            case "60098":
+                rankService = new RankStrategy4Density();
+                break;
+            default:
+                rankService = new RankService();
+                break;
+        }
+        return rankService.rank(param);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -19,6 +19,8 @@ import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.Getter;
+import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
@@ -40,9 +42,9 @@ import java.util.stream.Collectors;
 @Slf4j
 public class RankService {
     @Autowired
-    private RedisTemplate<String, String> redisTemplate;
+    public RedisTemplate<String, String> redisTemplate;
     @Autowired
-    private FeatureRemoteService featureRemoteService;
+    public FeatureRemoteService featureRemoteService;
     @Value("${newRankSwitch:false}")
     private boolean newRankSwitch;
     @Value("${newRankAbExpCode:528}")
@@ -79,16 +81,9 @@ public class RankService {
                 JSONUtils.toJson(flowPoolRank));
 
         // 融合排序
-        String abCode = param.getAbCode();
-        switch (abCode){
-            case "60098":
-                return this.mergeAndSort4Density(param, rovRecallRank, flowPoolRank);
-            default:
-                return mergeAndSort(param, rovRecallRank, flowPoolRank);
-        }
-    }
+        return mergeAndSort(param, rovRecallRank, flowPoolRank);    }
 
-    private List<Video> mergeAndRankRovRecall(RankParam param) {
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
         // TODO ab test
         // TODO 抽象成Strategy
         boolean hitTest = newRankSwitch
@@ -100,7 +95,7 @@ public class RankService {
         }
     }
 
-    private List<Video> mergeAndRankRovRecallOld(RankParam param) {
+    public List<Video> mergeAndRankRovRecallOld(RankParam param) {
         List<Video> rovRecallRank = new ArrayList<>();
         rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
         rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
@@ -182,7 +177,7 @@ public class RankService {
         return rovRecallRank;
     }
 
-    private void removeDuplicate(List<Video> rovRecallRank) {
+    public void removeDuplicate(List<Video> rovRecallRank) {
         if (CollectionUtils.isNotEmpty(rovRecallRank)) {
             Set<Long> videoIds = new HashSet<>();
             Iterator<Video> ite = rovRecallRank.iterator();
@@ -259,7 +254,7 @@ public class RankService {
         return scoreParam;
     }
 
-    private List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
         if (param.getAppType() == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
                 || param.getAppType() == AppTypeEnum.ZUI_JING_QI.getCode()) {
             if (param.getAbCode().equals("60054")
@@ -280,7 +275,7 @@ public class RankService {
         }
     }
 
-    private List<Video> extractAndSort(RankParam param, String pushFrom) {
+    public List<Video> extractAndSort(RankParam param, String pushFrom) {
         if (param == null
                 || param.getRecallResult() == null
                 || CollectionUtils.isEmpty(param.getRecallResult().getData())) {
@@ -299,7 +294,7 @@ public class RankService {
         return Collections.emptyList();
     }
 
-    private double getFlowPoolP(RankParam param) {
+    public double getFlowPoolP(RankParam param) {
         if (param.getAppType() == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
                 || param.getAppType() == AppTypeEnum.ZUI_JING_QI.getCode()) {
             return param.getFlowPoolP();
@@ -314,7 +309,7 @@ public class RankService {
         }
     }
 
-    private void removeDuplicate(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
+    public void removeDuplicate(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
         // TODO 重构 rov和流量池 融合排序
         //    去重原则：
         //        如果视频在ROV召回池topK，则保留ROV召回池，否则保留流量池
@@ -346,7 +341,7 @@ public class RankService {
         }
     }
 
-    private RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
+    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
         if (CollectionUtils.isEmpty(rovRecallRank)) {
             if (param.getSize() < flowPoolRank.size()) {
                 return new RankResult(flowPoolRank.subList(0, param.getSize()));
@@ -394,120 +389,6 @@ public class RankService {
         return new RankResult(result);
     }
 
-    private RankResult mergeAndSort4Density(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
-        // 测试，添加数据
-//        rovRecallRank.add(0, getTestVideo(1070462L, ""));
-//        rovRecallRank.add(0, getTestVideo(1085062L, ""));
-//        rovRecallRank.add(0, getTestVideo(1102552L, ""));
-//        rovRecallRank.add(0, getTestVideo(1115929L, ""));
-//
-//        flowPoolRank = new ArrayList<>();
-//        flowPoolRank.add(getTestVideo(1263931L, "flow"));
-//        flowPoolRank.add(getTestVideo(1284797L, "flow"));
-//        flowPoolRank.add(getTestVideo(1289001L, "flow"));
-//        flowPoolRank.add(getTestVideo(1331503L, "flow"));
-
-
-        if (CollectionUtils.isEmpty(rovRecallRank)) {
-            if (param.getSize() < flowPoolRank.size()) {
-                return new RankResult(flowPoolRank.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowPoolRank);
-            }
-        }
-        // 1 读取多样性密度控制规则
-        String appType = String.valueOf(param.getAppType());
-        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (ruleStr != null){
-            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
-                    new TypeToken<Map<String, Map<String, Object>>>() {},
-                    Collections.emptyMap());
-            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
-                String k = entry.getKey();
-                if (!entry.getValue().containsKey(appType)){
-                    continue;
-                }
-                JSONObject jb = (JSONObject) entry.getValue().get(appType);
-                try{
-                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
-                        densityRules.put(k, jb.getInteger("density"));
-                    }
-                }catch (Exception e){
-                    log.error("parse densityRules is wrong:", e);
-                }
-
-            }
-        }
-        // 2 读取video的tags
-        List<Long> videoIds = new ArrayList<>();
-        for (Video v : rovRecallRank) {
-            videoIds.add(v.getVideoId());
-        }
-        for (Video v : flowPoolRank) {
-            videoIds.add(v.getVideoId());
-        }
-        Map<Long, List<String>> videoTagDict = RankExtractorFeature.getVideoTags(this.redisTemplate, videoIds);
-        for (Video v : rovRecallRank) {
-            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
-        }
-        for (Video v : flowPoolRank) {
-            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
-        }
-        // ------------------读取video的tags完成---------------------
-
-
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovRecallRank.size(); i++) {
-            result.add(rovRecallRank.get(i));
-        }
-
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowPoolRank.size()) {
-                    result.add(flowPoolRank.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovRecallRank.size()) {
-                    result.add(rovRecallRank.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovRecallRank.size()) {
-            for (int i = flowPoolIndex; i < flowPoolRank.size() && result.size() < param.getSize(); i++) {
-                result.add(flowPoolRank.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowPoolRank.size()) {
-            for (int i = rovPoolIndex; i < rovRecallRank.size() && result.size() < param.getSize(); i++) {
-                result.add(rovRecallRank.get(i));
-            }
-        }
-
-        // 3 进行密度控制
-        Set<Long> videosSet = result.stream().map(r-> r.getVideoId()).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovRecallRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowPoolRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDnsity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-        return new RankResult(resultWithDnsity);
-    }
 
-//    public Video getTestVideo(Long id, String s){
-//        Video a1 = new Video();
-//        a1.setVideoId(id);
-//        a1.setFlowPool(s);
-//        return a1;
-//    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java

@@ -0,0 +1,148 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.SpecialRecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Service
+@Slf4j
+public class RankStrategy4Density extends RankService {
+
+    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
+        // 测试，添加数据
+//        rovRecallRank.add(0, getTestVideo(1070462L, ""));
+//        rovRecallRank.add(0, getTestVideo(1085062L, ""));
+//        rovRecallRank.add(0, getTestVideo(1102552L, ""));
+//        rovRecallRank.add(0, getTestVideo(1115929L, ""));
+//
+//        flowPoolRank = new ArrayList<>();
+//        flowPoolRank.add(getTestVideo(1263931L, "flow"));
+//        flowPoolRank.add(getTestVideo(1284797L, "flow"));
+//        flowPoolRank.add(getTestVideo(1289001L, "flow"));
+//        flowPoolRank.add(getTestVideo(1331503L, "flow"));
+
+
+        if (CollectionUtils.isEmpty(rovRecallRank)) {
+            if (param.getSize() < flowPoolRank.size()) {
+                return new RankResult(flowPoolRank.subList(0, param.getSize()));
+            } else {
+                return new RankResult(flowPoolRank);
+            }
+        }
+        // 1 读取多样性密度控制规则
+        String appType = String.valueOf(param.getAppType());
+        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (ruleStr != null){
+            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
+                    new TypeToken<Map<String, Map<String, Object>>>() {},
+                    Collections.emptyMap());
+            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
+                String k = entry.getKey();
+                if (!entry.getValue().containsKey(appType)){
+                    continue;
+                }
+                JSONObject jb = (JSONObject) entry.getValue().get(appType);
+                try{
+                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
+                        densityRules.put(k, jb.getInteger("density"));
+                    }
+                }catch (Exception e){
+                    log.error("parse densityRules is wrong:", e);
+                }
+
+            }
+        }
+        // 2 读取video的tags
+        List<Long> videoIds = new ArrayList<>();
+        for (Video v : rovRecallRank) {
+            videoIds.add(v.getVideoId());
+        }
+        for (Video v : flowPoolRank) {
+            videoIds.add(v.getVideoId());
+        }
+        Map<Long, List<String>> videoTagDict = RankExtractorFeature.getVideoTags(this.redisTemplate, videoIds);
+        for (Video v : rovRecallRank) {
+            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        }
+        for (Video v : flowPoolRank) {
+            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        }
+        // ------------------读取video的tags完成---------------------
+
+
+        List<Video> result = new ArrayList<>();
+        for (int i = 0; i < param.getTopK() && i < rovRecallRank.size(); i++) {
+            result.add(rovRecallRank.get(i));
+        }
+
+        double flowPoolP = getFlowPoolP(param);
+        int flowPoolIndex = 0;
+        int rovPoolIndex = param.getTopK();
+
+        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
+            double rand = RandomUtils.nextDouble(0, 1);
+            log.info("rand={}, flowPoolP={}", rand, flowPoolP);
+            if (rand < flowPoolP) {
+                if (flowPoolIndex < flowPoolRank.size()) {
+                    result.add(flowPoolRank.get(flowPoolIndex++));
+                } else {
+                    break;
+                }
+            } else {
+                if (rovPoolIndex < rovRecallRank.size()) {
+                    result.add(rovRecallRank.get(rovPoolIndex++));
+                } else {
+                    break;
+                }
+            }
+        }
+        if (rovPoolIndex >= rovRecallRank.size()) {
+            for (int i = flowPoolIndex; i < flowPoolRank.size() && result.size() < param.getSize(); i++) {
+                result.add(flowPoolRank.get(i));
+            }
+        }
+        if (flowPoolIndex >= flowPoolRank.size()) {
+            for (int i = rovPoolIndex; i < rovRecallRank.size() && result.size() < param.getSize(); i++) {
+                result.add(rovRecallRank.get(i));
+            }
+        }
+
+        // 3 进行密度控制
+        Set<Long> videosSet = result.stream().map(r-> r.getVideoId()).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovRecallRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowPoolRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDnsity = RankProcessorDensity.mergeDensityControl(result,
+                rovRecallRankNew, flowPoolRankNew, densityRules);
+        return new RankResult(resultWithDnsity);
+    }
+
+//    public Video getTestVideo(Long id, String s){
+//        Video a1 = new Video();
+//        a1.setVideoId(id);
+//        a1.setFlowPool(s);
+//        return a1;
+//    }
+
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RankModel.java

@@ -0,0 +1,166 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
+import com.tzld.piaoquan.recommend.server.model.MachineInfo;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.remote.FeatureRemoteService;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author dyp
+ */
+@Service
+@Slf4j
+public class RankStrategy4RankModel extends RankService {
+
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+
+        //-------------------地域内部融合-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+
+        removeDuplicate(rovRecallRank);
+        rovRecallRank = rovRecallRank.size() <= param.getSize()
+                ? rovRecallRank
+                : rovRecallRank.subList(0, param.getSize());
+
+        //-------------------地域 sim returnv2 融合-------------------
+        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
+        removeDuplicate(rovRecallRank);
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+        List<String> videoIdKeys = rovRecallRank.stream()
+                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+                .collect(Collectors.toList());
+        List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
+        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+                JSONUtils.toJson(videoScores));
+        if (CollectionUtils.isNotEmpty(videoScores)
+                && videoScores.size() == rovRecallRank.size()) {
+            for (int i = 0; i < videoScores.size(); i++) {
+                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+            }
+            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+        }
+
+        // todo zhangbo 增加排序str模型逻辑
+
+
+
+        return rovRecallRank;
+    }
+
+    public List<Video> model(List<Video> videos, RankParam param){
+        if (videos.isEmpty()){
+            return videos;
+        }
+
+        Map<String, Object> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = this.redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, Object>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}",
+                            this.CLASS_NAME, e));
+                }
+            }
+        }
+        log.info("userFeature in model = {}", JSONUtils.toJson(userFeatureMap));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = this.redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, Object> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, Object>>() {}, vfMap);
+                    rankItems.get(i).setFeatureMap(vfMap);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}",
+                            this.CLASS_NAME, e));
+                }
+            }
+        }
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        sceneFeatureMap.put("ctx_region", param.getProvince());
+        sceneFeatureMap.put("ctx_city", param.getCity());
+        Calendar calendar = Calendar.getInstance();
+
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+
+
+}