Merge branch 'master' into feature/sunxiaoyi_recall

# Conflicts:
#	recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -13,6 +13,9 @@ public class RankItem implements Comparable<RankItem> {
 
     // featureMap中保存所有的特征
     public Map<String, String> featureMap = new HashMap<>();
+    public Map<String, Double> scoresMap = new HashMap<>();
+    public Map<String, String> itemBasicFeature = new HashMap<>();
+    public Map<String, Map<String, Double>> itemRealTimeFeature = new HashMap<>();
     public long videoId;
     private double score; // 记录最终的score
     private Video video;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/Video.java

@@ -3,7 +3,9 @@ package com.tzld.piaoquan.recommend.server.model;
 import lombok.Data;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author dyp
@@ -30,9 +32,10 @@ public class Video {
     private List<String> tags = new ArrayList<>();
 
     // video的模型打分
-    private double scoreRos = 0.0D;
-    private double scoreStr = 0.0D;
+    public double scoreRos = 0.0D;
+    public double scoreStr = 0.0D;
     public double score = 0.0D;
     public double scoreRegion = 0.0D;
+    public Map<String, Double> scoresMap = new HashMap<>();
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -102,7 +102,7 @@ public abstract class AbstractFilterService {
             futures.add(future);
         }
         try {
-            cdl.await(600, TimeUnit.MILLISECONDS);
+            cdl.await(200, TimeUnit.MILLISECONDS);
         } catch (InterruptedException e) {
             log.error("filter error", e);
             return null;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -16,6 +16,16 @@ public class RankRouter {
     private RankStrategy4RankModel rankStrategy4RankModel;
     @Autowired
     private RankStrategy4Density rankStrategy4Density;
+    @Autowired
+    private RankStrategy4RegionMergeModelV1 rankStrategy4RegionMergeModelV1;
+    @Autowired
+    private RankStrategy4RegionMergeModelV2 rankStrategy4RegionMergeModelV2;
+    @Autowired
+    private RankStrategy4RegionMergeModelV3 rankStrategy4RegionMergeModelV3;
+    @Autowired
+    private RankStrategy4RegionMergeModelV4 rankStrategy4RegionMergeModelV4;
+    @Autowired
+    private RankStrategy4RegionMergeModelV5 rankStrategy4RegionMergeModelV5;
 
     @Autowired
     private RankStrategyFlowThompsonModel rankStrategyFlowThompsonModel;
@@ -44,10 +54,20 @@ public class RankRouter {
             case "60118":
             case "60119":
                 return rankStrategy4Density.rank(param);
-            case "60107":
+            case "60107": // 556
                 return rankStrategyFlowThompsonModel.rank(param);
             case "60120": // 576
                 return rankStrategy4RegionMerge.rank(param);
+            case "60121": // 536
+                return rankStrategy4RegionMergeModelV1.rank(param);
+            case "60122": // 537
+                return rankStrategy4RegionMergeModelV2.rank(param);
+            case "60123": // 541
+                return rankStrategy4RegionMergeModelV3.rank(param);
+            case "60124": // 546
+                return rankStrategy4RegionMergeModelV4.rank(param);
+            case "60125": // 547
+                return rankStrategy4RegionMergeModelV5.rank(param);
             case "60130":
                 // 先走默认排序，后续需要优化祝福类的视频排序
                 return rankService.rank(param);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV1.java

@@ -0,0 +1,441 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV1 extends RankService {
+//    @ApolloJsonValue("${video.model.weightv3:}")
+//    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV2.java

@@ -0,0 +1,440 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV2 extends RankService {
+//    @ApolloJsonValue("${video.model.weightv3:}")
+//    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
+                    Math.log(1 + item.scoresMap.getOrDefault("allreturnsScore", 0.0));
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV3.java

@@ -0,0 +1,441 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV3 extends RankService {
+//    @ApolloJsonValue("${video.model.weightv3:}")
+//    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double score = item.scoresMap.getOrDefault("view2playScore", 0.0) *
+                    item.scoresMap.getOrDefault("play2shareScore", 0.0) *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
+                    Math.log(1 + item.scoresMap.getOrDefault("allreturnsScore", 0.0));
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV4.java

@@ -0,0 +1,462 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV4 extends RankService {
+//    @ApolloJsonValue("${video.model.weightv3:}")
+//    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
+                    (1.0 + trendScore);
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+    public double calTrendScore(List<Double> data){
+        int cnt = 0;
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+            cnt ++;
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        return sum;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV5.java

@@ -0,0 +1,481 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV5 extends RankService {
+//    @ApolloJsonValue("${video.model.weightv3:}")
+//    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
+                    (1.0 + trendScore) *  (1.0 + newVideoScore)
+                    ;
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 7){
+            return 0.0;
+        }
+        if (existenceDays < 1){
+            existenceDays = 1.0;
+        }
+        double score = 1.0 / (existenceDays + 10.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        int cnt = 0;
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+            cnt ++;
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        return sum;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -89,6 +89,11 @@ public class RecallService implements ApplicationContextAware {
         } else {
             switch (abCode) {
                 case "60120": // 576
+                case "60121": // 536
+                case "60122": // 537
+                case "60123": // 541
+                case "60124": // 546
+                case "60125": // 547
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
@@ -143,6 +148,11 @@ public class RecallService implements ApplicationContextAware {
                 case "60103": // 增加地域1小时扩量，通过配置实现
                 case "60105": // 通过更改param中的配置实现使用不同数据源 data66 rule68 + 有排序模块
                 case "60120": // 576
+                case "60121": // 536
+                case "60122": // 537
+                case "60123": // 541
+                case "60124": // 546
+                case "60125": // 547
                     strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                     break;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/AbstractNewContentVideoRecallStrategy.java

@@ -34,8 +34,8 @@ public abstract class AbstractNewContentVideoRecallStrategy implements RecallStr
     @Value("${flow.pool.recent.top.video.daily.time.range:}")
     private String timeRangeJson;
 
-    @Autowired
-    protected RegionFilterService filterService;
+//    @Autowired
+//    protected RegionFilterService filterService;
 
     @Override
     public List<Video> recall(RecallParam param) {
@@ -53,17 +53,23 @@ public abstract class AbstractNewContentVideoRecallStrategy implements RecallStr
         }
         try {
             List<Long> videoIdList = JSONObject.parseArray(result.toString(), Long.class);
-            FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, Lists.newArrayList(videoIdList)));
-            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
-                return filterResult.getVideoIds().stream().map(vid -> {
-                    Video recallData = new Video();
-                    recallData.setVideoId(vid);
-                    recallData.setAbCode(param.getAbCode());
-                    recallData.setPushFrom(pushFrom());
-                    return recallData;
-                }).limit(5).collect(Collectors.toList());
-            }
-
+//            FilterResult filterResult = filterService.filter(FilterParamFactory.create(param, Lists.newArrayList(videoIdList)));
+//            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+//                return filterResult.getVideoIds().stream().map(vid -> {
+//                    Video recallData = new Video();
+//                    recallData.setVideoId(vid);
+//                    recallData.setAbCode(param.getAbCode());
+//                    recallData.setPushFrom(pushFrom());
+//                    return recallData;
+//                }).limit(5).collect(Collectors.toList());
+//            }
+            return videoIdList.stream().map(vid -> {
+                Video recallData = new Video();
+                recallData.setVideoId(vid);
+                recallData.setAbCode(param.getAbCode());
+                recallData.setPushFrom(pushFrom());
+                return recallData;
+            }).limit(5).collect(Collectors.toList());
         } catch (Exception e) {
             log.error("recall error, key={}, result={}", key, result, e);
         }