conflict repair

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -20,7 +20,10 @@ public class RankItem implements Comparable<RankItem> {
     // featureMap中保存所有的特征
     public Map<String, String> featureMap = new HashMap<>();
     public String id;
-    public long videoId;
+    public Map<String, Double> scoresMap = new HashMap<>();
+    public Map<String, String> itemBasicFeature = new HashMap<>();
+    public Map<String, Map<String, Double>> itemRealTimeFeature = new HashMap<>();
+gi    public long videoId;
     private double score; // 记录最终的score
     private Video video;
     private double scoreRos; // 记录ros的score

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/enums/AppTypeEnum.java

@@ -46,6 +46,7 @@ public enum AppTypeEnum {
     ZUI_JING_QI(19, "票圈最惊奇"),
     PIAO_QUAN_VIDEO_PLUS(21, "票圈视频+"),
     JOURNEY(22, "票圈足迹"),
+    PIAO_QUAN_MEIHAO_ZHUFU(25, "票圈美好祝福"),
     ADMIN_CRAWLER(888888, "后台-爬虫"),
     OTHERS(999999, "其他的调用方，比如后台管理系统");
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/Video.java

@@ -3,7 +3,9 @@ package com.tzld.piaoquan.recommend.server.model;
 import lombok.Data;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @author dyp
@@ -30,9 +32,10 @@ public class Video {
     private List<String> tags = new ArrayList<>();
 
     // video的模型打分
-    private double scoreRos = 0.0D;
-    private double scoreStr = 0.0D;
+    public double scoreRos = 0.0D;
+    public double scoreStr = 0.0D;
     public double score = 0.0D;
     public double scoreRegion = 0.0D;
+    public Map<String, Double> scoresMap = new HashMap<>();
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -53,9 +53,9 @@ public abstract class AbstractFilterService {
         obj.put("concurrent", param.concurrent);
         obj.put("notUsePreView", param.notUsePreView);
         obj.put("size1", videoIds.size());
-        if (param.isNotUsePreView()){
-           ;
-        }else {
+        if (param.isNotUsePreView()) {
+            ;
+        } else {
             videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
         }
         obj.put("size2", videoIds.size());
@@ -102,7 +102,7 @@ public abstract class AbstractFilterService {
             futures.add(future);
         }
         try {
-            cdl.await(600, TimeUnit.MILLISECONDS);
+            cdl.await(150, TimeUnit.MILLISECONDS);
         } catch (InterruptedException e) {
             log.error("filter error", e);
             return null;
@@ -241,11 +241,8 @@ public abstract class AbstractFilterService {
         if (CollectionUtils.isEmpty(videoIds)) {
             return Collections.emptyList();
         }
-        List<Long> result = videoIds.get(0);
-        if (CollectionUtils.isEmpty(result)) {
-            return Collections.emptyList();
-        }
-        for (int i = 1; i < videoIds.size(); ++i) {
+        List<Long> result = new ArrayList<>();
+        for (int i = 0; i < videoIds.size(); ++i) {
             result.retainAll(videoIds.get(i));
         }
         log.info("filter result {}", JSONUtils.toJson(result));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -16,6 +16,20 @@ public class RankRouter {
     private RankStrategy4RankModel rankStrategy4RankModel;
     @Autowired
     private RankStrategy4Density rankStrategy4Density;
+    @Autowired
+    private RankStrategy4RegionMergeModelV1 rankStrategy4RegionMergeModelV1;
+    @Autowired
+    private RankStrategy4RegionMergeModelV2 rankStrategy4RegionMergeModelV2;
+    @Autowired
+    private RankStrategy4RegionMergeModelV3 rankStrategy4RegionMergeModelV3;
+    @Autowired
+    private RankStrategy4RegionMergeModelV4 rankStrategy4RegionMergeModelV4;
+    @Autowired
+    private RankStrategy4RegionMergeModelV5 rankStrategy4RegionMergeModelV5;
+    @Autowired
+    private RankStrategy4RegionMergeModelV6 rankStrategy4RegionMergeModelV6;
+    @Autowired
+    private FestivalStrategy4RankModel festivalStrategy4RankModel;
 
     @Autowired
     private RankStrategyFlowThompsonModel rankStrategyFlowThompsonModel;
@@ -44,10 +58,25 @@ public class RankRouter {
             case "60118":
             case "60119":
                 return rankStrategy4Density.rank(param);
-            case "60107":
+            case "60107": // 556
                 return rankStrategyFlowThompsonModel.rank(param);
             case "60120": // 576
                 return rankStrategy4RegionMerge.rank(param);
+            case "60121": // 536
+                return rankStrategy4RegionMergeModelV1.rank(param);
+            case "60122": // 537
+                return rankStrategy4RegionMergeModelV2.rank(param);
+            case "60123": // 541
+                return rankStrategy4RegionMergeModelV3.rank(param);
+            case "60124": // 546
+                return rankStrategy4RegionMergeModelV4.rank(param);
+            case "60125": // 547
+                return rankStrategy4RegionMergeModelV5.rank(param);
+            case "60126": // 548
+                return rankStrategy4RegionMergeModelV6.rank(param);
+            case "60130":
+                // 先走默认排序，后续需要优化祝福类的视频排序
+                return festivalStrategy4RankModel.rank(param);
             default:
                 break;
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -65,7 +65,28 @@ public class RankService {
             }
             return null;
         }
+        // 1 通过 apptype 判断该小程序走怎样的排序策略。
+        if (param.getAppType() == AppTypeEnum.PIAO_QUAN_MEIHAO_ZHUFU.getCode()){
+            List<Video> results = new ArrayList<>();
+            results.addAll(extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM));
+            List<String> videoIdKeys = results.stream()
+                    .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+                    .collect(Collectors.toList());
+            List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
+            if (CollectionUtils.isNotEmpty(videoScores)
+                    && videoScores.size() == results.size()) {
+                for (int i = 0; i < videoScores.size(); i++) {
+                    results.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+                }
+                Collections.sort(results, Comparator.comparingDouble(o -> -o.getSortScore()));
+            }
+            results.addAll(extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM));
+            results.addAll(extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM));
+            removeDuplicate(results);
+            return new RankResult(results);
+        }
 
+        // 2 正常走分发 排序+冷启动
         List<Video> rovRecallRank = mergeAndRankRovRecall(param);
         // log.info("mergeAndRankRovRecall rovRecallRank={}", JSONUtils.toJson(rovRecallRank));
         List<Video> flowPoolRank = mergeAndRankFlowPoolRecall(param);
@@ -83,21 +104,14 @@ public class RankService {
         return mergeAndSort(param, rovRecallRank, flowPoolRank);
     }
 
-    public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank){
+    public void rankFilter(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
         return;
     }
 
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         // TODO ab test
         // TODO 抽象成Strategy
-//        boolean hitTest = newRankSwitch
-//                || CommonCollectionUtils.contains(param.getAbExpCodes(), newRankAbExpCode);
-//        if (hitTest) {
-//            return mergeAndRankRovRecallNew(param);
-//        } else {
-//            return mergeAndRankRovRecallOld(param);
-//        }
-        // 研发重写了代码，直接走新逻辑
+
         return mergeAndRankRovRecallOld(param);
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/RankExtractorItemTags.java

@@ -0,0 +1,51 @@
+package com.tzld.piaoquan.recommend.server.service.rank.extractor;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import org.springframework.data.redis.core.RedisTemplate;
+
+import java.util.*;
+
+public class RankExtractorItemTags {
+    public RedisTemplate<String, String> redisTemplate;
+    public RankExtractorItemTags(RedisTemplate<String, String> redisTemplate){
+        this.redisTemplate = redisTemplate;
+    }
+
+    public void processor(List<Video> rovVideos, List<Video> flowVideos){
+        List<Long> videoIds = new ArrayList<>();
+        for (Video v : rovVideos) {
+            videoIds.add(v.getVideoId());
+        }
+        for (Video v : flowVideos) {
+            videoIds.add(v.getVideoId());
+        }
+        Map<Long, List<String>> videoTagDict = getVideoTags(redisTemplate, videoIds);
+        for (Video v : rovVideos) {
+            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        }
+        for (Video v : flowVideos) {
+            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        }
+    }
+
+    public static Map<Long, List<String>> getVideoTags(RedisTemplate<String, String> redisHelper, List<Long> videoIds) {
+        String REDIS_PREFIX = "alg_recsys_video_tags_";
+        List<String> redisKeys = new ArrayList<>();
+        for (Long videoId : videoIds) {
+            redisKeys.add(REDIS_PREFIX + String.valueOf(videoId));
+        }
+        List<String> videoTags = redisHelper.opsForValue().multiGet(redisKeys);
+        Map<Long, List<String>> videoTagDict = new HashMap<>();
+        if (videoTags != null) {
+            for (int i = 0; i < videoTags.size(); i++) {
+                String tagsStr = videoTags.get(i);
+                List<String> tags = new ArrayList<>();
+                if (tagsStr != null && !tagsStr.isEmpty()) {
+                    String[] tagsArray = tagsStr.split(",");
+                    tags = new ArrayList<>(Arrays.asList(tagsArray));
+                }
+                videoTagDict.put(videoIds.get(i), tags);
+            }
+        }
+        return videoTagDict;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorBoost.java

@@ -0,0 +1,44 @@
+package com.tzld.piaoquan.recommend.server.service.rank.processor;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.math.BigDecimal;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 提权处理器
+ *
+ * @author sunxy
+ */
+public class RankProcessorBoost {
+
+    public static void boostByTag(List<Video> rovList, Map<String, Map<String, String>> rulesMap) {
+        if (CollectionUtils.isEmpty(rovList) || rulesMap == null || rulesMap.isEmpty()) {
+            return;
+        }
+        Map<String, Double> densityRules = new HashMap<>();
+        for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+            String key = entry.getKey();
+            Map<String, String> value = entry.getValue();
+            if (value.containsKey("boost")) {
+                densityRules.put(key, Double.valueOf(value.get("boost")));
+            }
+        }
+
+
+        for (Map.Entry<String, Double> entry : densityRules.entrySet()) {
+            rovList.stream().filter(video -> video.getTags().contains(entry.getKey()))
+                    .forEach(video -> video.setSortScore(
+                            BigDecimal.valueOf(video.getSortScore())
+                                    .multiply(BigDecimal.valueOf(entry.getValue()))
+                                    .doubleValue()));
+        }
+        rovList.sort((o1, o2) -> Double.compare(o2.getSortScore(), o1.getSortScore()));
+
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorInsert.java

@@ -0,0 +1,94 @@
+package com.tzld.piaoquan.recommend.server.service.rank.processor;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.util.MathUtil;
+import com.tzld.piaoquan.recommend.server.util.WeightRandom;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author sunxy
+ */
+public class RankProcessorInsert {
+
+    public static void insertByTag(RankParam param, List<Video> rovList,
+                                   Map<String, Map<String, String>> rulesMap) {
+        if (CollectionUtils.isEmpty(rovList) || rulesMap == null || rulesMap.isEmpty()) {
+            return;
+        }
+        String insertRules = null, tagName = null;
+        for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+            Map<String, String> value = entry.getValue();
+            if (value.containsKey("insert")) {
+                insertRules = value.get("insert");
+                tagName = entry.getKey();
+                break;
+            }
+        }
+        if (StringUtils.isBlank(insertRules) || StringUtils.isBlank(tagName)) {
+            return;
+        }
+        final String finalTagName = tagName;
+
+        // 判断是否前几个已经有该tag的视频了
+        boolean isExistResultTagVideo = rovList.stream()
+                .limit(param.getSize())
+                .anyMatch(video -> video.getTags().contains(finalTagName));
+        if (isExistResultTagVideo) {
+            return;
+        }
+
+        // 获取需要插入的视频
+        Video insertTagVideo = null;
+        int tagVideoIndex = -1;
+        for (int i = 0; i < rovList.size(); i++) {
+            if (rovList.get(i).getTags().contains(finalTagName)) {
+                insertTagVideo = rovList.get(i);
+                tagVideoIndex = i;
+                break;
+            }
+        }
+        if (insertTagVideo == null) {
+            return;
+        }
+
+        // 获取插入的权重
+        String[] insertWeight = insertRules.split(",");
+        List<Double> insertWeightList = Arrays.stream(insertWeight)
+                .map(Double::valueOf).collect(Collectors.toList());
+
+        if (CollectionUtils.size(insertWeightList) != 5) {
+            return;
+        }
+        Double showProbability = insertWeightList.get(0);
+        double randomDouble = MathUtil.nextDouble(0, 1);
+        if (randomDouble > showProbability) {
+            // 没有命中强插规则
+            return;
+        }
+        // 遍历权重，获取插入位置
+        List<WeightRandom.ItemWithWeight<Integer>> items = new ArrayList<>(insertWeightList.size());
+        for (int i = 1; i < insertWeightList.size(); i++) {
+            WeightRandom.ItemWithWeight<Integer> itemWithWeight = new WeightRandom.ItemWithWeight<>(i, insertWeightList.get(i));
+            items.add(itemWithWeight);
+        }
+        WeightRandom<Integer> integerWeightRandom = new WeightRandom<>(items);
+        Integer insertIndex = integerWeightRandom.choose();
+        if (insertIndex == null) {
+            return;
+        }
+        // 删除原视频
+        rovList.remove(tagVideoIndex);
+        // 插入该视频到指定位置
+        rovList.add(insertIndex - 1, insertTagVideo);
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/processor/RankProcessorTagFilter.java

@@ -1,46 +1,60 @@
 package com.tzld.piaoquan.recommend.server.service.rank.processor;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import org.apache.commons.lang3.tuple.Pair;
 
 import java.text.SimpleDateFormat;
 import java.util.*;
-
 public class RankProcessorTagFilter {
 
-    public static void tagFitlter(List<Video> rov, List<Video> flow, Map<String, Map<String, String>> rule) {
+    public static void processor(List<Video> rov, List<Video> flow, Map<String, Map<String, String>> rule) {
 
-        Map<String, List<String>> tag2Dates = new HashMap<>();
-//        Map<String, List<String>> tag2Apps = new HashMap<>();
-        Map<String, Integer> tag2Hour = new HashMap<>();
+        Map<String, Double> tag2Rate = new HashMap<>();
+        Map<String, List<Pair<String, String>>> tag2Dates = new HashMap<>();
         for (Map.Entry<String, Map<String, String>> entry : rule.entrySet()){
             String key = entry.getKey();
             Map<String, String> value = entry.getValue();
-            if (value.containsKey("date")){
-                tag2Dates.put(key, Arrays.asList(value.get("date").split(",")));
+            if (value.containsKey("filter_rate")){
+                tag2Rate.put(key, Double.valueOf(value.get("filter_rate")));
+            }
+            if (value.containsKey("filter_date")){
+                List<Pair<String, String>> tmpList = new ArrayList<>();
+                for (String tmp : value.get("filter_date").split(",")){
+                    String start = tmp.split("-")[0];
+                    String end = tmp.split("-")[1];
+                    tmpList.add(Pair.of(start, end));
+                }
+                tag2Dates.put(key, tmpList);
             }
-//            if (value.containsKey("app")){
-//                tag2Apps.put(key, Arrays.asList(value.get("app").split(",")));
-//            }
-            if (value.containsKey("hour")){
-                tag2Hour.put(key, Integer.valueOf(value.get("hour")));
+        }
+
+        // 通过过滤概率获取本轮过滤tag集合
+        Set<String> filterTags = new HashSet<>();
+        for (Map.Entry<String, Double> entry : tag2Rate.entrySet()) {
+            String key = entry.getKey();
+            Double value = entry.getValue();
+            if (Math.random() <= value){
+                filterTags.add(key);
             }
         }
 
+        // 获取系统时间
         Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        Integer hour = Integer.valueOf(new SimpleDateFormat("HH").format(calendar.getTime()));
+        String dateHour = new SimpleDateFormat("yyyyMMddHH").format(calendar.getTime());
 
+        //执行过滤
         Iterator<Video> iterator = rov.iterator();
         while (iterator.hasNext()) {
             Video video = iterator.next();
             List<String> tags = video.getTags();
             boolean filter = false;
             for (String tag : tags){
-                if (tag2Dates.containsKey(tag) && !tag2Dates.get(tag).isEmpty() && !tag2Dates.get(tag).contains(date)){
-                    // 如果有日期存在，但日期没有命中，则不过滤，跳过即可。
+                if (!filterTags.contains(tag)){
                     continue;
                 }
-                if (tag2Hour.containsKey(tag) && hour >= tag2Hour.get(tag)){
+                boolean flag = ifFiter(tag2Dates, tag, dateHour);
+                if (flag){
                     filter = true;
+                    break;
                 }
             }
             if (filter){
@@ -54,12 +68,13 @@ public class RankProcessorTagFilter {
             List<String> tags = video.getTags();
             boolean filter = false;
             for (String tag : tags){
-                if (tag2Dates.containsKey(tag) && !tag2Dates.get(tag).isEmpty() && !tag2Dates.get(tag).contains(date)){
-                    // 如果有日期存在，但日期没有命中，则不过滤，跳过即可。
+                if (!filterTags.contains(tag)){
                     continue;
                 }
-                if (tag2Hour.containsKey(tag) && hour >= tag2Hour.get(tag)){
+                boolean flag = ifFiter(tag2Dates, tag, dateHour);
+                if (flag){
                     filter = true;
+                    break;
                 }
             }
             if (filter){
@@ -68,5 +83,16 @@ public class RankProcessorTagFilter {
         }
     }
 
+    public static boolean ifFiter(Map<String, List<Pair<String, String>>> tag2Dates, String tag, String dateHour){
+        if (!tag2Dates.containsKey(tag) || tag2Dates.get(tag).isEmpty()){
+            return false;
+        }
+        for (Pair<String, String> d: tag2Dates.get(tag)){
+            if (dateHour.compareTo(d.getLeft()) >= 0 && dateHour.compareTo(d.getRight()) < 0){
+                return true;
+            }
+        }
+        return false;
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/FestivalStrategy4RankModel.java

@@ -0,0 +1,75 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author sunxy
+ */
+@Service
+@Slf4j
+public class FestivalStrategy4RankModel extends RankService {
+
+    @Resource
+    private RankStrategy4Density rankStrategy4Density;
+
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        List<Video> rovRecallRank = new ArrayList<>();
+        rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        List<Video> festivalRecallVideoList = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        // 截断
+        if (CollectionUtils.isNotEmpty(festivalRecallVideoList)) {
+            rovRecallRank.addAll(festivalRecallVideoList.stream().limit(30).collect(Collectors.toList()));
+        }
+        // 补充不分地域小时数据
+        rovRecallRank.addAll(extractAndSort(param, RegionHWithoutDupRecallStrategy.PUSH_FORM));
+        // merge sim recall 和 return recall
+        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
+        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
+        // 地域召回要做截取，再做融合排序
+        removeDuplicate(rovRecallRank);
+
+        // 融合排序
+        List<String> videoIdKeys = rovRecallRank.stream()
+                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+                .collect(Collectors.toList());
+        List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
+//            log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                    JSONUtils.toJson(videoScores));
+        if (CollectionUtils.isNotEmpty(videoScores)
+                && videoScores.size() == rovRecallRank.size()) {
+            for (int i = 0; i < videoScores.size(); i++) {
+                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+            }
+            Collections.sort(rovRecallRank,
+                    Comparator.comparingDouble(o -> -(o.getSortScore() * 0.1 * (o.getRovScore() / 100))));
+        }
+
+        return rovRecallRank;
+    }
+
+    @Override
+    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank,
+                                   List<Video> flowPoolRank) {
+        return rankStrategy4Density.mergeAndSort(param, rovRecallRank, flowPoolRank);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java

@@ -1,17 +1,17 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 
-import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
+import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
@@ -21,132 +21,107 @@ import java.util.*;
 import java.util.stream.Collectors;
 
 /**
- * @author zhangbo
- * @desc 带密度控制的后处理 排序实验
+ * @author zhangbo sunxiaoyi
+ * @desc 后处理规则 + roc池flow池的合并
+ * 后处理参考文档：https://w42nne6hzg.feishu.cn/wiki/MYaGwCnF1iTFXUkSddAcA6CanFe
+ * roc池flow池的合并 文档： 暂无
  */
 @Service
 @Slf4j
 public class RankStrategy4Density extends RankService {
     @ApolloJsonValue("${RankStrategy4DensityFilter:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules;
+    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank) {
+    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
 
-        if (CollectionUtils.isEmpty(rovRecallRank)) {
-            if (param.getSize() < flowPoolRank.size()) {
-                return new RankResult(flowPoolRank.subList(0, param.getSize()));
+         //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
+        if (CollectionUtils.isEmpty(rovVideos)) {
+            if (param.getSize() < flowVideos.size()) {
+                return new RankResult(flowVideos.subList(0, param.getSize()));
             } else {
-                return new RankResult(flowPoolRank);
-            }
-        }
-        // 1 读取多样性密度控制规则------------------
-//        String appType = String.valueOf(param.getAppType());
-//        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
-//        Map<String, Integer> densityRules = new HashMap<>();
-//        if (ruleStr != null){
-//            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
-//                    new TypeToken<Map<String, Map<String, Object>>>() {},
-//                    Collections.emptyMap());
-//            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
-//                String k = entry.getKey();
-//                if (!entry.getValue().containsKey(appType)){
-//                    continue;
-//                }
-//                JSONObject jb = (JSONObject) entry.getValue().get(appType);
-//                try{
-//                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
-//                        densityRules.put(k, jb.getInteger("density"));
-//                    }
-//                }catch (Exception e){
-//                    log.error("parse densityRules is wrong:", e);
-//                }
-//            }
-//        }
-        Map<String, Integer> densityRules = new HashMap<>();
-        String abCode = param.getAbCode();
-        if (this.filterRules != null && this.filterRules.containsKey(abCode)){
-            Map<String, Map<String, String>> rule = this.filterRules.get(abCode);
-            for (Map.Entry<String, Map<String, String>> entry : rule.entrySet()){
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")){
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
+                return new RankResult(flowVideos);
             }
         }
 
+        //2 根据实验号解析阿波罗参数。
+        String abCode = param.getAbCode();
+        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
 
-        // 2 读取video的tags------------------
-        List<Long> videoIds = new ArrayList<>();
-        for (Video v : rovRecallRank) {
-            videoIds.add(v.getVideoId());
-        }
-        for (Video v : flowPoolRank) {
-            videoIds.add(v.getVideoId());
-        }
-        Map<Long, List<String>> videoTagDict = RankExtractorFeature.getVideoTags(this.redisTemplate, videoIds);
-        for (Video v : rovRecallRank) {
-            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        //3 标签读取
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
+            extractorItemTags.processor(rovVideos, flowVideos);
         }
-        for (Video v : flowPoolRank) {
-            v.setTags(videoTagDict.getOrDefault(v.getVideoId(), new ArrayList<>()));
+        //6 合并结果时间卡控
+        if (rulesMap != null && !rulesMap.isEmpty()){
+            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
         }
 
-        // 3 读取过滤规则，根据tag和过滤规则进行过滤------------------
-        if (this.filterRules != null && this.filterRules.containsKey(abCode)){
-            Map<String, Map<String, String>> rule = this.filterRules.get(abCode);
-            RankProcessorTagFilter.tagFitlter(rovRecallRank, flowPoolRank, rule);
-        }
+        //4 rov池提权功能
+        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+
+        //5 rov池强插功能
+        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
 
-        // 4 流量池按比例强插---------------------
+        //7 流量池按比例强插
         List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovRecallRank.size(); i++) {
-            result.add(rovRecallRank.get(i));
+        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
+            result.add(rovVideos.get(i));
         }
-
         double flowPoolP = getFlowPoolP(param);
         int flowPoolIndex = 0;
         int rovPoolIndex = param.getTopK();
-
         for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
             double rand = RandomUtils.nextDouble(0, 1);
             log.info("rand={}, flowPoolP={}", rand, flowPoolP);
             if (rand < flowPoolP) {
-                if (flowPoolIndex < flowPoolRank.size()) {
-                    result.add(flowPoolRank.get(flowPoolIndex++));
+                if (flowPoolIndex < flowVideos.size()) {
+                    result.add(flowVideos.get(flowPoolIndex++));
                 } else {
                     break;
                 }
             } else {
-                if (rovPoolIndex < rovRecallRank.size()) {
-                    result.add(rovRecallRank.get(rovPoolIndex++));
+                if (rovPoolIndex < rovVideos.size()) {
+                    result.add(rovVideos.get(rovPoolIndex++));
                 } else {
                     break;
                 }
             }
         }
-        if (rovPoolIndex >= rovRecallRank.size()) {
-            for (int i = flowPoolIndex; i < flowPoolRank.size() && result.size() < param.getSize(); i++) {
-                result.add(flowPoolRank.get(i));
+        if (rovPoolIndex >= rovVideos.size()) {
+            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(flowVideos.get(i));
             }
         }
-        if (flowPoolIndex >= flowPoolRank.size()) {
-            for (int i = rovPoolIndex; i < rovRecallRank.size() && result.size() < param.getSize(); i++) {
-                result.add(rovRecallRank.get(i));
+        if (flowPoolIndex >= flowVideos.size()) {
+            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
+                result.add(rovVideos.get(i));
             }
         }
 
-        // 3 进行密度控制------------------
-        Set<Long> videosSet = result.stream().map(r-> r.getVideoId()).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovRecallRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowPoolRank.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDnsity = RankProcessorDensity.mergeDensityControl(result,
+        //8 合并结果密度控制
+        Map<String, Integer> densityRules = new HashMap<>();
+        if (rulesMap != null && !rulesMap.isEmpty()) {
+            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, String> value = entry.getValue();
+                if (value.containsKey("density")) {
+                    densityRules.put(key, Integer.valueOf(value.get("density")));
+                }
+            }
+        }
+        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
+        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
+        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
                 rovRecallRankNew, flowPoolRankNew, densityRules);
-//        log.info("zhangbo22 old={}, new={}", JSONUtils.toJson(result),
-//                    JSONUtils.toJson(resultWithDnsity));
-        return new RankResult(resultWithDnsity);
+
+        return new RankResult(resultWithDensity);
     }
 
+}
+
+
 
 
 //    public Video getTestVideo(Long id, String s){
@@ -156,6 +131,26 @@ public class RankStrategy4Density extends RankService {
 //        return a1;
 //    }
 
-
-
-}
+// 1 读取多样性密度控制规则------------------
+//        String appType = String.valueOf(param.getAppType());
+//        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
+//        Map<String, Integer> densityRules = new HashMap<>();
+//        if (ruleStr != null){
+//            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
+//                    new TypeToken<Map<String, Map<String, Object>>>() {},
+//                    Collections.emptyMap());
+//            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
+//                String k = entry.getKey();
+//                if (!entry.getValue().containsKey(appType)){
+//                    continue;
+//                }
+//                JSONObject jb = (JSONObject) entry.getValue().get(appType);
+//                try{
+//                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
+//                        densityRules.put(k, jb.getInteger("density"));
+//                    }
+//                }catch (Exception e){
+//                    log.error("parse densityRules is wrong:", e);
+//                }
+//            }
+//        }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV1.java

@@ -0,0 +1,441 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV1 extends RankService {
+//    @ApolloJsonValue("${video.model.weightv3:}")
+//    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0);
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV2.java

@@ -0,0 +1,440 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV2 extends RankService {
+//    @ApolloJsonValue("${video.model.weightv3:}")
+//    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
+                    Math.log(1 + item.scoresMap.getOrDefault("allreturnsScore", 0.0));
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV3.java

@@ -0,0 +1,538 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV3 extends RankService {
+    @ApolloJsonValue("${rank.score.merge.weight:}")
+    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
+        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
+        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
+            return quickFlowPoolVideos;
+        } else {
+            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
+        }
+    }
+    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
+
+        //初始化 userid
+        UserFeature userFeature = new UserFeature();
+        userFeature.setMid(param.getMid());
+
+        // 初始化RankItem
+        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
+                .filter(d -> d.getPushFrom().equals(pushFrom))
+                .findFirst();
+        List<Video> videoList = data.get().getVideos();
+        if (videoList == null) {
+            return Collections.emptyList();
+        }
+        List<RankItem> rankItems = new ArrayList<>();
+        for (int i = 0; i < videoList.size(); i++) {
+            RankItem rankItem = new RankItem(videoList.get(i));
+            rankItems.add(rankItem);
+        }
+
+        // 初始化上下文参数
+        ScoreParam scoreParam = convert(param);
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
+                .scoring(scoreParam, userFeature, rankItems);
+
+        if (rovRecallScore == null) {
+            return Collections.emptyList();
+        }
+
+        return CommonCollectionUtils.toList(rovRecallScore, i -> {
+            // hard code 将排序分数 赋值给video的sortScore
+            Video v = i.getVideo();
+            v.setSortScore(i.getScore());
+            return v;
+        });
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(10, v7.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
+        double beta = this.mergeWeight.getOrDefault("beta", 1.0);
+        for (RankItem item : items){
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
+                    + alpha * trendScore
+                    + beta * newVideoScore
+                    ;
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 8){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 5.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                                List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV4.java

@@ -0,0 +1,465 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV4 extends RankService {
+    @ApolloJsonValue("${rank.score.merge.weight:}")
+    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
+        for (RankItem item : items){
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
+                    + alpha * trendScore
+                    ;
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+    public double calTrendScore(List<Double> data){
+        int cnt = 0;
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+            cnt ++;
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        return sum;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV5.java

@@ -0,0 +1,484 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV5 extends RankService {
+    @ApolloJsonValue("${rank.score.merge.weight:}")
+    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
+        double beta = this.mergeWeight.getOrDefault("beta", 1.0);
+        for (RankItem item : items){
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
+                    + alpha * trendScore
+                    + beta * newVideoScore
+                    ;
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 8){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 5.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                             List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV6.java

@@ -0,0 +1,486 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.springframework.stereotype.Service;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+/**
+ * @author zhangbo
+ * @desc 地域召回融合
+ */
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV6 extends RankService {
+    @ApolloJsonValue("${rank.score.merge.weight:}")
+    private Map<String, Double> mergeWeight;
+    final private String CLASS_NAME = this.getClass().getSimpleName();
+    public void duplicate(Set<Long> setVideo, List<Video> videos){
+        Iterator<Video> iterator = videos.iterator();
+        while(iterator.hasNext()){
+            Video v = iterator.next();
+            if (setVideo.contains(v.getVideoId())){
+                iterator.remove();
+            }else{
+                setVideo.add(v.getVideoId());
+            }
+        }
+    }
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        //-------------------地域内部融合+去重复-------------------
+        List<Video> rovRecallRank = new ArrayList<>();
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v1);
+        this.duplicate(setVideo, v2);
+        this.duplicate(setVideo, v3);
+        this.duplicate(setVideo, v4);
+        //-------------------地域 sim returnv2 融合+去重复-------------------
+        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
+        this.duplicate(setVideo, v7);
+
+//        rovRecallRank.addAll(v1);
+//        rovRecallRank.addAll(v2);
+//        rovRecallRank.addAll(v3);
+//        rovRecallRank.addAll(v4);
+//        rovRecallRank.addAll(v5);
+//        rovRecallRank.addAll(v6);
+
+        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
+        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
+        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
+        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
+        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(10, v7.size())));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+//        List<String> videoIdKeys = rovRecallRank.stream()
+//                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
+//                .collect(Collectors.toList());
+//        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
+//        log.info("rank mergeAndRankRovRecall videoIdKeys={}, videoScores={}", JSONUtils.toJson(videoIdKeys),
+//                JSONUtils.toJson(videoScores));
+//        if (CollectionUtils.isNotEmpty(videoScores)
+//                && videoScores.size() == rovRecallRank.size()) {
+//            for (int i = 0; i < videoScores.size(); i++) {
+//                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
+//            }
+//            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
+//        }
+
+        // 1 模型分
+        List<String> rtFeaPart = new ArrayList<>();
+        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+
+            // rov的趋势
+            double trendScore = calTrendScore(view2return);
+            item.scoresMap.put("trendScore", trendScore);
+
+            // 新视频提取
+            double newVideoScore = calNewVideoScore(itemBasicMap);
+            item.scoresMap.put("newVideoScore", newVideoScore);
+
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
+        double beta = this.mergeWeight.getOrDefault("beta", 1.0);
+        for (RankItem item : items){
+            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
+            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
+                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
+                    + alpha * trendScore
+                    + beta * newVideoScore
+                    ;
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+    public double calNewVideoScore(Map<String, String> itemBasicMap){
+        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
+        if (existenceDays > 8){
+            return 0.0;
+        }
+        double score = 1.0 / (existenceDays + 5.0);
+        return score;
+    }
+    public double calTrendScore(List<Double> data){
+        double sum = 0.0;
+        int size = data.size();
+        for (int i=0; i<size-4; ++i){
+            sum += data.get(i) - data.get(i+4);
+        }
+        if (sum * 10 > 0.6){
+            sum = 0.6;
+        }else{
+            sum = sum * 10;
+        }
+        if (sum > 0){
+            // 为了打断点
+            sum = sum;
+        }
+        return sum;
+    }
+
+    public Double calScoreWeight(List<Double> data){
+        Double up = 0.0;
+        Double down = 0.0;
+        for (int i=0; i<data.size(); ++i){
+            up += 1.0 / (i + 1) * data.get(i);
+            down += 1.0 / (i + 1);
+        }
+        return down > 1E-8? up / down: 0.0;
+    }
+    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
+        List<Double> data = new LinkedList<>();
+        for(int i=0; i<ups.size(); ++i){
+            data.add(
+                    (ups.get(i) + up) / (downs.get(i) + down)
+            );
+        }
+        return data;
+    }
+    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
+                                      List<String> datehours, String key){
+        List<Double> views = new LinkedList<>();
+        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
+        for (String dh : datehours){
+            views.add(tmp.getOrDefault(dh, 0.0D) +
+                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
+            );
+        }
+        return views;
+    }
+
+    public List<RankItem> model(List<Video> videos, RankParam param,
+                                List<String> rtFeaPart){
+        List<RankItem> result = new ArrayList<>();
+        if (videos.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.CLASS_NAME);
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
+        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<String>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.CLASS_NAME);
+        log.info(obj.toString());
+        return rovRecallScore;
+    }
+
+    private Map<String, String> getSceneFeature(RankParam param) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        String provinceCn = param.getProvince();
+        provinceCn = provinceCn.replaceAll("省$", "");
+        sceneFeatureMap.put("ctx_region", provinceCn);
+        String city = param.getCity();
+        if ("台北市".equals(city) |
+                "高雄市".equals(city) |
+                "台中市".equals(city) |
+                "桃园市".equals(city) |
+                "新北市".equals(city) |
+                "台南市".equals(city) |
+                "基隆市".equals(city) |
+                "吉林市".equals(city) |
+                "新竹市".equals(city) |
+                "嘉义市".equals(city)
+        ){
+            ;
+        }else{
+            city = city.replaceAll("市$", "");
+        }
+        sceneFeatureMap.put("ctx_city", city);
+
+        Calendar calendar = Calendar.getInstance();
+        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
+        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
+
+        return sceneFeatureMap;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -86,9 +86,20 @@ public class RecallService implements ApplicationContextAware {
         if (param.getAppType() == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
                 || param.getAppType() == AppTypeEnum.ZUI_JING_QI.getCode()) {
             strategies.addAll(getRegionRecallStrategy(param));
-        } else {
+        } else if (param.getAppType() == AppTypeEnum.PIAO_QUAN_MEIHAO_ZHUFU.getCode()){
+            strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
+            strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
+            strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
+            return strategies;
+        }else {
             switch (abCode) {
                 case "60120": // 576
+                case "60121": // 536
+                case "60122": // 537
+                case "60123": // 541
+                case "60124": // 546
+                case "60125": // 547
+                case "60126": // 548
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
@@ -143,8 +154,18 @@ public class RecallService implements ApplicationContextAware {
                 case "60103": // 增加地域1小时扩量，通过配置实现
                 case "60105": // 通过更改param中的配置实现使用不同数据源 data66 rule68 + 有排序模块
                 case "60120": // 576
+                case "60121": // 536
+                case "60122": // 537
+                case "60124": // 546
+                case "60125": // 547
+                    strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                    break;
+                case "60123": // 541
+                case "60126": // 548
                     strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
                     break;
                 case "60104": // 去掉sim的对比实验
                     strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
@@ -155,6 +176,11 @@ public class RecallService implements ApplicationContextAware {
                     strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
                     strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                     break;
+                case "60130":
+                    strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
+                    break;
                 default:
                     // todo 做兜底吗？
                     break;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FestivalRecallStrategyV1.java

@@ -0,0 +1,80 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+/**
+ * @author zhangbo
+ */
+@Slf4j
+@Component
+public class FestivalRecallStrategyV1 implements RecallStrategy {
+    public static final String PUSH_FORM = "recall_strategy_festival";
+    @Autowired
+    private RegionFilterService filterService;
+    @Override
+    public List<Video> recall(RecallParam param) {
+
+        Map<String, String> param4Model = new HashMap<>(1);
+        // 2 通过model拿到召回list
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_score_config_festival.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
+        List<Pair<Long, Double>> result = results.get(0);
+        for (int i=1; i<results.size(); ++i){
+            result.addAll(results.get(i));
+        }
+        Map<Long, Double> videoMap = new LinkedHashMap<>();
+        for (Pair<Long, Double> v: result){
+            videoMap.put(v.getLeft(), v.getRight());
+        }
+        long t1 = new Long(System.currentTimeMillis());
+        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        long t2 = new Long(System.currentTimeMillis());
+        JSONObject obj = new JSONObject();
+        obj.put("name", "FestivalRecallStrategyV1");
+        obj.put("filter_time", t2-t1);
+        obj.put("sizeOld", videoMap.size());
+        List<Video> videosResult = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            obj.put("sizeNew", filterResult.getVideoIds().size());
+            filterResult.getVideoIds().stream().forEach(vid -> {
+                Video video = new Video();
+                video.setVideoId(vid);
+                video.setAbCode(param.getAbCode());
+                video.setRovScore(videoMap.get(vid));
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            });
+        }
+        log.info(obj.toString());
+        Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
+        return videosResult;
+    }
+
+    @Override
+    public String pushFrom(){
+        return PUSH_FORM;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -37,6 +37,7 @@ public final class ScorerUtils {
         ScorerUtils.init4Recall("feeds_recall_config_region_v2.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v3.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v4.conf");
+        ScorerUtils.init4Recall("feeds_score_config_festival.conf");
     }
 
     private ScorerUtils() {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/DynamicGaussianFunction.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+
+/**
+ * 动态高斯函数
+ *
+ * @author sunxy
+ */
+public class DynamicGaussianFunction {
+    private static final double SIGMA = 24.0; // 标准差，可以根据需要调整这个值
+
+    public static double calculateValue(LocalDateTime testTime, LocalDateTime startTime, LocalDateTime endTime,
+                                        LocalDateTime peakTime) {
+        long testTimestamp = testTime.toEpochSecond(ZoneOffset.UTC) * 1000;
+        // 检查时间是否在指定范围内
+        if (testTimestamp < startTime.toEpochSecond(ZoneOffset.UTC) * 1000 ||
+                testTimestamp > endTime.toEpochSecond(ZoneOffset.UTC) * 1000) {
+            return 0;
+        }
+
+        // 计算 t_0，即从开始时间到峰值时间的小时数
+        double t_0 = (peakTime.toEpochSecond(ZoneOffset.UTC) - startTime.toEpochSecond(ZoneOffset.UTC)) / 3600.0;
+
+        // 计算时间差（以小时为单位）
+        double t = (testTimestamp / 1000.0 - startTime.toEpochSecond(ZoneOffset.UTC)) / 3600.0;
+
+        // 计算高斯函数值
+        return 100 * Math.exp(-Math.pow(t - t_0, 2) / (2 * Math.pow(SIGMA, 2)));
+    }
+
+    public static void main(String[] args) {
+        // 示例：计算2024-02-03 09:00的值
+        LocalDateTime startTime = LocalDateTime.of(2024, 1, 29, 0, 0);
+        LocalDateTime endTime = LocalDateTime.of(2024, 2, 3, 23, 59);
+        LocalDateTime peakTime = LocalDateTime.of(2024, 2, 3, 9, 0);
+        LocalDateTime testTime = LocalDateTime.of(2024, 1, 31, 22, 0);
+
+        double value = calculateValue(testTime, startTime, endTime, peakTime);
+        System.out.println("Function value at " + testTime + ": " + value);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/FestivalRecallScore.java

@@ -0,0 +1,290 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.score.ScorerConfigInfo;
+import com.tzld.piaoquan.recommend.server.service.score4recall.AbstractScorer4Recall;
+import com.tzld.piaoquan.recommend.server.service.score4recall.model4recall.Model4RecallKeyValue;
+import com.tzld.piaoquan.recommend.server.util.ListMerger;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.stream.Collectors;
+
+
+public class FestivalRecallScore extends AbstractScorer4Recall {
+
+    private static final Map<String, String> DAILY_BLESSING_TIME_MAP = new HashMap<String, String>() {
+        {
+            put("晚安", "daily 21:00-24:00");
+            put("晚上好", "daily 18:00-20:00");
+            put("下午好", "daily 15:00-16:00");
+            put("中午好 ", "daily 11:00-13:00");
+            put("早上好", "daily 00:00-08:00");
+        }
+    };
+
+    private static final Map<String, List<String>> YEARLY_FESTIVAL_TIME_MAP = new HashMap<String, List<String>>() {
+        {
+            put("圣诞节", Arrays.asList("2024-12-20 00:00~2024-12-25 08:00", "2025-12-20 00:00~2025-12-25 08:00", "2026-12-20 " +
+                    "00:00~2026-12-25 08:00"));
+            put("平安夜", Arrays.asList("2024-12-19 00:00~2024-12-24 08:00", "2025-12-19 00:00~2025-12-24 08:00", "2026-12-19 " +
+                    "00:00~2026-12-24 08:00"));
+            put("冬至", Arrays.asList("2024-12-19 00:00~2024-12-21 08:00", "2025-12-19 00:00~2025-12-21 08:00", "2026-12-20 " +
+                    "00:00~2026-12-22 08:00"));
+            put("公祭日", Arrays.asList("2024-12-08 00:00~2024-12-13 08:00", "2025-12-08 00:00~2025-12-13 08:00", "2026-12-08 " +
+                    "00:00~2026-12-13 08:00"));
+            put("大雪", Arrays.asList("2024-12-04 00:00~2024-12-06 08:00", "2025-12-05 00:00~2025-12-07 08:00", "2026-12-05 " +
+                    "00:00~2026-12-07 08:00"));
+            put("感恩节", Arrays.asList("2024-11-23 00:00~2024-11-28 08:00", "2025-11-22 00:00~2025-11-27 08:00", "2026-11-21 " +
+                    "00:00~2026-11-26 08:00"));
+            put("小雪", Arrays.asList("2024-11-20 00:00~2024-11-22 08:00", "2025-11-20 00:00~2025-11-22 08:00", "2026-11-20 " +
+                    "00:00~2026-11-22 08:00"));
+            put("立冬", Arrays.asList("2024-11-05 00:00~2024-11-07 08:00", "2025-11-05 00:00~2025-11-07 08:00", "2026-11-05 " +
+                    "00:00~2026-11-07 08:00"));
+            put("霜降", Arrays.asList("2024-10-21 00:00~2024-10-23 08:00", "2025-10-21 00:00~2025-10-23 08:00", "2026-10-21 " +
+                    "00:00~2026-10-23 08:00"));
+            put("重阳节", Arrays.asList("2024-10-06 00:00~2024-10-11 08:00", "2025-10-24 00:00~2025-10-29 08:00", "2026-10-13 " +
+                    "00:00~2026-10-18 08:00"));
+            put("寒露", Arrays.asList("2024-10-06 00:00~2024-10-08 08:00", "2025-10-06 00:00~2025-10-08 08:00", "2026-10-06 " +
+                    "00:00~2026-10-08 08:00"));
+            put("国庆节", Arrays.asList("2024-09-26 00:00~2024-10-01 08:00", "2025-09-26 00:00~2025-10-01 08:00", "2026-09-26 " +
+                    "00:00~2026-10-01 08:00"));
+            put("秋分", Arrays.asList("2024-09-20 00:00~2024-09-22 08:00", "2025-09-21 00:00~2025-09-23 08:00", "2026-09-21 " +
+                    "00:00~2026-09-23 08:00"));
+            put("中秋节", Arrays.asList("2024-09-12 00:00~2024-09-17 08:00", "2025-10-01 00:00~2025-10-06 08:00", "2026-09-20 " +
+                    "00:00~2026-09-25 08:00"));
+            put("白露", Arrays.asList("2024-09-05 00:00~2024-09-07 08:00", "2025-09-05 00:00~2025-09-07 08:00", "2026-09-05 " +
+                    "00:00~2026-09-07 08:00"));
+            put("处暑", Arrays.asList("2024-08-20 00:00~2024-08-22 08:00", "2025-08-21 00:00~2025-08-23 08:00", "2026-08-21 " +
+                    "00:00~2026-08-23 08:00"));
+            put("中元节", Arrays.asList("2024-08-13 00:00~2024-08-18 08:00", "2025-09-01 00:00~2025-09-06 08:00", "2026-08-22 " +
+                    "00:00~2026-08-27 08:00"));
+            put("七夕节", Arrays.asList("2024-08-05 00:00~2024-08-10 08:00", "2025-08-24 00:00~2025-08-29 08:00", "2026-08-14 " +
+                    "00:00~2026-08-19 08:00"));
+            put("立秋", Arrays.asList("2024-08-05 00:00~2024-08-07 08:00", "2025-08-05 00:00~2025-08-07 08:00", "2026-08-05 " +
+                    "00:00~2026-08-07 08:00"));
+            put("建军节", Arrays.asList("2024-07-27 00:00~2024-08-01 08:00", "2025-07-27 00:00~2025-08-01 08:00", "2026-07-27 " +
+                    "00:00~2026-08-01 08:00"));
+            put("大暑", Arrays.asList("2024-07-20 00:00~2024-07-22 08:00", "2025-07-20 00:00~2025-07-22 08:00", "2026-07-21 " +
+                    "00:00~2026-07-23 08:00"));
+            put("小暑", Arrays.asList("2024-07-04 00:00~2024-07-06 08:00", "2025-07-05 00:00~2025-07-07 08:00", "2026-07-05 " +
+                    "00:00~2026-07-07 08:00"));
+            put("七七事变",Arrays.asList("2024-07-02 00:00~2024-07-07 08:00", "2025-07-02 00:00~2025-07-07 08:00", "2026" +
+                    "-07-02 " +
+                    "00:00~2026-07-07 08:00"));
+            put("建党节", Arrays.asList("2024-06-26 00:00~2024-07-01 08:00", "2025-06-26 00:00~2025-07-01 08:00", "2026-06-26 " +
+                    "00:00~2026-07-01 08:00"));
+            put("夏至", Arrays.asList("2024-06-19 00:00~2024-06-21 08:00", "2025-06-19 00:00~2025-06-21 08:00", "2026-06-19 " +
+                    "00:00~2026-06-21 08:00"));
+            put("父亲节", Arrays.asList("2024-06-11 00:00~2024-06-16 08:00", "2025-06-10 00:00~2025-06-15 08:00", "2026-06-16 " +
+                    "00:00~2026-06-21 08:00"));
+            put("端午节", Arrays.asList("2024-06-05 00:00~2024-06-10 08:00", "2025-05-26 00:00~2025-05-31 08:00", "2026-06-14 " +
+                    "00:00~2026-06-19 08:00"));
+            put("芒种", Arrays.asList("2024-06-03 00:00~2024-06-05 08:00", "2025-06-03 00:00~2025-06-05 08:00", "2026-06-03 " +
+                    "00:00~2026-06-05 08:00"));
+            put("儿童节", Arrays.asList("2024-05-27 00:00~2024-06-01 08:00", "2025-05-27 00:00~2025-06-01 08:00", "2026-05-27 " +
+                    "00:00~2026-06-01 08:00"));
+            put("小满", Arrays.asList("2024-05-18 00:00~2024-05-20 08:00", "2025-05-19 00:00~2025-05-21 08:00", "2026-05-19 " +
+                    "00:00~2026-05-21 08:00"));
+            put("母亲节", Arrays.asList("2024-05-07 00:00~2024-05-12 08:00", "2025-05-06 00:00~2025-05-11 08:00", "2026-05-05 " +
+                    "00:00~2026-05-10 08:00"));
+            put("立夏", Arrays.asList("2024-05-03 00:00~2024-05-05 08:00", "2025-05-03 00:00~2025-05-05 08:00", "2026-05-03 " +
+                    "00:00~2026-05-05 08:00"));
+            put("劳动节", Arrays.asList("2024-04-26 00:00~2024-05-01 08:00", "2025-04-26 00:00~2025-05-01 08:00", "2026-04-26 " +
+                    "00:00~2026-05-01 08:00"));
+            put("谷雨", Arrays.asList("2024-04-17 00:00~2024-04-19 08:00", "2025-04-18 00:00~2025-04-20 08:00", "2026-04-18 " +
+                    "00:00~2026-04-20 08:00"));
+            put("清明", Arrays.asList("2024-04-02 00:00~2024-04-04 08:00", "2025-04-02 00:00~2025-04-04 08:00", "2026-04-03 " +
+                    "00:00~2026-04-05 08:00"));
+            put("春分", Arrays.asList("2024-03-18 00:00~2024-03-20 08:00", "2025-03-18 00:00~2025-03-20 08:00", "2026-03-18 " +
+                    "00:00~2026-03-20 08:00"));
+            put("龙抬头", Arrays.asList("2024-03-06 00:00~2024-03-11 08:00", "2025-02-24 00:00~2025-03-01 08:00", "2026-03-15 " +
+                    "00:00~2026-03-20 08:00"));
+            put("妇女节", Arrays.asList("2024-03-03 00:00~2024-03-08 08:00", "2025-03-03 00:00~2025-03-08 08:00", "2026-03-03 " +
+                    "00:00~2026-03-08 08:00"));
+            put("惊蛰", Arrays.asList("2024-03-03 00:00~2024-03-05 08:00", "2025-03-03 00:00~2025-03-05 08:00", "2026-03-03 " +
+                    "00:00~2026-03-05 08:00"));
+            put("元宵节", Arrays.asList("2024-02-19 00:00~2024-02-24 08:00", "2025-02-17 00:00~2025-02-22 08:00", "2026-02-26 " +
+                    "00:00~2026-03-03 08:00"));
+            put("雨水", Arrays.asList("2024-02-17 00:00~2024-02-19 08:00", "2025-02-16 00:00~2025-02-18 08:00", "2026-02-16 " +
+                    "00:00~2026-02-18 08:00"));
+            put("情人节", Arrays.asList("2024-02-09 00:00~2024-02-14 08:00", "2025-02-09 00:00~2025-02-14 08:00", "2026-02-09 " +
+                    "00:00~2026-02-14 08:00"));
+            put("春节", Arrays.asList("2024-02-05 00:00~2024-02-10 08:00", "2025-01-24 00:00~2025-01-29 08:00", "2026-02-12 " +
+                    "00:00~2026-02-17 08:00"));
+            put("除夕", Arrays.asList("2024-02-04 00:00~2024-02-09 08:00", "2025-01-23 00:00~2025-01-28 08:00", "2026-02-11 " +
+                    "00:00~2026-02-16 08:00"));
+            put("立春", Arrays.asList("2024-02-02 00:00~2024-02-04 08:00", "2025-02-01 00:00~2025-02-03 08:00", "2026-02-02 " +
+                    "00:00~2026-02-04 08:00"));
+            put("小年", Arrays.asList("2024-01-29 00:00~2024-02-03 20:00", "2025-01-18 00:00~2025-01-23 08:00", "2026-02-06 " +
+                    "00:00~2026-02-11 08:00"));
+            put("大寒", Arrays.asList("2024-01-18 00:00~2024-01-20 08:00", "2025-01-18 00:00~2025-01-20 08:00", "2026-01-18 " +
+                    "00:00~2026-01-20 08:00"));
+            put("腊八节", Arrays.asList("2024-01-13 00:00~2024-01-18 08:00", "2025-01-02 00:00~2025-01-07 08:00", "2026-01-21 " +
+                    "00:00~2026-01-26 08:00"));
+            put("小寒", Arrays.asList("2024-01-04 00:00~2024-01-06 08:00", "2025-01-03 00:00~2025-01-05 08:00", "2026-01-03 " +
+                    "00:00~2026-01-05 08:00"));
+            put("元旦", Arrays.asList("2023-12-27 00:00~2024-01-01 08:00", "2024-12-27 00:00~2025-01-01 08:00", "2025-12-27 " +
+                    "00:00~2026-01-01 08:00"));
+        }
+    };
+
+    public FestivalRecallScore(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(Model4RecallKeyValue.class);
+    }
+
+    @Override
+    public List<Pair<Long, Double>> recall(Map<String, String> params) {
+        // 节假日、时效性，判断
+        Model4RecallKeyValue model = (Model4RecallKeyValue) this.getModel();
+        if (model == null || model.kv == null) {
+            return new ArrayList<>();
+        }
+        LocalDateTime now = LocalDateTime.now();
+        // 节日祝福-每年
+        List<Pair<Long, Double>> yearResult = new ArrayList<>();
+        for (Map.Entry<String, List<String>> entry : YEARLY_FESTIVAL_TIME_MAP.entrySet()) {
+            String festival = entry.getKey();
+            List<String> timeRangeList = entry.getValue();
+            if (isFestivalTime(now, timeRangeList)) {
+                Pair<LocalDateTime, LocalDateTime> startTimeAndEndTime = getStartTimeAndEndTime(timeRangeList.get(0));
+                if (startTimeAndEndTime == null) {
+                    continue;
+                }
+                // 节日峰值设置为结束时间的当天的9点
+                double weight = DynamicGaussianFunction.calculateValue(LocalDateTime.now(), startTimeAndEndTime.getLeft(),
+                        startTimeAndEndTime.getRight(), startTimeAndEndTime.getRight().withHour(9));
+
+                List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault(festival, new ArrayList<>());
+                if (festivalLists.isEmpty()) {
+                    continue;
+                }
+                festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), weight))
+                        .limit(Math.min(50, festivalLists.size()))
+                        .collect(Collectors.toList());
+                yearResult.addAll(festivalLists);
+            }
+        }
+        List<Pair<Long, Double>> dayResult = new ArrayList<>();
+        // 每日祝福-每天固定时间段
+        for (Map.Entry<String, String> entry : DAILY_BLESSING_TIME_MAP.entrySet()) {
+            String festival = entry.getKey();
+            String timeRange = entry.getValue();
+            if (isFestivalTime(now, Collections.singletonList(timeRange))) {
+                List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault(festival, new ArrayList<>());
+                if (festivalLists.isEmpty()) {
+                    continue;
+                }
+                festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), 0.0))
+                        .limit(Math.min(50, festivalLists.size()))
+                        .collect(Collectors.toList());
+                dayResult.addAll(festivalLists);
+            }
+        }
+        // 常规祝福类的小程序-任意时间
+        List<Pair<Long, Double>> anyResult = new ArrayList<>();
+        List<Pair<Long, Double>> festivalLists = model.kv.getOrDefault("祝福", new ArrayList<>());
+        if (!festivalLists.isEmpty()) {
+            festivalLists = festivalLists.stream().map(pair -> Pair.of(pair.getLeft(), 0.0))
+                    .limit(Math.min(50, festivalLists.size()))
+                    .collect(Collectors.toList());
+            anyResult.addAll(festivalLists);
+        }
+        return ListMerger.mergeLists(yearResult, dayResult, anyResult);
+    }
+
+    public Pair<LocalDateTime, LocalDateTime> getStartTimeAndEndTime(String timeRangeList) {
+        if (timeRangeList == null || timeRangeList.isEmpty()) {
+            return null;
+        }
+        // 时间格式 2024-12-20 00:00~2024-12-25 08:00
+        if (StringUtils.startsWith(timeRangeList, "daily")) {
+            // 判断是否是 daily 开头
+            return null;
+        } else {
+            String[] split = StringUtils.split(timeRangeList, "~");
+            if (split.length != 2) {
+                return null;
+            }
+            String startTime = split[0];
+            String endTime = split[1];
+            // 解析 startTime endTime
+            DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
+            LocalDateTime startLocalDateTime = LocalDateTime.parse(startTime, dateTimeFormatter);
+            LocalDateTime endLocalDateTime = LocalDateTime.parse(endTime, dateTimeFormatter);
+            return Pair.of(startLocalDateTime, endLocalDateTime);
+        }
+    }
+
+    public boolean isFestivalTime(LocalDateTime now, List<String> timeRangeList) {
+        if (timeRangeList == null || timeRangeList.isEmpty()) {
+            return false;
+        }
+        for (String timeRange : timeRangeList) {
+            // 判断是否是 daily 开头
+            if (StringUtils.startsWith(timeRange, "daily")) {
+                // 判断是否是 daily 开头
+                String dailyTimeRange = StringUtils.substring(timeRange, 6);
+                String[] split = StringUtils.split(dailyTimeRange, "-");
+                if (split.length != 2) {
+                    continue;
+                }
+                String startTime = split[0];
+                String endTime = split[1];
+                // 获取当前时间的小时和分钟
+                int hour = now.getHour();
+                int minute = now.getMinute();
+                // startTime: 21:00 endTime: 23:00
+                String[] startSplit = StringUtils.split(startTime, ":");
+                String[] endSplit = StringUtils.split(endTime, ":");
+                if (startSplit.length != 2 || endSplit.length != 2) {
+                    continue;
+                }
+                int startHour = Integer.parseInt(startSplit[0]);
+                int startMinute = Integer.parseInt(startSplit[1]);
+                int endHour = Integer.parseInt(endSplit[0]);
+                int endMinute = Integer.parseInt(endSplit[1]);
+                if (hour > startHour && hour < endHour) {
+                    return true;
+                } else if (hour == startHour && hour == endHour) {
+                    if (minute >= startMinute && minute <= endMinute) {
+                        return true;
+                    }
+                } else if (hour == startHour) {
+                    if (minute >= startMinute) {
+                        return true;
+                    }
+                } else if (hour == endHour) {
+                    if (minute <= endMinute) {
+                        return true;
+                    }
+                }
+                continue;
+            }
+            // 时间格式 2024-12-20 00:00~2024-12-25 08:00
+            String[] split = StringUtils.split(timeRange, "~");
+            if (split.length != 2) {
+                continue;
+            }
+            String startTime = split[0];
+            String endTime = split[1];
+            // 解析 startTime endTime
+            DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
+            LocalDateTime startLocalDateTime = LocalDateTime.parse(startTime, dateTimeFormatter);
+            LocalDateTime endLocalDateTime = LocalDateTime.parse(endTime, dateTimeFormatter);
+            if (now.isAfter(startLocalDateTime) && now.isBefore(endLocalDateTime)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/ListMerger.java

@@ -0,0 +1,43 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 合并多个List
+ *
+ * @author sunxy
+ */
+public class ListMerger {
+
+    public static <T> List<T> mergeLists(List<T> list1, List<T> list2, List<T> list3) {
+        List<T> result = new ArrayList<>();
+        int maxSize = Math.max(list1.size(), Math.max(list2.size(), list3.size()));
+
+        for (int i = 0; i < maxSize; i++) {
+            if (i < list1.size()) {
+                result.add(list1.get(i));
+            }
+            if (i < list2.size()) {
+                result.add(list2.get(i));
+            }
+            if (i < list3.size()) {
+                result.add(list3.get(i));
+            }
+        }
+
+        return result;
+    }
+
+    public static void main(String[] args) {
+        // 示例
+        List<Integer> list1 = Lists.newArrayList(1, 4, 7);
+        List<Integer> list2 = Lists.newArrayList(2, 5, 8, 10, 11);
+        List<Integer> list3 = Lists.newArrayList(3, 6, 9);
+
+        List<Integer> mergedList = mergeLists(list1, list2, list3);
+        System.out.println(mergedList);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/MathUtil.java

@@ -0,0 +1,17 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import java.util.Random;
+
+/**
+ * @author sunxy
+ */
+public class MathUtil {
+
+    /**
+     * 生成max到min范围的浮点数
+     * */
+    public static double nextDouble(final double min, final double max) {
+        return min + ((max - min) * new Random().nextDouble());
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/WeightRandom.java

@@ -0,0 +1,119 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * 基于权重随机算法选择
+ *
+ * @author sunxy
+ */
+public class WeightRandom<T> {
+
+    private final List<T> items = new ArrayList<>();
+    private double[] weights;
+
+    public WeightRandom(List<ItemWithWeight<T>> itemsWithWeight) {
+        this.calWeights(itemsWithWeight);
+    }
+
+    /**
+     * 计算权重，初始化或者重新定义权重时使用
+     */
+    private void calWeights(List<ItemWithWeight<T>> itemsWithWeight) {
+        items.clear();
+
+        // 计算权重总和
+        double originWeightSum = 0;
+        for (ItemWithWeight<T> itemWithWeight : itemsWithWeight) {
+            double weight = itemWithWeight.getWeight();
+            if (weight <= 0) {
+                continue;
+            }
+
+            items.add(itemWithWeight.getItem());
+            if (Double.isInfinite(weight)) {
+                weight = 10000.0D;
+            }
+            if (Double.isNaN(weight)) {
+                weight = 1.0D;
+            }
+            originWeightSum += weight;
+        }
+
+        // 计算每个item的实际权重比例
+        double[] actualWeightRatios = new double[items.size()];
+        int index = 0;
+        for (ItemWithWeight<T> itemWithWeight : itemsWithWeight) {
+            double weight = itemWithWeight.getWeight();
+            if (weight <= 0) {
+                continue;
+            }
+            actualWeightRatios[index++] = weight / originWeightSum;
+        }
+
+        // 计算每个item的权重范围
+        // 权重范围起始位置
+        weights = new double[items.size()];
+        double weightRangeStartPos = 0;
+        for (int i = 0; i < index; i++) {
+            weights[i] = weightRangeStartPos + actualWeightRatios[i];
+            weightRangeStartPos += actualWeightRatios[i];
+        }
+    }
+
+    /**
+     * 基于权重随机算法选择
+     */
+    public T choose() {
+        if (items.isEmpty()) {
+            return null;
+        }
+        double random = ThreadLocalRandom.current().nextDouble();
+        int index = Arrays.binarySearch(weights, random);
+        if (index < 0) {
+            index = -index - 1;
+        } else {
+            return items.get(index);
+        }
+
+        if (index < weights.length && random < weights[index]) {
+            return items.get(index);
+        }
+
+        // 通常不会走到这里，为了保证能得到正确的返回，这里随便返回一个
+        return items.get(0);
+    }
+
+    public static class ItemWithWeight<T> {
+        T item;
+        double weight;
+
+        public ItemWithWeight() {
+        }
+
+        public ItemWithWeight(T item, double weight) {
+            this.item = item;
+            this.weight = weight;
+        }
+
+        public T getItem() {
+            return item;
+        }
+
+        public void setItem(T item) {
+            this.item = item;
+        }
+
+        public double getWeight() {
+            return weight;
+        }
+
+        public void setWeight(double weight) {
+            this.weight = weight;
+        }
+    }
+
+}

recommend-server-service/src/main/resources/feeds_score_config_festival.conf

@@ -0,0 +1,7 @@
+scorer-config = {
+  festival-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score4recall.strategy.FestivalRecallScore"
+    scorer-priority = 100
+    model-path = "alg_recall_file/05_festival.txt"
+  }
+}