Merge branch 'feature_refactor' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/ABTestRemoteService.java

@@ -1,36 +0,0 @@
-package com.tzld.piaoquan.recommend.server.remote;
-
-import com.tzld.piaoquan.abtest.client.ABTestClient;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.Map;
-
-
-/**
- * @author dyp
- */
-@Component
-@Slf4j
-public class ABTestRemoteService {
-
-    @Autowired
-    private ABTestClient client;
-
-//    public String getGroup(String mid, String uid, int appType, String group) {
-//        if (StringUtils.isBlank(mid)) {
-//            return null;
-//        }
-//        return client.getGroup(mid, uid, appType, group);
-//    }
-
-    public Map<String, String> getExp(String mid, String uid, int appType, String group) {
-        if (StringUtils.isBlank(mid)) {
-            return null;
-        }
-        return client.getExp(mid, uid, appType, group);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -6,15 +6,12 @@ import com.google.common.base.Strings;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.RedisKeyConstants;
-import com.tzld.piaoquan.recommend.server.common.base.Constant;
 import com.tzld.piaoquan.recommend.server.gen.common.Result;
 import com.tzld.piaoquan.recommend.server.gen.recommend.*;
 import com.tzld.piaoquan.recommend.server.model.MachineInfo;
 import com.tzld.piaoquan.recommend.server.model.RecommendParam;
 import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.remote.ABTestRemoteService;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConfigService;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolService;
@@ -27,7 +24,6 @@ import com.tzld.piaoquan.recommend.server.service.recall.RecallService;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import com.tzld.piaoquan.recommend.server.util.ParserUtils;
 import com.tzld.piaoquan.recommend.server.util.TraceUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -86,9 +82,6 @@ public class RecommendService {
     @Autowired
     private TimerLogService timerLogService;
 
-    @Autowired
-    private ABTestRemoteService abTestRemoteService;
-
     private ThreadLocal<Map<String, Object>> timerLogMapTL = ThreadLocal.withInitial(HashMap::new);
 
     @Value("${risk.video.filter.switch:true}")

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -24,14 +24,6 @@ public abstract class AbstractFilterService {
 
     private final ExecutorService pool = ThreadPoolFactory.filterPool();
 
-    private List<Long> truncation(List<Long> videoIds, int forceTruncation) {
-        if (forceTruncation == 0) {
-            return videoIds;
-        } else {
-            return videoIds.subList(0, Math.min(forceTruncation, videoIds.size()));
-        }
-    }
-
     protected List<Long> viewFilter(FilterParam param) {
 
         List<FilterStrategy> strategies = getStrategies(param);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -9,122 +9,41 @@ import org.springframework.stereotype.Service;
 @Service
 @Slf4j
 public class RankRouter {
-    @Autowired
-    private RankService rankService;
-    @Autowired
-    private RankStrategy4Rankv2Model rankStrategy4Rankv2Model;
-    @Autowired
-    private RankStrategy4RankModel rankStrategy4RankModel;
-    @Autowired
-    private RankStrategy4Density rankStrategy4Density;
     @Autowired
     private RankStrategy4RegionMergeModelV536 rankStrategy4RegionMergeModelV536;
     @Autowired
-    private RankStrategy4RegionMergeModelV546 rankStrategy4RegionMergeModelV546;
-    @Autowired
-    private RankStrategy4RegionMergeModelV547 rankStrategy4RegionMergeModelV547;
-    @Autowired
-    private RankStrategy4RegionMergeModelV548 rankStrategy4RegionMergeModelV548;
-    @Autowired
-    private RankStrategy4RegionMergeModelV551 rankStrategy4RegionMergeModelV551;
-    @Autowired
-    private RankStrategy4RegionMergeModelV552 rankStrategy4RegionMergeModelV552;
-    @Autowired
-    private RankStrategy4RegionMergeModelV553 rankStrategy4RegionMergeModelV553;
-    @Autowired
     private RankStrategy4RegionMergeModelV562 rankStrategy4RegionMergeModelV562;
     @Autowired
     private RankStrategy4RegionMergeModelV563 rankStrategy4RegionMergeModelV563;
     @Autowired
     private RankStrategy4RegionMergeModelV564 rankStrategy4RegionMergeModelV564;
     @Autowired
-    private RankStrategy4RegionMergeModelV565 rankStrategy4RegionMergeModelV565;
-    @Autowired
-    private RankStrategy4RegionMergeModelV566 rankStrategy4RegionMergeModelV566;
-    @Autowired
     private RankStrategy4RegionMergeModelV567 rankStrategy4RegionMergeModelV567;
     @Autowired
-    private RankStrategy4RegionMergeModelV999 rankStrategy4RegionMergeModelV999;
-    @Autowired
     private RankStrategy4RegionMergeModelV569 rankStrategy4RegionMergeModelV569;
-    @Autowired
-    private RankStrategy4RegionMergeModelV650 rankStrategy4RegionMergeModelV650;
-    @Autowired
-    private RankStrategy4RegionMergeModelV654 rankStrategy4RegionMergeModelV654;
-    @Autowired
-    private RankStrategy4RegionMergeModelV655 rankStrategy4RegionMergeModelV655;
-    @Autowired
-    private RankStrategy4RegionMergeModelV656 rankStrategy4RegionMergeModelV656;
-    @Autowired
-    private FestivalStrategy4RankModel festivalStrategy4RankModel;
-
-    @Autowired
-    private RankStrategyFlowThompsonModel rankStrategyFlowThompsonModel;
-    @Autowired
-    private RankStrategy4RegionMerge rankStrategy4RegionMerge;
-    @Autowired
-    private RankStrategy4ShareDeepAndWidth rankStrategy4ShareDeepAndWidth;
 
     public RankResult rank(RankParam param) {
         String abCode = param.getAbCode();
         if (StringUtils.isBlank(abCode)) {
-            return rankService.rank(param);
+            return rankStrategy4RegionMergeModelV536.rank(param);
         }
+        // 536 562 563 564 567 569
         switch (abCode) {
-            case "60105": // 551
-                return rankStrategy4RegionMergeModelV551.rank(param);
-            case "60106": // 552
-                return rankStrategy4RegionMergeModelV552.rank(param);
-            case "60107": // 553
-                return rankStrategy4RegionMergeModelV553.rank(param);
+
             case "60112": // 562
                 return rankStrategy4RegionMergeModelV562.rank(param);
-            case "60101":
-                return rankStrategy4RankModel.rank(param);
             case "60113": // 563
-                return rankStrategy4RegionMergeModelV563.rank(param);
+                return rankStrategy4RegionMergeModelV563.rank(param); // 60113 CF召回 doing
             case "60114": // 564
-                return rankStrategy4RegionMergeModelV564.rank(param);
-            case "60115": // 565
-                return rankStrategy4RegionMergeModelV565.rank(param);
-            case "60116": // 566
-                return rankStrategy4RegionMergeModelV566.rank(param);
+                return rankStrategy4RegionMergeModelV564.rank(param); // 60114 排序r-cnt实验 doing
             case "60117": // 567
-                return rankStrategy4RegionMergeModelV567.rank(param);
-            case "60118": // 568
-                return rankStrategy4RegionMergeModelV999.rank(param);
+                return rankStrategy4RegionMergeModelV567.rank(param); // 60117 cf叠加r-cnt模型 doing
             case "60119": // 569
-                return rankStrategy4RegionMergeModelV569.rank(param);
-            case "60120": // 576
-                return rankStrategy4RegionMerge.rank(param);
+                return rankStrategy4RegionMergeModelV569.rank(param); // rov fm模型 60119 实验关闭
             case "60121": // 536
-                return rankStrategy4RegionMergeModelV536.rank(param);
-            case "60122": // 537
-                return rankStrategy4RegionMergeModelV546.rank(param);
-            case "60124": // 546
-                return rankStrategy4RegionMergeModelV546.rank(param);
-            case "60125": // 547
-                return rankStrategy4RegionMergeModelV547.rank(param);
-            case "60126": // 548
-                return rankStrategy4RegionMergeModelV548.rank(param);
-            case "60130":
-            case "60131":
-            case "60132":
-                return festivalStrategy4RankModel.rank(param);
-            case "60150": // 645
-                return rankStrategy4ShareDeepAndWidth.rank(param);
-            case "60151": // 650
-                return rankStrategy4RegionMergeModelV650.rank(param);
-            case "60654": // 654
-                return rankStrategy4RegionMergeModelV654.rank(param);
-            case "60655": // 655
-                return rankStrategy4RegionMergeModelV655.rank(param);
-            case "60656": // 656
-                return rankStrategy4RegionMergeModelV656.rank(param);
-            default:
-                break;
+            default: // base 536
+                return rankStrategy4RegionMergeModelV536.rank(param); // vor统计量 doing
         }
-        return rankService.rank(param);
     }
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/FestivalStrategy4RankModel.java

@@ -1,83 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.Resource;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * @author sunxy
- */
-@Service
-@Slf4j
-public class FestivalStrategy4RankModel extends RankService {
-
-    @Resource
-    private RankStrategy4Density rankStrategy4Density;
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(rovRecallRank);
-        String abCode = param.getAbCode();
-        int sizeReturn = param.getSize();
-        if (abCode != null && this.regionRecallReturnSize != null &&
-                this.regionRecallReturnSize.containsKey(abCode) &&
-                this.regionRecallReturnSize.get(abCode) != null) {
-            sizeReturn = this.regionRecallReturnSize.get(abCode).getOrDefault("result_size", sizeReturn);
-        }
-        rovRecallRank = rovRecallRank.size() <= sizeReturn
-                ? rovRecallRank
-                : rovRecallRank.subList(0, sizeReturn);
-
-        List<Video> festivalRecallVideoList = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        // 截断
-        if (CollectionUtils.isNotEmpty(festivalRecallVideoList)) {
-            rovRecallRank.addAll(festivalRecallVideoList.stream().limit(30).collect(Collectors.toList()));
-        }
-        // merge sim recall 和 return recall
-        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
-        // 地域召回要做截取，再做融合排序
-        removeDuplicate(rovRecallRank);
-
-        // 融合排序
-        List<String> videoIdKeys = rovRecallRank.stream()
-                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-                .collect(Collectors.toList());
-        List<String> videoScores = redisTemplate.opsForValue().multiGet(videoIdKeys);
-        if (CollectionUtils.isNotEmpty(videoScores)
-                && videoScores.size() == rovRecallRank.size()) {
-            for (int i = 0; i < videoScores.size(); i++) {
-                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-            }
-            Collections.sort(rovRecallRank,
-                    Comparator.comparingDouble(o -> -(o.getSortScore() * 0.1 * (o.getRovScore() / 100))));
-        }
-
-        return rovRecallRank;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank,
-                                   List<Video> flowPoolRank) {
-        return rankStrategy4Density.mergeAndSort(param, rovRecallRank, flowPoolRank);
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Density.java

@@ -1,156 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo sunxiaoyi
- * @desc 后处理规则 + roc池flow池的合并
- * 后处理参考文档：https://w42nne6hzg.feishu.cn/wiki/MYaGwCnF1iTFXUkSddAcA6CanFe
- * roc池flow池的合并 文档： 暂无
- */
-@Service
-@Slf4j
-public class RankStrategy4Density extends RankService {
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
-
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-}
-
-
-
-
-//    public Video getTestVideo(Long id, String s){
-//        Video a1 = new Video();
-//        a1.setVideoId(id);
-//        a1.setFlowPool(s);
-//        return a1;
-//    }
-
-// 1 读取多样性密度控制规则------------------
-//        String appType = String.valueOf(param.getAppType());
-//        String ruleStr = this.redisTemplate.opsForValue().get("TAGS_FILTER_RULE_V1_JSON");
-//        Map<String, Integer> densityRules = new HashMap<>();
-//        if (ruleStr != null){
-//            Map<String, Map<String, Object>> ruleOrigin = JSONUtils.fromJson(ruleStr,
-//                    new TypeToken<Map<String, Map<String, Object>>>() {},
-//                    Collections.emptyMap());
-//            for (Map.Entry<String, Map<String, Object>> entry : ruleOrigin.entrySet()){
-//                String k = entry.getKey();
-//                if (!entry.getValue().containsKey(appType)){
-//                    continue;
-//                }
-//                JSONObject jb = (JSONObject) entry.getValue().get(appType);
-//                try{
-//                    if (jb.containsKey("density") && jb.get("density") instanceof Integer){
-//                        densityRules.put(k, jb.getInteger("density"));
-//                    }
-//                }catch (Exception e){
-//                    log.error("parse densityRules is wrong:", e);
-//                }
-//            }
-//        }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RankModel.java

@@ -1,370 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 模型的排序实验
- */
-@Service
-@Slf4j
-public class RankStrategy4RankModel extends RankService {
-
-    @ApolloJsonValue("${video.model.weightv1:}")
-    private Map<String, Double> mergeWeightNew;
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-
-        //-------------------地域内部融合-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-//        rovRecallRank.add(0, getTestVideo(1070462L, ""));
-//        rovRecallRank.add(0, getTestVideo(1085062L, ""));
-        rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-
-        //-------------------地域内部去重+截断-------------------
-        removeDuplicate(rovRecallRank);
-        rovRecallRank = rovRecallRank.size() <= param.getSize()
-                ? rovRecallRank
-                : rovRecallRank.subList(0, param.getSize());
-
-        //-------------------地域 sim returnv2 融合-------------------
-        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
-        //-------------------地域 sim returnv2 去重-------------------
-        removeDuplicate(rovRecallRank);
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-        List<String> videoIdKeys = rovRecallRank.stream()
-                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-                .collect(Collectors.toList());
-        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-        if (CollectionUtils.isNotEmpty(videoScores)
-                && videoScores.size() == rovRecallRank.size()) {
-            for (int i = 0; i < videoScores.size(); i++) {
-                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-            }
-            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-        }
-
-        //------------------- todo zhangbo 增加排序str ros模型逻辑 合并二者得分-------------------
-        List<Video> videosWithModel = model(rovRecallRank, param);
-        Map<String, Double> mergeWeight = this.mergeWeightNew == null ? new HashMap<>() : this.mergeWeightNew;
-        double alpha = mergeWeight.getOrDefault("alpha", 1.0D);
-        double beta = mergeWeight.getOrDefault("beta", 0.0D);
-        double gamma = mergeWeight.getOrDefault("gamma", 0.0D);
-        for (Video v : videosWithModel) {
-            double score = alpha * v.getSortScore() + beta * v.getScoreStr() + gamma * v.getScoreRos();
-            if (mergeWeight.containsKey("mul") && mergeWeight.getOrDefault("mul", 0.0D) > 0.5) {
-                score = alpha * v.getSortScore() + (beta + v.getScoreStr()) * (gamma + v.getScoreRos());
-            }
-            v.setScoreRegion(v.getSortScore());
-            v.score = score;
-            v.setSortScore(score);
-        }
-        videosWithModel.sort(Comparator.comparingDouble(o -> -o.score));
-
-        //------------------- 增加日志 -------------------
-        int size = 4;
-        List<Long> oldRes = rovRecallRank.subList(0, Math.min(rovRecallRank.size(), size)).stream().map(r -> r.getVideoId()).collect(Collectors.toList());
-        List<Long> newRes = videosWithModel.subList(0, Math.min(videosWithModel.size(), size)).stream().map(r -> r.getVideoId()).collect(Collectors.toList());
-        int diffpos = 0;
-        int difftop = 0;
-        for (int i = 0; i < newRes.size(); ++i) {
-            if (!oldRes.get(i).equals(newRes.get(i))) {
-                ++diffpos;
-            }
-            if (!oldRes.contains(newRes.get(i))) {
-                ++difftop;
-            }
-        }
-
-        return videosWithModel;
-    }
-
-    public List<Video> model(List<Video> videos, RankParam param) {
-        if (videos.isEmpty()) {
-            return videos;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            } else {
-                return videos;
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPart = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPart != null) {
-            if (rtFeaPart.get(0) != null) {
-                rtFeaPart1day = rtFeaPart.get(0);
-            }
-            if (rtFeaPart.get(1) != null) {
-                rtFeaPart1h = rtFeaPart.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setScoreStr(i.getScoreStr());
-            v.setScoreRos(i.getScoreRos());
-            return v;
-        });
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    public static void main(String[] args) {
-        Calendar calendar = Calendar.getInstance();
-        calendar.set(Calendar.YEAR, 2022);
-        calendar.set(Calendar.MONTH, 0); // January is 0
-        calendar.set(Calendar.DAY_OF_MONTH, 1);
-        calendar.set(Calendar.HOUR_OF_DAY, 0);
-        calendar.set(Calendar.MINUTE, 12);
-        calendar.set(Calendar.SECOND, 30);
-        System.out.println(new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        String provinceCn = "吉林省2";
-        provinceCn = provinceCn.replaceAll("省$", "");
-        System.out.println(provinceCn);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Rankv2Model.java

@@ -1,374 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 模型的排序实验
- */
-@Service
-@Slf4j
-public class RankStrategy4Rankv2Model extends RankService {
-
-    @ApolloJsonValue("${video.model.weightv2:}")
-    private Map<String, Double> mergeWeight;
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-//    public Video getTestVideo(Long id, String s){
-//        Video a1 = new Video();
-//        a1.setVideoId(id);
-//        a1.setFlowPool(s);
-//        a1.setPushFrom("recall_pool_region_h");
-//        return a1;
-//    }
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-
-        //-------------------地域内部融合-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-//        rovRecallRank.add(0, getTestVideo(1070462L, ""));
-//        rovRecallRank.add(0, getTestVideo(1085062L, ""));
-        rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-
-        //-------------------地域内部去重+截断-------------------
-        removeDuplicate(rovRecallRank);
-        rovRecallRank = rovRecallRank.size() <= param.getSize()
-                ? rovRecallRank
-                : rovRecallRank.subList(0, param.getSize());
-
-        //-------------------地域 sim returnv2 融合-------------------
-        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
-        //-------------------地域 sim returnv2 去重-------------------
-        removeDuplicate(rovRecallRank);
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-        List<String> videoIdKeys = rovRecallRank.stream()
-                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-                .collect(Collectors.toList());
-        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-        if (CollectionUtils.isNotEmpty(videoScores)
-                && videoScores.size() == rovRecallRank.size()) {
-            for (int i = 0; i < videoScores.size(); i++) {
-                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-            }
-            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-        }
-
-        //------------------- todo zhangbo 增加排序str ros模型逻辑 合并二者得分-------------------
-        List<Video> videosWithModel = model(rovRecallRank, param);
-        Map<String, Double> mergeWeight = this.mergeWeight == null? new HashMap<>(): this.mergeWeight;
-        double alpha = mergeWeight.getOrDefault("alpha", 1.0D);
-        double beta = mergeWeight.getOrDefault("beta", 0.0D);
-        double gamma = mergeWeight.getOrDefault("gamma", 0.0D);
-        for (Video v : videosWithModel){
-            double score = alpha * v.getSortScore() + beta * v.getScoreStr() + gamma * v.getScoreRos();
-            if (mergeWeight.containsKey("mul") && mergeWeight.getOrDefault("mul", 0.0D) > 0.5){
-                score = alpha * v.getSortScore() + (beta + v.getScoreStr()) * (gamma + v.getScoreRos());
-            }
-            v.setScoreRegion(v.getSortScore());
-            v.score = score;
-            v.setSortScore(score);
-        }
-        videosWithModel.sort(Comparator.comparingDouble(o -> -o.score));
-
-        //------------------- 增加日志 -------------------
-        int size = 4;
-        List<Long> oldRes = rovRecallRank.subList(0, Math.min(rovRecallRank.size(), size)).stream().map(r-> r.getVideoId()).collect(Collectors.toList());
-        List<Long> newRes = videosWithModel.subList(0, Math.min(videosWithModel.size(), size)).stream().map(r-> r.getVideoId()).collect(Collectors.toList());
-        int diffpos = 0;
-        int difftop = 0;
-        for (int i=0; i<newRes.size(); ++i){
-            if (!oldRes.get(i).equals(newRes.get(i))){
-                ++diffpos;
-            }
-            if (!oldRes.contains(newRes.get(i))){
-                ++difftop;
-            }
-        }
-
-        return videosWithModel;
-    }
-
-    public List<Video> model(List<Video> videos, RankParam param){
-        if (videos.isEmpty()){
-            return videos;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()){
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null){
-                try{
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {},
-                            userFeatureMap);
-                }catch (Exception e){
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }else{
-                return videos;
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-            "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-            "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-            "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                    "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                    "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                    "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-                ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null){
-            for (int i=0; i<videoFeatures.size(); ++i){
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null){
-                    continue;
-                }
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                            "total_time", "play_count_total",
-                            "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                            "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                }catch (Exception e){
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPart = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPart != null){
-            if (rtFeaPart.get(0) != null){
-                rtFeaPart1day = rtFeaPart.get(0);
-            }
-            if (rtFeaPart.get(1) != null){
-                rtFeaPart1h = rtFeaPart.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null){
-            int j = 0;
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setScoreStr(i.getScoreStr());
-            v.setScoreRos(i.getScoreRos());
-            return v;
-        });
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-            "高雄市".equals(city) |
-            "台中市".equals(city) |
-            "桃园市".equals(city) |
-            "新北市".equals(city) |
-            "台南市".equals(city) |
-            "基隆市".equals(city) |
-            "吉林市".equals(city) |
-            "新竹市".equals(city) |
-            "嘉义市".equals(city)
-        ){
-        }else{
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    public static void main(String[] args) {
-        Calendar calendar = Calendar.getInstance();
-        calendar.set(Calendar.YEAR, 2022);
-        calendar.set(Calendar.MONTH, 0); // January is 0
-        calendar.set(Calendar.DAY_OF_MONTH, 1);
-        calendar.set(Calendar.HOUR_OF_DAY, 0);
-        calendar.set(Calendar.MINUTE, 12);
-        calendar.set(Calendar.SECOND, 30);
-        System.out.println(new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        String provinceCn = "吉林省2";
-        provinceCn = provinceCn.replaceAll("省$", "");
-        System.out.println(provinceCn);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMerge.java

@@ -1,93 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMerge extends RankService {
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos){
-        Iterator<Video> iterator = videos.iterator();
-        while(iterator.hasNext()){
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())){
-                iterator.remove();
-            }else{
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        //-------------------地域内部融合+去重复-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------地域 sim returnv2 融合+去重复-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-
-//        rovRecallRank.addAll(v1);
-//        rovRecallRank.addAll(v2);
-//        rovRecallRank.addAll(v3);
-//        rovRecallRank.addAll(v4);
-//        rovRecallRank.addAll(v5);
-//        rovRecallRank.addAll(v6);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(20, v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(15, v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(10, v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(5, v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(10, v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(10, v6.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-        List<String> videoIdKeys = rovRecallRank.stream()
-                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-                .collect(Collectors.toList());
-        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-        if (CollectionUtils.isNotEmpty(videoScores)
-                && videoScores.size() == rovRecallRank.size()) {
-            for (int i = 0; i < videoScores.size(); i++) {
-                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-            }
-            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-        }
-        return rovRecallRank;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV4.java

@@ -1,618 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV4 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv4:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-    @Override
-    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
-        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
-        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
-            return quickFlowPoolVideos;
-        } else {
-            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
-        }
-    }
-    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
-
-        //初始化 userid
-        UserFeature userFeature = new UserFeature();
-        userFeature.setMid(param.getMid());
-
-        // 初始化RankItem
-        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
-                .filter(d -> d.getPushFrom().equals(pushFrom))
-                .findFirst();
-        if (!data.isPresent()){
-            return Collections.emptyList();
-        }
-        List<Video> videoList = data.get().getVideos();
-        if (videoList == null) {
-            return Collections.emptyList();
-        }
-        List<RankItem> rankItems = new ArrayList<>();
-        for (int i = 0; i < videoList.size(); i++) {
-            RankItem rankItem = new RankItem(videoList.get(i));
-            rankItems.add(rankItem);
-        }
-
-        // 初始化上下文参数
-        ScoreParam scoreParam = convert(param);
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
-                .scoring(scoreParam, userFeature, rankItems);
-
-        if (rovRecallScore == null) {
-            return Collections.emptyList();
-        }
-
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setSortScore(i.getScore());
-            return v;
-        });
-    }
-    public void duplicate(Set<Long> setVideo, List<Video> videos){
-        Iterator<Video> iterator = videos.iterator();
-        while(iterator.hasNext()){
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())){
-                iterator.remove();
-            }else{
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i=0; i<24; ++i){
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items){
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(returns, shares, 1.0, 10.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeight(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeight(returns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 1.0);
-        double b = mergeWeight.getOrDefault("b", 1.0);
-        double c = mergeWeight.getOrDefault("c", 0.0002);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
-        for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5){
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-            }else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-    public double calNewVideoScore(Map<String, String> itemBasicMap){
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5){
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-    public double calTrendScore(List<Double> data){
-        double sum = 0.0;
-        int size = data.size();
-        for (int i=0; i<size-4; ++i){
-            sum += data.get(i) - data.get(i+4);
-        }
-        if (sum * 10 > 0.6){
-            sum = 0.6;
-        }else{
-            sum = sum * 10;
-        }
-        if (sum > 0){
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-    public Double calScoreWeight(List<Double> data){
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i=0; i<data.size(); ++i){
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
-        }
-        return down > 1E-8? up / down: 0.0;
-    }
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
-        List<Double> data = new LinkedList<>();
-        for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
-        }
-        return data;
-    }
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key){
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours){
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
-            );
-        }
-        return views;
-    }
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart){
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()){
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()){
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null){
-                try{
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {},
-                            userFeatureMap);
-                }catch (Exception e){
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null){
-            for (int i=0; i<videoFeatures.size(); ++i){
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null){
-                    continue;
-                }
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                }catch (Exception e){
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(0) != null){
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null){
-            int j = 0;
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ){
-        }else{
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV5.java

@@ -1,631 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV5 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv5:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-    @Override
-    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
-        List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
-        if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
-            return quickFlowPoolVideos;
-        } else {
-            return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
-        }
-    }
-    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
-
-        //初始化 userid
-        UserFeature userFeature = new UserFeature();
-        userFeature.setMid(param.getMid());
-
-        // 初始化RankItem
-        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
-                .filter(d -> d.getPushFrom().equals(pushFrom))
-                .findFirst();
-        if (!data.isPresent()){
-            return Collections.emptyList();
-        }
-        List<Video> videoList = data.get().getVideos();
-        if (videoList == null) {
-            return Collections.emptyList();
-        }
-        List<RankItem> rankItems = new ArrayList<>();
-        for (int i = 0; i < videoList.size(); i++) {
-            RankItem rankItem = new RankItem(videoList.get(i));
-            rankItems.add(rankItem);
-        }
-
-        // 初始化上下文参数
-        ScoreParam scoreParam = convert(param);
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
-                .scoring(scoreParam, userFeature, rankItems);
-
-        if (rovRecallScore == null) {
-            return Collections.emptyList();
-        }
-
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setSortScore(i.getScore());
-            return v;
-        });
-    }
-    public void duplicate(Set<Long> setVideo, List<Video> videos){
-        Iterator<Video> iterator = videos.iterator();
-        while(iterator.hasNext()){
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())){
-                iterator.remove();
-            }else{
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<String> videoIdKeys = rovRecallRank.stream()
-                .map(t -> param.getRankKeyPrefix() + t.getVideoId())
-                .collect(Collectors.toList());
-        List<String> videoScores = this.redisTemplate.opsForValue().multiGet(videoIdKeys);
-        if (CollectionUtils.isNotEmpty(videoScores)
-                && videoScores.size() == rovRecallRank.size()) {
-            for (int i = 0; i < videoScores.size(); i++) {
-                rovRecallRank.get(i).setSortScore(NumberUtils.toDouble(videoScores.get(i), 0.0));
-            }
-            Collections.sort(rovRecallRank, Comparator.comparingDouble(o -> -o.getSortScore()));
-        }
-        return rovRecallRank;
-
-//        // 1 模型分
-//        List<String> rtFeaPart = new ArrayList<>();
-//        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-//        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-//        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-//        Calendar calendar = Calendar.getInstance();
-//        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-//        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-//        String rtFeaPart1h = date + hour;
-//        if (rtFeaPartKeyResult != null){
-//            if (rtFeaPartKeyResult.get(1) != null){
-//                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-//            }
-//        }
-//        // 2 统计分
-//        String cur = rtFeaPart1h;
-//        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-//        for (int i=0; i<24; ++i){
-//            datehours.add(cur);
-//            cur = ExtractorUtils.subtractHours(cur, 1);
-//        }
-//        for (RankItem item : items){
-//            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-//            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-//            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-//            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-//            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-//            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-//            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-//
-//            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
-//            Double share2returnScore = calScoreWeight(share2return);
-//            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-//            Double view2returnScore = calScoreWeight(view2return);
-//            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-//            Double view2playScore = calScoreWeight(view2play);
-//            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-//            Double play2shareScore = calScoreWeight(play2share);
-//            item.scoresMap.put("share2returnScore", share2returnScore);
-//            item.scoresMap.put("view2returnScore", view2returnScore);
-//            item.scoresMap.put("view2playScore", view2playScore);
-//            item.scoresMap.put("play2shareScore", play2shareScore);
-//
-//            // 全部回流
-//            Double allreturnsScore = calScoreWeight(allreturns);
-//            item.scoresMap.put("allreturnsScore", allreturnsScore);
-//
-//            // 平台回流
-//            Double preturnsScore = calScoreWeight(returns);
-//            item.scoresMap.put("preturnsScore", preturnsScore);
-//
-//            // rov的趋势
-//            double trendScore = calTrendScore(view2return);
-//            item.scoresMap.put("trendScore", trendScore);
-//
-//            // 新视频提取
-//            double newVideoScore = calNewVideoScore(itemBasicMap);
-//            item.scoresMap.put("newVideoScore", newVideoScore);
-//        }
-//        // 3 融合公式
-//        List<Video> result = new ArrayList<>();
-//        double a = mergeWeight.getOrDefault("a", 1.0);
-//        double b = mergeWeight.getOrDefault("b", 1.0);
-//        double c = mergeWeight.getOrDefault("c", 0.0002);
-//        double d = mergeWeight.getOrDefault("d", 1.0);
-//        double e = mergeWeight.getOrDefault("e", 1.0);
-//        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
-//        for (RankItem item : items){
-//            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-//                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-//            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-//                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-//            double strScore = item.getScoreStr();
-//            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-//            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-//            double score = 0.0;
-//            if (ifAdd < 0.5){
-//                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-//                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-//            }else {
-//                score = a * strScore + b * rosScore + c * preturnsScore +
-//                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-//            }
-//            Video video = item.getVideo();
-//            video.setScore(score);
-//            video.setSortScore(score);
-//            video.setScoreStr(item.getScoreStr());
-//            video.setScoresMap(item.getScoresMap());
-//            result.add(video);
-//        }
-//        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-//        return result;
-    }
-    public double calNewVideoScore(Map<String, String> itemBasicMap){
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5){
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-    public double calTrendScore(List<Double> data){
-        double sum = 0.0;
-        int size = data.size();
-        for (int i=0; i<size-4; ++i){
-            sum += data.get(i) - data.get(i+4);
-        }
-        if (sum * 10 > 0.6){
-            sum = 0.6;
-        }else{
-            sum = sum * 10;
-        }
-        if (sum > 0){
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-    public Double calScoreWeight(List<Double> data){
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i=0; i<data.size(); ++i){
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
-        }
-        return down > 1E-8? up / down: 0.0;
-    }
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
-        List<Double> data = new LinkedList<>();
-        for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
-        }
-        return data;
-    }
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key){
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours){
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
-            );
-        }
-        return views;
-    }
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart){
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()){
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()){
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null){
-                try{
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {},
-                            userFeatureMap);
-                }catch (Exception e){
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null){
-            for (int i=0; i<videoFeatures.size(); ++i){
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null){
-                    continue;
-                }
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                }catch (Exception e){
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(0) != null){
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null){
-            int j = 0;
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ){
-        }else{
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV546.java

@@ -1,654 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV546 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv546:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
-        removeDuplicate(oldRovs);
-        oldRovs = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, oldRovs);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        rovRecallRank.addAll(oldRovs);
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.8);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 240.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5) {
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            } else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-
-            }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV547 job = new RankStrategy4RegionMergeModelV547();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
-
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV547.java

@@ -1,654 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV547 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv547:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
-        removeDuplicate(oldRovs);
-        oldRovs = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, oldRovs);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        rovRecallRank.addAll(oldRovs);
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.6);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 240.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5) {
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            } else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-
-            }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV547 job = new RankStrategy4RegionMergeModelV547();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
-
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV548.java

@@ -1,617 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV548 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv548:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
-        removeDuplicate(oldRovs);
-        oldRovs = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, oldRovs);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        rovRecallRank.addAll(oldRovs);
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
-
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeight(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeight(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeight(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeight(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.8);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 240.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5) {
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            } else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-
-            }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    public Double calScoreWeight(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV551.java

@@ -1,374 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.FeatureService;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.MapUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.*;
-import java.util.stream.Collectors;
-
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV551 extends RankStrategy4RegionMergeModelBasic {
-    @ApolloJsonValue("${rank.score.merge.weightv551:}")
-    private Map<String, Double> mergeWeight;
-
-    @Autowired
-    private FeatureService featureService;
-
-    Map<String, double[]> bucketsMap = new HashMap<>();
-    Map<String, Double> bucketsLen = new HashMap<>();
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        List<Video> rovRecallRank = new ArrayList<>(v0);
-        //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
-        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
-
-        // k1:视频、k2:表、k3:特征、v:特征值
-        String provinceCn = param.getProvince().replaceAll("省$", "");
-        String headVid = String.valueOf(param.getHeadVid());
-        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
-                String.valueOf(param.getAppType()), provinceCn, headVid);
-        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
-        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
-
-
-        // TODO 2 特征处理
-        Map<String, Double> userFeatureMapDouble = new HashMap<>();
-        String mid = param.getMid();
-        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
-        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
-        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
-        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
-        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
-        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
-        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
-        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
-        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
-
-        if (!c1.isEmpty()) {
-            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
-            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
-            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
-            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
-        }
-        if (!c2.isEmpty()) {
-            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
-            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
-            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
-            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
-            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
-            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
-            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
-            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
-        }
-
-        Map<String, String> c34567Map = new HashMap<>(15);
-        List<Tuple2> tmpList0 = Arrays.asList(
-                new Tuple2(c3, "c3_feature"),
-                new Tuple2(c4, "c4_feature"),
-                new Tuple2(c5, "c5_feature"),
-                new Tuple2(c6, "c6_feature"),
-                new Tuple2(c7, "c7_feature")
-        );
-        for (Tuple2 tuple2 : tmpList0) {
-            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                String tags = tuple2.first.getOrDefault(key_time, "");
-                if (!tags.isEmpty()) {
-                    c34567Map.put(tuple2.name + "_" + key_time, tags);
-                }
-            }
-        }
-
-        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
-        List<Tuple2> tmpList1 = Arrays.asList(
-                new Tuple2(c8, "c8_feature"),
-                new Tuple2(c9, "c9_feature")
-        );
-        for (Tuple2 tuple2 : tmpList1) {
-            for (String key_action : Arrays.asList("share", "return")) {
-                String cfListStr = tuple2.first.getOrDefault(key_action, "");
-                if (!cfListStr.isEmpty()) {
-                    Map<String, String[]> cfMap = new HashMap<>();
-                    String[] entries = cfListStr.split(",");
-                    for (String entry : entries) {
-                        String[] rList = entry.split(":");
-                        if (rList.length >= 4) { // 确保分割后有四个元素
-                            String key = rList[0];
-                            String value1 = rList[1];
-                            String value2 = rList[2];
-                            String value3 = rList[3];
-                            String[] strs = {value1, value2, value3};
-                            cfMap.put(key, strs);
-                        }
-                    }
-                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
-                }
-            }
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
-        for (RankItem item : rankItems) {
-            Map<String, Double> featureMap = new HashMap<>();
-            String vid = item.getVideoId() + "";
-            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
-            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
-            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
-            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
-            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
-
-            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
-            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
-            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
-            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
-            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
-            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
-            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
-            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
-            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
-
-            List<Tuple4> originData = Arrays.asList(
-                    new Tuple4(b1, b2, b3, "b123"),
-                    new Tuple4(b1, b6, b7, "b167"),
-                    new Tuple4(b8, b9, b10, "b8910"),
-                    new Tuple4(b11, b12, b13, "b111213"),
-                    new Tuple4(b17, b18, b19, "b171819")
-            );
-
-            for (Tuple4 tuple4 : originData) {
-                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
-                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
-                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
-                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
-
-                    double f1 = ExtractorUtils.calDiv(share, exp);
-                    double f2 = ExtractorUtils.calLog(share);
-                    double f3 = ExtractorUtils.calDiv(returns, exp);
-                    double f4 = ExtractorUtils.calLog(returns);
-                    double f5 = f3 * f4;
-
-                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
-                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
-                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
-                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
-                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
-
-                    featureMap.put(key1, f1);
-                    featureMap.put(key2, f2);
-                    featureMap.put(key3, f3);
-                    featureMap.put(key4, f4);
-                    featureMap.put(key5, f5);
-                }
-            }
-
-            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
-            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
-            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
-
-            String title = videoInfo.getOrDefault("title", "");
-            if (!title.isEmpty()) {
-                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
-                        if (!tags.isEmpty()) {
-                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
-                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
-                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
-                        }
-                    }
-                }
-            }
-
-            if (!vid.isEmpty()) {
-                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
-                    for (String key_action : Arrays.asList("share", "return")) {
-                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
-                        if (cfMap.containsKey(vid)) {
-                            String[] scores = cfMap.get(vid);
-                            Double score1 = Double.parseDouble(scores[0]);
-                            Double score2 = Double.parseDouble(scores[1]);
-                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
-                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
-                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
-                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
-                        }
-                    }
-                }
-            }
-            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
-            if (!d1.isEmpty()) {
-                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
-                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
-                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
-            }
-            item.featureMapDouble = featureMap;
-        }
-
-        // 3 连续值特征分桶
-        readBucketFile();
-        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
-        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
-            String name = entry.getKey();
-            Double score = entry.getValue();
-            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                Double bucketNum = this.bucketsLen.get(name);
-                double[] buckets = this.bucketsMap.get(name);
-                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                userFeatureMap.put(name, String.valueOf(scoreNew));
-            }
-        }
-
-        for (RankItem item : rankItems) {
-            Map<String, String> featureMap = new HashMap<>();
-            Map<String, Double> featureMapDouble = item.featureMapDouble;
-
-            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
-                String name = entry.getKey();
-                Double score = entry.getValue();
-                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                    Double bucketNum = this.bucketsLen.get(name);
-                    double[] buckets = this.bucketsMap.get(name);
-                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                    featureMap.put(name, String.valueOf(scoreNew));
-                }
-            }
-            item.featureMap = featureMap;
-        }
-
-        // TODO 3 排序
-        Map<String, String> sceneFeatureMap = new HashMap<>(0);
-
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240806.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
-        List<Video> result = new ArrayList<>();
-        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
-        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
-
-        for (RankItem item : items) {
-            double score = 0.0;
-            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault(hasReturnRovKey, "0"));
-            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double fmRov = item.getScoreRov();
-            item.getScoresMap().put("fmRov", fmRov);
-            if (chooseFunction == 0){
-                score = fmRov * (1 + hasReturnRovScore);
-            }else if (chooseFunction == 1){
-                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
-            }else {
-                score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
-            }
-
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoresMap(item.getScoresMap());
-            video.setAllFeatureMap(item.getAllFeatureMap());
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getVideoFeature())
-                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
-                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
-            }
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
-                video.getMetaFeatureMap().putAll(feature.getUserFeature());
-            }
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-
-        return result;
-    }
-
-    public void readBucketFile() {
-        InputStream resourceStream = RankStrategy4RegionMergeModelV551.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
-        if (resourceStream != null) {
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
-                Map<String, double[]> bucketsMap = new HashMap<>();
-                Map<String, Double> bucketsLen = new HashMap<>();
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    // 替换空格和换行符，过滤空行
-                    line = line.replace(" ", "").replaceAll("\n", "");
-                    if (!line.isEmpty()) {
-                        String[] rList = line.split("\t");
-                        if (rList.length == 3) {
-                            String key = rList[0];
-                            double value1 = Double.parseDouble(rList[1]);
-                            bucketsLen.put(key, value1);
-                            double[] value2 = Arrays.stream(rList[2].split(","))
-                                    .mapToDouble(Double::valueOf)
-                                    .toArray();
-                            bucketsMap.put(key, value2);
-                        }
-                    }
-                }
-                this.bucketsMap = bucketsMap;
-                this.bucketsLen = bucketsLen;
-            } catch (IOException e) {
-                log.error("something is wrong in parse bucket file:" + e);
-            }
-        } else {
-            log.error("no bucket file");
-        }
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV552.java

@@ -1,414 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.FeatureService;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.math3.util.Pair;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV552 extends RankStrategy4RegionMergeModelBasic {
-    @ApolloJsonValue("${rank.score.merge.weightv552:}")
-    private Map<String, Double> mergeWeight;
-
-    @Autowired
-    private FeatureService featureService;
-
-    Map<String, double[]> bucketsMap = new HashMap<>();
-    Map<String, Double> bucketsLen = new HashMap<>();
-
-    @Value("${similarity.concurrent: false}")
-    private boolean similarityConcurrent;
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        List<Video> rovRecallRank = new ArrayList<>(v0);
-        //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
-        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
-
-        // k1:视频、k2:表、k3:特征、v:特征值
-        String provinceCn = param.getProvince().replaceAll("省$", "");
-        String headVid = String.valueOf(param.getHeadVid());
-        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
-                String.valueOf(param.getAppType()), provinceCn, headVid);
-        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
-        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
-
-
-        // 2 特征处理
-        Map<String, Double> userFeatureMapDouble = new HashMap<>();
-        String mid = param.getMid();
-        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
-        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
-        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
-        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
-        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
-        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
-        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
-        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
-        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
-
-        if (!c1.isEmpty()) {
-            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
-            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
-            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
-            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
-        }
-        if (!c2.isEmpty()) {
-            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
-            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
-            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
-            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
-            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
-            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
-            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
-            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
-        }
-
-        Map<String, String> c34567Map = new HashMap<>(15);
-        List<Tuple2> tmpList0 = Arrays.asList(
-                new Tuple2(c3, "c3_feature"),
-                new Tuple2(c4, "c4_feature"),
-                new Tuple2(c5, "c5_feature"),
-                new Tuple2(c6, "c6_feature"),
-                new Tuple2(c7, "c7_feature")
-        );
-        for (Tuple2 tuple2 : tmpList0) {
-            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                String tags = tuple2.first.getOrDefault(key_time, "");
-                if (!tags.isEmpty()) {
-                    c34567Map.put(tuple2.name + "_" + key_time, tags);
-                }
-            }
-        }
-
-        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
-        List<Tuple2> tmpList1 = Arrays.asList(
-                new Tuple2(c8, "c8_feature"),
-                new Tuple2(c9, "c9_feature")
-        );
-        for (Tuple2 tuple2 : tmpList1) {
-            for (String key_action : Arrays.asList("share", "return")) {
-                String cfListStr = tuple2.first.getOrDefault(key_action, "");
-                if (!cfListStr.isEmpty()) {
-                    Map<String, String[]> cfMap = new HashMap<>();
-                    String[] entries = cfListStr.split(",");
-                    for (String entry : entries) {
-                        String[] rList = entry.split(":");
-                        if (rList.length >= 4) { // 确保分割后有四个元素
-                            String key = rList[0];
-                            String value1 = rList[1];
-                            String value2 = rList[2];
-                            String value3 = rList[3];
-                            String[] strs = {value1, value2, value3};
-                            cfMap.put(key, strs);
-                        }
-                    }
-                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
-                }
-            }
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
-        for (RankItem item : rankItems) {
-            Map<String, Double> featureMap = new HashMap<>();
-            String vid = item.getVideoId() + "";
-            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
-            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
-            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
-            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
-            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
-
-            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
-            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
-            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
-            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
-            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
-            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
-            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
-            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
-            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
-
-            List<Tuple4> originData = Arrays.asList(
-                    new Tuple4(b1, b2, b3, "b123"),
-                    new Tuple4(b1, b6, b7, "b167"),
-                    new Tuple4(b8, b9, b10, "b8910"),
-                    new Tuple4(b11, b12, b13, "b111213"),
-                    new Tuple4(b17, b18, b19, "b171819")
-            );
-
-            for (Tuple4 tuple4 : originData) {
-                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
-                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
-                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
-                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
-
-                    double f1 = ExtractorUtils.calDiv(share, exp);
-                    double f2 = ExtractorUtils.calLog(share);
-                    double f3 = ExtractorUtils.calDiv(returns, exp);
-                    double f4 = ExtractorUtils.calLog(returns);
-                    double f5 = f3 * f4;
-
-                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
-                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
-                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
-                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
-                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
-
-                    featureMap.put(key1, f1);
-                    featureMap.put(key2, f2);
-                    featureMap.put(key3, f3);
-                    featureMap.put(key4, f4);
-                    featureMap.put(key5, f5);
-                }
-            }
-
-            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
-            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
-            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
-
-            String title = videoInfo.getOrDefault("title", "");
-            if (!title.isEmpty()) {
-                if (similarityConcurrent) {
-                    List<Future<Pair<String, Double[]>>> futures = new ArrayList<>();
-                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                            String key = name + "_" + key_time;
-                            String tags = c34567Map.getOrDefault(key, "");
-                            if (!tags.isEmpty()) {
-                                Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                                    Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                    return Pair.create(key, doubles);
-                                });
-                                futures.add(future);
-                            }
-                        }
-                    }
-                    try {
-                        for (Future<Pair<String, Double[]>> future : futures) {
-                            Pair<String, Double[]> pair = future.get(1000, TimeUnit.MILLISECONDS);
-                            featureMap.put(pair.getFirst() + "_matchnum", pair.getSecond()[0]);
-                            featureMap.put(pair.getFirst() + "_maxscore", pair.getSecond()[1]);
-                            featureMap.put(pair.getFirst() + "_avgscore", pair.getSecond()[2]);
-                        }
-                    } catch (Exception e) {
-                        log.error("concurrent similarity error", e);
-                    }
-                } else {
-                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                            String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
-                            if (!tags.isEmpty()) {
-                                Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
-                                featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
-                                featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
-                            }
-                        }
-                    }
-                }
-            }
-
-            if (!vid.isEmpty()) {
-                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
-                    for (String key_action : Arrays.asList("share", "return")) {
-                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
-                        if (cfMap.containsKey(vid)) {
-                            String[] scores = cfMap.get(vid);
-                            Double score1 = Double.parseDouble(scores[0]);
-                            Double score2 = Double.parseDouble(scores[1]);
-                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
-                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
-                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
-                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
-                        }
-                    }
-                }
-            }
-            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
-            if (!d1.isEmpty()) {
-                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
-                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
-                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
-            }
-            item.featureMapDouble = featureMap;
-        }
-
-        // 3 连续值特征分桶
-        readBucketFile();
-        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
-        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
-            String name = entry.getKey();
-            Double score = entry.getValue();
-            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                Double bucketNum = this.bucketsLen.get(name);
-                double[] buckets = this.bucketsMap.get(name);
-                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                userFeatureMap.put(name, String.valueOf(scoreNew));
-            }
-        }
-
-        for (RankItem item : rankItems) {
-            Map<String, String> featureMap = new HashMap<>();
-            Map<String, Double> featureMapDouble = item.featureMapDouble;
-
-            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
-                String name = entry.getKey();
-                Double score = entry.getValue();
-                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                    Double bucketNum = this.bucketsLen.get(name);
-                    double[] buckets = this.bucketsMap.get(name);
-                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                    featureMap.put(name, String.valueOf(scoreNew));
-                }
-            }
-            item.featureMap = featureMap;
-        }
-
-        // 3 排序
-        Map<String, String> sceneFeatureMap = new HashMap<>(0);
-
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240807.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
-        List<Video> result = new ArrayList<>();
-        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
-        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
-
-        for (RankItem item : items) {
-            double score = 0.0;
-            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault(hasReturnRovKey, "0"));
-            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double fmRovOrigin = item.getScoreRov();
-            item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
-            double fmRov = restoreScore(fmRovOrigin);
-            item.getScoresMap().put("fmRov", fmRov);
-            if (chooseFunction == 0){
-                score = fmRov * (1 + hasReturnRovScore);
-            }else if (chooseFunction == 1){
-                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
-            }else {
-                score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
-            }
-
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoresMap(item.getScoresMap());
-            video.setAllFeatureMap(item.getAllFeatureMap());
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getVideoFeature())
-                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
-                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
-            }
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
-                video.getMetaFeatureMap().putAll(feature.getUserFeature());
-            }
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-
-        return result;
-    }
-
-    public void readBucketFile() {
-        InputStream resourceStream = RankStrategy4RegionMergeModelV552.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
-        if (resourceStream != null) {
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
-                Map<String, double[]> bucketsMap = new HashMap<>();
-                Map<String, Double> bucketsLen = new HashMap<>();
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    // 替换空格和换行符，过滤空行
-                    line = line.replace(" ", "").replaceAll("\n", "");
-                    if (!line.isEmpty()) {
-                        String[] rList = line.split("\t");
-                        if (rList.length == 3) {
-                            String key = rList[0];
-                            double value1 = Double.parseDouble(rList[1]);
-                            bucketsLen.put(key, value1);
-                            double[] value2 = Arrays.stream(rList[2].split(","))
-                                    .mapToDouble(Double::valueOf)
-                                    .toArray();
-                            bucketsMap.put(key, value2);
-                        }
-                    }
-                }
-                this.bucketsMap = bucketsMap;
-                this.bucketsLen = bucketsLen;
-            } catch (IOException e) {
-                log.error("something is wrong in parse bucket file:" + e);
-            }
-        } else {
-            log.error("no bucket file");
-        }
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV553.java

@@ -1,403 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.FeatureService;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.math3.util.Pair;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV553 extends RankStrategy4RegionMergeModelBasic {
-    @ApolloJsonValue("${rank.score.merge.weightv553:}")
-    private Map<String, Double> mergeWeight;
-
-    @Autowired
-    private FeatureService featureService;
-
-
-    @Value("${similarity.concurrent: true}")
-    private boolean similarityConcurrent;
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        List<Video> rovRecallRank = new ArrayList<>(v0);
-        //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------老内容召回------------------
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV7VovLongTermV1.PUSH_FORM);
-        v2 = v2.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v2 = v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 2.0).intValue(), v2.size()));
-        rovRecallRank.addAll(v2);
-        setVideo.addAll(v2.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV7VovLongTermV2.PUSH_FORM);
-        v3 = v3.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v3 = v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 2.0).intValue(), v3.size()));
-        rovRecallRank.addAll(v3);
-        setVideo.addAll(v3.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV7VovLongTermV3.PUSH_FORM);
-        v4 = v4.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v4 = v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 2.0).intValue(), v4.size()));
-        rovRecallRank.addAll(v4);
-        setVideo.addAll(v4.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        List<Video> v5 = extractAndSort(param, RegionRealtimeRecallStrategyV7VovLongTermV4.PUSH_FORM);
-        v5 = v5.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v5 = v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 2.0).intValue(), v5.size()));
-        rovRecallRank.addAll(v5);
-        setVideo.addAll(v5.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
-        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
-
-        // k1:视频、k2:表、k3:特征、v:特征值
-        String provinceCn = param.getProvince().replaceAll("省$", "");
-        String headVid = String.valueOf(param.getHeadVid());
-        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
-                String.valueOf(param.getAppType()), provinceCn, headVid);
-        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
-        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
-
-
-        // 2 特征处理
-        Map<String, Double> userFeatureMapDouble = new HashMap<>();
-        String mid = param.getMid();
-        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
-        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
-        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
-        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
-        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
-        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
-        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
-        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
-        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
-
-        if (!c1.isEmpty()) {
-            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
-            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
-            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
-            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
-        }
-        if (!c2.isEmpty()) {
-            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
-            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
-            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
-            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
-            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
-            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
-            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
-            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
-        }
-
-        Map<String, String> c34567Map = new HashMap<>(15);
-        List<Tuple2> tmpList0 = Arrays.asList(
-                new Tuple2(c3, "c3_feature"),
-                new Tuple2(c4, "c4_feature"),
-                new Tuple2(c5, "c5_feature"),
-                new Tuple2(c6, "c6_feature"),
-                new Tuple2(c7, "c7_feature")
-        );
-        for (Tuple2 tuple2 : tmpList0) {
-            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                String tags = tuple2.first.getOrDefault(key_time, "");
-                if (!tags.isEmpty()) {
-                    c34567Map.put(tuple2.name + "_" + key_time, tags);
-                }
-            }
-        }
-
-        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
-        List<Tuple2> tmpList1 = Arrays.asList(
-                new Tuple2(c8, "c8_feature"),
-                new Tuple2(c9, "c9_feature")
-        );
-        for (Tuple2 tuple2 : tmpList1) {
-            for (String key_action : Arrays.asList("share", "return")) {
-                String cfListStr = tuple2.first.getOrDefault(key_action, "");
-                if (!cfListStr.isEmpty()) {
-                    Map<String, String[]> cfMap = new HashMap<>();
-                    String[] entries = cfListStr.split(",");
-                    for (String entry : entries) {
-                        String[] rList = entry.split(":");
-                        if (rList.length >= 4) { // 确保分割后有四个元素
-                            String key = rList[0];
-                            String value1 = rList[1];
-                            String value2 = rList[2];
-                            String value3 = rList[3];
-                            String[] strs = {value1, value2, value3};
-                            cfMap.put(key, strs);
-                        }
-                    }
-                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
-                }
-            }
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
-        for (RankItem item : rankItems) {
-            Map<String, Double> featureMap = new HashMap<>();
-            String vid = item.getVideoId() + "";
-            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
-            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
-            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
-            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
-            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
-
-            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
-            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
-            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
-            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
-            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
-            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
-            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
-            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
-            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
-
-            List<Tuple4> originData = Arrays.asList(
-                    new Tuple4(b1, b2, b3, "b123"),
-                    new Tuple4(b1, b6, b7, "b167"),
-                    new Tuple4(b8, b9, b10, "b8910"),
-                    new Tuple4(b11, b12, b13, "b111213"),
-                    new Tuple4(b17, b18, b19, "b171819")
-            );
-
-            for (Tuple4 tuple4 : originData) {
-                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
-                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
-                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
-                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
-
-                    double f1 = ExtractorUtils.calDiv(share, exp);
-                    double f2 = ExtractorUtils.calLog(share);
-                    double f3 = ExtractorUtils.calDiv(returns, exp);
-                    double f4 = ExtractorUtils.calLog(returns);
-                    double f5 = f3 * f4;
-
-                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
-                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
-                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
-                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
-                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
-
-                    featureMap.put(key1, f1);
-                    featureMap.put(key2, f2);
-                    featureMap.put(key3, f3);
-                    featureMap.put(key4, f4);
-                    featureMap.put(key5, f5);
-                }
-            }
-
-            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
-            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
-            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
-
-            String title = videoInfo.getOrDefault("title", "");
-            if (!title.isEmpty()) {
-                if (similarityConcurrent) {
-                    List<Future<Pair<String, Double[]>>> futures = new ArrayList<>();
-                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                            String key = name + "_" + key_time;
-                            String tags = c34567Map.getOrDefault(key, "");
-                            if (!tags.isEmpty()) {
-                                Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                                    Double[] doubles = null;
-                                    if (param.getAbExpCodes().contains(word2vecExp)) {
-                                        doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
-                                    } else {
-                                        doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                    }
-                                    return Pair.create(key, doubles);
-                                });
-                                futures.add(future);
-                            }
-                        }
-                    }
-                    try {
-                        for (Future<Pair<String, Double[]>> future : futures) {
-                            Pair<String, Double[]> pair = future.get(1000, TimeUnit.MILLISECONDS);
-                            featureMap.put(pair.getFirst() + "_matchnum", pair.getSecond()[0]);
-                            featureMap.put(pair.getFirst() + "_maxscore", pair.getSecond()[1]);
-                            featureMap.put(pair.getFirst() + "_avgscore", pair.getSecond()[2]);
-                        }
-                    } catch (Exception e) {
-                        log.error("concurrent similarity error", e);
-                    }
-                } else {
-                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                            String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
-                            if (!tags.isEmpty()) {
-                                Double[] doubles = null;
-                                if (param.getAbExpCodes().contains(word2vecExp)) {
-                                    doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
-                                } else {
-                                    doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                }
-                                featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
-                                featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
-                                featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
-                            }
-                        }
-                    }
-                }
-            }
-
-            if (!vid.isEmpty()) {
-                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
-                    for (String key_action : Arrays.asList("share", "return")) {
-                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
-                        if (cfMap.containsKey(vid)) {
-                            String[] scores = cfMap.get(vid);
-                            Double score1 = Double.parseDouble(scores[0]);
-                            Double score2 = Double.parseDouble(scores[1]);
-                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
-                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
-                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
-                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
-                        }
-                    }
-                }
-            }
-            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
-            if (!d1.isEmpty()) {
-                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
-                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
-                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
-            }
-            item.featureMapDouble = featureMap;
-        }
-
-        // 3 连续值特征分桶
-        readBucketFile();
-        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
-        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
-            String name = entry.getKey();
-            Double score = entry.getValue();
-            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                Double bucketNum = this.bucketsLen.get(name);
-                double[] buckets = this.bucketsMap.get(name);
-                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                userFeatureMap.put(name, String.valueOf(scoreNew));
-            }
-        }
-
-        for (RankItem item : rankItems) {
-            Map<String, String> featureMap = new HashMap<>();
-            Map<String, Double> featureMapDouble = item.featureMapDouble;
-
-            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
-                String name = entry.getKey();
-                Double score = entry.getValue();
-                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                    Double bucketNum = this.bucketsLen.get(name);
-                    double[] buckets = this.bucketsMap.get(name);
-                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                    featureMap.put(name, String.valueOf(scoreNew));
-                }
-            }
-            item.featureMap = featureMap;
-        }
-        // 4 排序模型计算
-        Map<String, String> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240807.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        // 5 排序公式特征
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
-        Map<String, Map<String, String>> vid2VovFeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vovh24pred:");
-        double alpha_vov = mergeWeight.getOrDefault("alpha_vov", 0.05);
-        double func = mergeWeight.getOrDefault("func", 1.0);
-        List<Video> result = new ArrayList<>();
-        for (RankItem item : items) {
-            item.getScoresMap().put("alpha_vov", alpha_vov);
-            double score = 0.0;
-            double fmRovOrigin = item.getScoreRov();
-            item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
-            double fmRov = restoreScore(fmRovOrigin);
-            item.getScoresMap().put("fmRov", fmRov);
-            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rate_n", "0"));
-            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double vovScore = this.calVovScore(item, vid2VovFeatureMap);
-            item.getScoresMap().put("vovScore", vovScore);
-            if (func == 1){
-                score = fmRov * (1 + hasReturnRovScore) + alpha_vov * vovScore;
-            }else{
-                score = fmRov * (1 + hasReturnRovScore) * (1.0 + alpha_vov * vovScore);
-            }
-
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoresMap(item.getScoresMap());
-            video.setAllFeatureMap(item.getAllFeatureMap());
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getVideoFeature())
-                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
-                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
-            }
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
-                video.getMetaFeatureMap().putAll(feature.getUserFeature());
-            }
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV561.java

@@ -1,561 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV561 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv561:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
-        removeDuplicate(rovRecallRank);
-        rovRecallRank = rovRecallRank.size() <= sizeReturn
-                ? rovRecallRank
-                : rovRecallRank.subList(0, sizeReturn);
-        rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
-        rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
-        removeDuplicate(rovRecallRank);
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<RankItem> items = model(rovRecallRank, param);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i=0; i<24; ++i){
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items){
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeight(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeight(returns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 1.0);
-        double b = mergeWeight.getOrDefault("b", 1.0);
-        double c = mergeWeight.getOrDefault("c", 0.0002);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
-        for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScoreModel = item.getScoreRos();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5){
-                score = Math.pow(strScore, a) * Math.pow(rosScoreModel, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-            }else {
-                score = a * strScore + b * rosScoreModel + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoreRos(item.getScoreRos());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap){
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5){
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-    public double calTrendScore(List<Double> data){
-        double sum = 0.0;
-        int size = data.size();
-        for (int i=0; i<size-4; ++i){
-            sum += data.get(i) - data.get(i+4);
-        }
-        if (sum * 10 > 0.6){
-            sum = 0.6;
-        }else{
-            sum = sum * 10;
-        }
-        if (sum > 0){
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-    public Double calScoreWeight(List<Double> data){
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i=0; i<data.size(); ++i){
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
-        }
-        return down > 1E-8? up / down: 0.0;
-    }
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
-        List<Double> data = new LinkedList<>();
-        for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
-        }
-        return data;
-    }
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key){
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours){
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
-            );
-        }
-        return views;
-    }
-    public List<RankItem> model(List<Video> videos, RankParam param){
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()){
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()){
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null){
-                try{
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {},
-                            userFeatureMap);
-                }catch (Exception e){
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }else{
-                JSONObject obj = new JSONObject();
-                obj.put("name", "user_key_in_model_is_null");
-                obj.put("class", this.CLASS_NAME);
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
-                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
-        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
-        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
-                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
-                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null){
-            for (int i=0; i<videoFeatures.size(); ++i){
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null){
-                    continue;
-                }
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
-                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
-                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
-                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                }catch (Exception e){
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(0) != null){
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-        if (videoRtFeatures != null){
-            int j = 0;
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        JSONObject obj = new JSONObject();
-        obj.put("name", "user_key_in_model_is_not_null");
-        obj.put("class", this.CLASS_NAME);
-        return rovRecallScore;
-    }
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ){
-            ;
-        }else{
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV565.java

@@ -1,545 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.FeatureService;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.math3.util.Pair;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.*;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeModelBasic {
-    @ApolloJsonValue("${rank.score.merge.weightv565:}")
-    private Map<String, Double> mergeWeight;
-    @Autowired
-    private FeatureService featureService;
-
-    Map<String, double[]> bucketsMap = new HashMap<>();
-    Map<String, Double> bucketsLen = new HashMap<>();
-
-    @Value("${similarity.concurrent: false}")
-    private boolean similarityConcurrent;
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        List<Video> rovRecallRank = new ArrayList<>(v0);
-        //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
-        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
-
-        // k1:视频、k2:表、k3:特征、v:特征值
-        String provinceCn = param.getProvince().replaceAll("省$", "");
-        String headVid = String.valueOf(param.getHeadVid());
-        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
-                String.valueOf(param.getAppType()), provinceCn, headVid);
-        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
-        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
-
-
-        // 2 特征处理
-        Map<String, Double> userFeatureMapDouble = new HashMap<>();
-        String mid = param.getMid();
-        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
-        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
-        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
-        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
-        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
-        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
-        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
-        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
-        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
-
-        if (!c1.isEmpty()) {
-            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
-            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
-            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
-            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
-        }
-        if (!c2.isEmpty()) {
-            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
-            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
-            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
-            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
-            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
-            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
-            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
-            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
-        }
-
-        Map<String, String> c34567Map = new HashMap<>(15);
-        List<Tuple2> tmpList0 = Arrays.asList(
-                new Tuple2(c3, "c3_feature"),
-                new Tuple2(c4, "c4_feature"),
-                new Tuple2(c5, "c5_feature"),
-                new Tuple2(c6, "c6_feature"),
-                new Tuple2(c7, "c7_feature")
-        );
-        for (Tuple2 tuple2 : tmpList0) {
-            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                String tags = tuple2.first.getOrDefault(key_time, "");
-                if (!tags.isEmpty()) {
-                    c34567Map.put(tuple2.name + "_" + key_time, tags);
-                }
-            }
-        }
-
-        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
-        List<Tuple2> tmpList1 = Arrays.asList(
-                new Tuple2(c8, "c8_feature"),
-                new Tuple2(c9, "c9_feature")
-        );
-        for (Tuple2 tuple2 : tmpList1) {
-            for (String key_action : Arrays.asList("share", "return")) {
-                String cfListStr = tuple2.first.getOrDefault(key_action, "");
-                if (!cfListStr.isEmpty()) {
-                    Map<String, String[]> cfMap = new HashMap<>();
-                    String[] entries = cfListStr.split(",");
-                    for (String entry : entries) {
-                        String[] rList = entry.split(":");
-                        if (rList.length >= 4) { // 确保分割后有四个元素
-                            String key = rList[0];
-                            String value1 = rList[1];
-                            String value2 = rList[2];
-                            String value3 = rList[3];
-                            String[] strs = {value1, value2, value3};
-                            cfMap.put(key, strs);
-                        }
-                    }
-                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
-                }
-            }
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
-        for (RankItem item : rankItems) {
-            Map<String, Double> featureMap = new HashMap<>();
-            String vid = item.getVideoId() + "";
-            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
-            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
-            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
-            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
-            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
-
-            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
-            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
-            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
-            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
-            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
-            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
-            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
-            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
-            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
-
-            List<Tuple4> originData = Arrays.asList(
-                    new Tuple4(b1, b2, b3, "b123"),
-                    new Tuple4(b1, b6, b7, "b167"),
-                    new Tuple4(b8, b9, b10, "b8910"),
-                    new Tuple4(b11, b12, b13, "b111213"),
-                    new Tuple4(b17, b18, b19, "b171819")
-            );
-
-            for (Tuple4 tuple4 : originData) {
-                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
-                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
-                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
-                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
-
-                    double f1 = ExtractorUtils.calDiv(share, exp);
-                    double f2 = ExtractorUtils.calLog(share);
-                    double f3 = ExtractorUtils.calDiv(returns, exp);
-                    double f4 = ExtractorUtils.calLog(returns);
-                    double f5 = f3 * f4;
-
-                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
-                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
-                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
-                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
-                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
-
-                    featureMap.put(key1, f1);
-                    featureMap.put(key2, f2);
-                    featureMap.put(key3, f3);
-                    featureMap.put(key4, f4);
-                    featureMap.put(key5, f5);
-                }
-            }
-
-            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
-            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
-            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
-
-            String title = videoInfo.getOrDefault("title", "");
-            if (!title.isEmpty()) {
-                if (similarityConcurrent) {
-                    List<Future<Pair<String, Double[]>>> futures = new ArrayList<>();
-                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                            String key = name + "_" + key_time;
-                            String tags = c34567Map.getOrDefault(key, "");
-                            if (!tags.isEmpty()) {
-                                Future<Pair<String, Double[]>> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                                    Double[] doubles;
-                                    if (param.getAbExpCodes().contains(word2vecExp)) {
-                                        doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
-                                    } else {
-                                        doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                    }
-                                    return Pair.create(key, doubles);
-                                });
-                                futures.add(future);
-                            }
-                        }
-                    }
-                    try {
-                        for (Future<Pair<String, Double[]>> future : futures) {
-                            Pair<String, Double[]> pair = future.get(1000, TimeUnit.MILLISECONDS);
-                            featureMap.put(pair.getFirst() + "_matchnum", pair.getSecond()[0]);
-                            featureMap.put(pair.getFirst() + "_maxscore", pair.getSecond()[1]);
-                            featureMap.put(pair.getFirst() + "_avgscore", pair.getSecond()[2]);
-                        }
-                    } catch (Exception e) {
-                        log.error("concurrent similarity error", e);
-                    }
-                } else {
-                    for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                        for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                            String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
-                            if (!tags.isEmpty()) {
-                                Double[] doubles = null;
-                                if (param.getAbExpCodes().contains(word2vecExp)) {
-                                    doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
-                                } else {
-                                    doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                                }
-                                featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
-                                featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
-                                featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
-                            }
-                        }
-                    }
-                }
-            }
-
-            if (!vid.isEmpty()) {
-                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
-                    for (String key_action : Arrays.asList("share", "return")) {
-                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
-                        if (cfMap.containsKey(vid)) {
-                            String[] scores = cfMap.get(vid);
-                            Double score1 = Double.parseDouble(scores[0]);
-                            Double score2 = Double.parseDouble(scores[1]);
-                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
-                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
-                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
-                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
-                        }
-                    }
-                }
-            }
-            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
-            if (!d1.isEmpty()) {
-                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
-                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
-                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
-            }
-            item.featureMapDouble = featureMap;
-        }
-
-        // 3 连续值特征分桶
-        readBucketFile();
-        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
-        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
-            String name = entry.getKey();
-            Double score = entry.getValue();
-            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                Double bucketNum = this.bucketsLen.get(name);
-                double[] buckets = this.bucketsMap.get(name);
-                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                userFeatureMap.put(name, String.valueOf(scoreNew));
-            }
-        }
-
-        for (RankItem item : rankItems) {
-            Map<String, String> featureMap = new HashMap<>();
-            Map<String, Double> featureMapDouble = item.featureMapDouble;
-
-            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
-                String name = entry.getKey();
-                Double score = entry.getValue();
-                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                    Double bucketNum = this.bucketsLen.get(name);
-                    double[] buckets = this.bucketsMap.get(name);
-                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                    featureMap.put(name, String.valueOf(scoreNew));
-                }
-            }
-            item.featureMap = featureMap;
-        }
-
-        // 3 排序
-        Map<String, String> sceneFeatureMap = new HashMap<>(0);
-
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240807.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-
-
-        // 获取VoV预测模型参数
-        // 融合权重
-        double alpha_vov = mergeWeight.getOrDefault("alpha_vov", 1.0);
-
-        double vov_thresh = mergeWeight.getOrDefault("vov_thresh", 0d);
-
-        double view_thresh = mergeWeight.getOrDefault("view_thresh", 1500.0);
-
-        double level50_vov = mergeWeight.getOrDefault("level50_vov", 0.08);
-
-        double level_95_vov = mergeWeight.getOrDefault("level_95_vov", 0.178);
-
-        double beta_vov = mergeWeight.getOrDefault("beta_vov", 10.0);
-
-        List<Double> weightList = new ArrayList<>(3);
-        // weightList.add(mergeWeight.getOrDefault("d2_ago_vov_w", 0.0));
-        // weightList.add(mergeWeight.getOrDefault("d1_ago_vov_w", 0.0));
-        // weightList.add(mergeWeight.getOrDefault("h48_ago_vov_w", 0.0));
-        // weightList.add(mergeWeight.getOrDefault("h24_ago_vov_w", 0.0));
-        weightList.add(mergeWeight.getOrDefault("h3_ago_vov_w", 0.0));
-        weightList.add(mergeWeight.getOrDefault("h2_ago_vov_w", 0.0));
-        weightList.add(mergeWeight.getOrDefault("h1_ago_vov_w", 0.0));
-
-
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
-        Map<String, Map<String, String>> vid2VovFeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vovhour4rank:");
-        List<Video> result = new ArrayList<>();
-//        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
-//        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
-
-        for (RankItem item : items) {
-            double score = 0.0;
-            // 获取其他模型输出score
-            double fmRovOrigin = item.getScoreRov();
-            item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
-            double fmRov = restoreScore(fmRovOrigin);
-            item.getScoresMap().put("fmRov", fmRov);
-
-
-            // 获取VoV输入特征
-            double h1_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("h1_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
-            double h2_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("h2_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
-            double h3_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("h3_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
-            double h24_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("h24_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
-            double h48_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("h48_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
-            double d1_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("d1_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
-            double d2_ago_vov = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("d2_ago_vov", "-2")); // 如果没有时，默认为多少？？ 需要考虑
-
-            double h1_ago_view = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("h1_ago_view", "-2")); // 如果没有时，默认为多少？？ 需要考虑
-
-            item.getScoresMap().put("h1_ago_vov", h1_ago_vov);
-            item.getScoresMap().put("h2_ago_vov", h2_ago_vov);
-            item.getScoresMap().put("h3_ago_vov", h3_ago_vov);
-            item.getScoresMap().put("h24_ago_vov", h24_ago_vov);
-            item.getScoresMap().put("h48_ago_vov", h48_ago_vov);
-            item.getScoresMap().put("d1_ago_vov", d1_ago_vov);
-            item.getScoresMap().put("d2_ago_vov", d2_ago_vov);
-
-            item.getScoresMap().put("h1_ago_view", h1_ago_view);
-            item.getScoresMap().put("alpha_vov", alpha_vov);
-            item.getScoresMap().put("view_thresh", view_thresh);
-            item.getScoresMap().put("vov_thresh", vov_thresh);
-
-
-            List<Double> featureList = new ArrayList<>(3);
-            // featureList.add(d2_ago_vov);
-            // featureList.add(d1_ago_vov);
-            // featureList.add(h48_ago_vov);
-            // featureList.add(h24_ago_vov);
-            featureList.add(h3_ago_vov);
-            featureList.add(h2_ago_vov);
-            featureList.add(h1_ago_vov);
-
-            // todo 线性加权 预测VoV
-
-
-            double vov_p = calculateScore(featureList, weightList, item, vov_thresh, view_thresh, h1_ago_view, level50_vov, level_95_vov, beta_vov);
-
-
-            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault("rate_n", "0"));
-            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            score = fmRov * (1 + hasReturnRovScore) * (1.0 + alpha_vov * vov_p);
-
-
-            item.getScoresMap().put("vov_p", vov_p);
-
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoresMap(item.getScoresMap());
-            video.setAllFeatureMap(item.getAllFeatureMap());
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getVideoFeature())
-                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
-                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
-            }
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
-                video.getMetaFeatureMap().putAll(feature.getUserFeature());
-            }
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-
-        return result;
-    }
-
-
-    private double calculateScore(List<Double> featureList, List<Double> weightList, RankItem rankItem,
-                                  double vov_thresh, double view_thresh, double h1_ago_view, double level50_vov, double level_95_vov,
-                                  double beta_vov) {
-        // 检查 h1_ago_view 条件
-        if (h1_ago_view == -2 || h1_ago_view == -1 || h1_ago_view < view_thresh) {
-            rankItem.getScoresMap().put("origin_vov_p", 0d);
-            return 0;
-        }
-
-        // // 检查 featureList 是否全为 -1
-        // if (featureList.stream().allMatch(f -> f == -1)) {
-        //     rankItem.getScoresMap().put("origin_vov_p", 0d);
-        //     return 0;
-        // }
-
-        // 计算有效特征的总权重和得分
-        double score = 0;
-        List<Integer> validIndices = new ArrayList<>();
-
-        for (int i = 0; i < featureList.size(); i++) {
-            if (featureList.get(i) != -1) {
-                validIndices.add(i);
-            }
-        }
-
-        // 如果没有有效特征，返回 0
-        if (validIndices.isEmpty()) {
-            rankItem.getScoresMap().put("origin_vov_p", 0d);
-            return 0;
-        }
-
-        // 计算得分，动态调整权重
-        for (int index : validIndices) {
-            double weight = weightList.get(index);
-            score += featureList.get(index) * weight;
-        }
-        rankItem.getScoresMap().put("origin_vov_p", score);
-        // 调整vov
-        // 调整vov
-        // if (score < vov_thresh) {
-        //     score = 0;
-        // } else {
-        //     score = 1 / (1 + Math.exp(-1 * beta_vov * (score - level50_vov)));
-        // }
-        return score;
-    }
-
-
-    public void readBucketFile() {
-        InputStream resourceStream = RankStrategy4RegionMergeModelV562.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
-        if (resourceStream != null) {
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
-                Map<String, double[]> bucketsMap = new HashMap<>();
-                Map<String, Double> bucketsLen = new HashMap<>();
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    // 替换空格和换行符，过滤空行
-                    line = line.replace(" ", "").replaceAll("\n", "");
-                    if (!line.isEmpty()) {
-                        String[] rList = line.split("\t");
-                        if (rList.length == 3) {
-                            String key = rList[0];
-                            double value1 = Double.parseDouble(rList[1]);
-                            bucketsLen.put(key, value1);
-                            double[] value2 = Arrays.stream(rList[2].split(","))
-                                    .mapToDouble(Double::valueOf)
-                                    .toArray();
-                            bucketsMap.put(key, value2);
-                        }
-                    }
-                }
-                this.bucketsMap = bucketsMap;
-                this.bucketsLen = bucketsLen;
-            } catch (IOException e) {
-                log.error("something is wrong in parse bucket file:" + e);
-            }
-        } else {
-            log.error("no bucket file");
-        }
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV566.java

@@ -1,653 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV566 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv566:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
-        removeDuplicate(oldRovs);
-        oldRovs = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, oldRovs);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        rovRecallRank.addAll(oldRovs);
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.6);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 240.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5) {
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            } else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-
-            }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV566 job = new RankStrategy4RegionMergeModelV566();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
-
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV569.java

@@ -353,7 +353,8 @@ public class RankStrategy4RegionMergeModelV569 extends RankStrategy4RegionMergeM
     }
 
     public void readBucketFile() {
-        InputStream resourceStream = RankStrategy4RegionMergeModelV999.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        InputStream resourceStream = RankStrategy4RegionMergeModelV569.class.getClassLoader().getResourceAsStream(
+                "20240609_bucket_274.txt");
         if (resourceStream != null) {
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
                 Map<String, double[]> bucketsMap = new HashMap<>();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV6.java

@@ -1,623 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV6 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv6:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private final Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos){
-        Iterator<Video> iterator = videos.iterator();
-        while(iterator.hasNext()){
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())){
-                iterator.remove();
-            }else{
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
-        removeDuplicate(oldRovs);
-        oldRovs = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, oldRovs);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_default.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3_default.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        rovRecallRank.addAll(oldRovs);
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 0.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
-
-
-
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<RankItem> items = model(rovRecallRank, param);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i=0; i<24; ++i){
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items){
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(returns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeight(share2allreturn);
-            List<Double> view2allreturn = getRateData(returns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeight(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeight(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeight(returns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double bb = mergeWeight.getOrDefault("bb", 0.005);
-        double c = mergeWeight.getOrDefault("c", 0.0002);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
-        double h = mergeWeight.getOrDefault("h", 20.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScoreModel = item.getScoreRos();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5){
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-            }else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-
-            }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h){
-                score += (bb * rosScoreModel + f * share2allreturnScore + g * view2allreturnScore);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoreRos(item.getScoreRos());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-    public double calNewVideoScore(Map<String, String> itemBasicMap){
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5){
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-    public double calTrendScore(List<Double> data){
-        double sum = 0.0;
-        int size = data.size();
-        for (int i=0; i<size-4; ++i){
-            sum += data.get(i) - data.get(i+4);
-        }
-        if (sum * 10 > 0.6){
-            sum = 0.6;
-        }else{
-            sum = sum * 10;
-        }
-        if (sum > 0){
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-    public Double calScoreWeight(List<Double> data){
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i=0; i<data.size(); ++i){
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
-        }
-        return down > 1E-8? up / down: 0.0;
-    }
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
-        List<Double> data = new LinkedList<>();
-        for(int i=0; i<ups.size(); ++i){
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)){
-                data.add(0.0);
-            }else{
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key){
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours){
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
-            );
-        }
-        return views;
-    }
-    public List<RankItem> model(List<Video> videos, RankParam param){
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()){
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()){
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null){
-                try{
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {},
-                            userFeatureMap);
-                }catch (Exception e){
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }else{
-                JSONObject obj = new JSONObject();
-                obj.put("name", "user_key_in_model_is_null");
-                obj.put("class", this.CLASS_NAME);
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
-                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
-        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
-        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
-                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
-                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null){
-            for (int i=0; i<videoFeatures.size(); ++i){
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null){
-                    continue;
-                }
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
-                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
-                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
-                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                }catch (Exception e){
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(0) != null){
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-        if (videoRtFeatures != null){
-            int j = 0;
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        JSONObject obj = new JSONObject();
-        obj.put("name", "user_key_in_model_is_not_null");
-        obj.put("class", this.CLASS_NAME);
-        return rovRecallScore;
-    }
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ){
-        }else{
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV650.java

@@ -1,386 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV650 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv650:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private final Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        //-------------------老地域召回-------------------
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        //-------------------节日特殊召回-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        //-------------------基于CB的tag召回-------------------
-        List<Video> v10 = extractAndSort(param, ContentBaseRecallStrategy.PUSH_FORM);
-
-
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v7);
-        this.duplicate(setVideo, v10);
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
-        rovRecallRank.addAll(v10.subList(0, Math.min(mergeWeight.getOrDefault("v10", 6.0).intValue(), v10.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
-        for (RankItem item : items) {
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double score = 0.0;
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > 50) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            } else {
-                score += (f * share2allreturnScore + g * view2allreturnScore) * 0.01;
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-
-        // 2-2: item 实时特征处理
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys2);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-
-        return rankItems;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV569 job = new RankStrategy4RegionMergeModelV569();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
-
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV654.java

@@ -1,168 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV654 extends RankStrategy4RegionMergeModelBasic {
-    @ApolloJsonValue("${rank.score.merge.weightv654:}")
-    private Map<String, Double> mergeWeight;
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        Set<Long> setVideo = new HashSet<>();
-        //-------------------老地域召回-------------------
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        this.duplicate(setVideo, v0);
-        rovRecallRank.addAll(v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------sim相似召回------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        v5 = v5.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v5 = v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size()));
-        rovRecallRank.addAll(v5);
-        setVideo.addAll(v5.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------节日特殊召回-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        v7 = v7.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v7 = v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size()));
-        rovRecallRank.addAll(v7);
-        setVideo.addAll(v7.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------基于cf rov n 召回-------------------
-        List<Video> v8 = extractAndSort(param, CFRovnRecallStrategyV1.PUSH_FORM);
-        v8 = v8.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v8 = v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 6.0).intValue(), v8.size()));
-        rovRecallRank.addAll(v8);
-        setVideo.addAll(v8.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
-        for (RankItem item : items) {
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double score = 0.0;
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > 50) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }else{
-                score += (f * share2allreturnScore + g * view2allreturnScore) * 0.01;
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV655.java

@@ -1,235 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV655 extends RankStrategy4RegionMergeModelBasic {
-    @ApolloJsonValue("${rank.score.merge.weightv655:}")
-    private Map<String, Double> mergeWeight;
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        Set<Long> setVideo = new HashSet<>();
-        //-------------------老地域召回-------------------
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        rovRecallRank.addAll(v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------sim相似召回------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        v5 = v5.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v5 = v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size()));
-        rovRecallRank.addAll(v5);
-        setVideo.addAll(v5.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回v1------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 25.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回v2------------------
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2_sort.PUSH_FORM);
-        v2 = v2.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v2 = v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size()));
-        rovRecallRank.addAll(v2);
-        setVideo.addAll(v2.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回v3------------------
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        v3 = v3.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v3 = v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size()));
-        rovRecallRank.addAll(v3);
-        setVideo.addAll(v3.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回v4------------------
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        v4 = v4.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v4 = v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size()));
-        rovRecallRank.addAll(v4);
-        setVideo.addAll(v4.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------节日特殊召回-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        v7 = v7.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v7 = v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size()));
-        rovRecallRank.addAll(v7);
-        setVideo.addAll(v7.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------基于cf rov n 召回-------------------
-        List<Video> v8 = extractAndSort(param, CFRovnRecallStrategyV1.PUSH_FORM);
-        v8 = v8.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v8 = v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 6.0).intValue(), v8.size()));
-        rovRecallRank.addAll(v8);
-        setVideo.addAll(v8.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        List<String> datehoursRoot = new LinkedList<>();
-        for (int i = 0; i < 24; ++i) {
-            datehoursRoot.add(String.valueOf(i+1));
-        }
-        // 2.1 item特征提取
-        this.getVideoFeatureFromRedis(items);
-
-
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealRootMap = item.getItemRealTimeRootFeature();
-            List<Double> views_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "exp");
-            List<Double> share_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "share");
-            List<Double> return_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "return");
-            List<Double> rov_20240410 = getRateData(return_20240410, views_20240410, 0.0, 0.0);
-            Double rovScore_20240410 = calScoreWeightNoTimeDecay(rov_20240410);
-            List<Double> ros_20240410 = getRateData(return_20240410, share_20240410, 1.0, 10.0);
-            Double rosScore_20240410 = calScoreWeightNoTimeDecay(ros_20240410);
-            item.scoresMap.put("rovScore_20240410", rovScore_20240410);
-            item.scoresMap.put("rosScore_20240410", rosScore_20240410);
-
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 50.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double rovScore_20240410 = item.scoresMap.getOrDefault("rovScore_20240410", 0.0);
-            double rosScore_20240410 = item.scoresMap.getOrDefault("rosScore_20240410", 0.0);
-
-            double score = 0.0;
-            score = a * strScore + b * rosScore + c * preturnsScore +
-                    (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * rosScore_20240410 + g * rovScore_20240410);
-            }else{
-                score += (f * rosScore_20240410 + g * rovScore_20240410) * 0.01;
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV656.java

@@ -1,393 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV656 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv656:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-
-
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------流量池直接送 融合+去重-------------------
-        List<Video> v9 = extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v9);
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size())));
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
-        for (RankItem item : items) {
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double score = 0.0;
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > 50) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }else{
-                score += (f * share2allreturnScore + g * view2allreturnScore) * 0.01;
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-
-        // 2-2: item 实时特征处理
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys2);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-
-        return rankItems;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV656 job = new RankStrategy4RegionMergeModelV656();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
-
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV999.java

@@ -1,425 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.FeatureService;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.MapUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.*;
-import java.util.stream.Collectors;
-
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeModelBasic {
-    @ApolloJsonValue("${rank.score.merge.weightv568:}")
-    private Map<String, Double> mergeWeight;
-
-    @Autowired
-    private FeatureService featureService;
-
-    Map<String, double[]> bucketsMap = new HashMap<>();
-    Map<String, Double> bucketsLen = new HashMap<>();
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        List<Video> rovRecallRank = new ArrayList<>(v0);
-        //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------7天ROVn召回------------------
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        v2 = v2.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v2 = v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 0.0).intValue(), v2.size()));
-        rovRecallRank.addAll(v2);
-        setVideo.addAll(v2.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
-        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
-
-        // k1:视频、k2:表、k3:特征、v:特征值
-        String provinceCn = param.getProvince().replaceAll("省$", "");
-        String headVid = String.valueOf(param.getHeadVid());
-        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
-                String.valueOf(param.getAppType()), provinceCn, headVid);
-        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
-        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
-
-
-        // TODO 2 特征处理
-        Map<String, Double> userFeatureMapDouble = new HashMap<>();
-        String mid = param.getMid();
-        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
-        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
-        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
-        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
-        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
-        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags", new HashMap<>());
-        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags", new HashMap<>());
-        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
-        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
-
-        if (!c1.isEmpty()) {
-            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
-            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
-            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
-            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
-        }
-        if (!c2.isEmpty()) {
-            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
-            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
-            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
-            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
-            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
-            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
-            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
-            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
-        }
-
-        Map<String, String> c34567Map = new HashMap<>(15);
-        List<Tuple2> tmpList0 = Arrays.asList(
-                new Tuple2(c3, "c3_feature"),
-                new Tuple2(c4, "c4_feature"),
-                new Tuple2(c5, "c5_feature"),
-                new Tuple2(c6, "c6_feature"),
-                new Tuple2(c7, "c7_feature")
-        );
-        for (Tuple2 tuple2 : tmpList0) {
-            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                String tags = tuple2.first.getOrDefault(key_time, "");
-                if (!tags.isEmpty()) {
-                    c34567Map.put(tuple2.name + "_" + key_time, tags);
-                }
-            }
-        }
-
-        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
-        List<Tuple2> tmpList1 = Arrays.asList(
-                new Tuple2(c8, "c8_feature"),
-                new Tuple2(c9, "c9_feature")
-        );
-        for (Tuple2 tuple2 : tmpList1) {
-            for (String key_action : Arrays.asList("share", "return")) {
-                String cfListStr = tuple2.first.getOrDefault(key_action, "");
-                if (!cfListStr.isEmpty()) {
-                    Map<String, String[]> cfMap = new HashMap<>();
-                    String[] entries = cfListStr.split(",");
-                    for (String entry : entries) {
-                        String[] rList = entry.split(":");
-                        if (rList.length >= 4) { // 确保分割后有四个元素
-                            String key = rList[0];
-                            String value1 = rList[1];
-                            String value2 = rList[2];
-                            String value3 = rList[3];
-                            String[] strs = {value1, value2, value3};
-                            cfMap.put(key, strs);
-                        }
-                    }
-                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
-                }
-            }
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
-        for (RankItem item : rankItems) {
-            Map<String, Double> featureMap = new HashMap<>();
-            String vid = item.getVideoId() + "";
-            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp", new HashMap<>());
-            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
-            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
-            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share", new HashMap<>());
-            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
-
-            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp", new HashMap<>());
-            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share", new HashMap<>());
-            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return", new HashMap<>());
-            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp", new HashMap<>());
-            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share", new HashMap<>());
-            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return", new HashMap<>());
-            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp", new HashMap<>());
-            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share", new HashMap<>());
-            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return", new HashMap<>());
-
-            List<Tuple4> originData = Arrays.asList(
-                    new Tuple4(b1, b2, b3, "b123"),
-                    new Tuple4(b1, b6, b7, "b167"),
-                    new Tuple4(b8, b9, b10, "b8910"),
-                    new Tuple4(b11, b12, b13, "b111213"),
-                    new Tuple4(b17, b18, b19, "b171819")
-            );
-
-            for (Tuple4 tuple4 : originData) {
-                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
-                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
-                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
-                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
-
-                    double f1 = ExtractorUtils.calDiv(share, exp);
-                    double f2 = ExtractorUtils.calLog(share);
-                    double f3 = ExtractorUtils.calDiv(returns, exp);
-                    double f4 = ExtractorUtils.calLog(returns);
-                    double f5 = f3 * f4;
-
-                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
-                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
-                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
-                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
-                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
-
-                    featureMap.put(key1, f1);
-                    featureMap.put(key2, f2);
-                    featureMap.put(key3, f3);
-                    featureMap.put(key4, f4);
-                    featureMap.put(key5, f5);
-                }
-            }
-
-            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
-            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
-            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
-
-            String title = videoInfo.getOrDefault("title", "");
-            if (!title.isEmpty()) {
-                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
-                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
-                        if (!tags.isEmpty()) {
-                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
-                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
-                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
-                        }
-                    }
-                }
-            }
-
-            if (!vid.isEmpty()) {
-                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
-                    for (String key_action : Arrays.asList("share", "return")) {
-                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
-                        if (cfMap.containsKey(vid)) {
-                            String[] scores = cfMap.get(vid);
-                            Double score1 = Double.parseDouble(scores[0]);
-                            Double score2 = Double.parseDouble(scores[1]);
-                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
-                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
-                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
-                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
-                        }
-                    }
-                }
-            }
-            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
-            if (!d1.isEmpty()) {
-                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
-                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
-                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
-            }
-            item.featureMapDouble = featureMap;
-        }
-
-        // 3 连续值特征分桶
-        readBucketFile();
-        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
-        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
-            String name = entry.getKey();
-            Double score = entry.getValue();
-            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                Double bucketNum = this.bucketsLen.get(name);
-                double[] buckets = this.bucketsMap.get(name);
-                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                userFeatureMap.put(name, String.valueOf(scoreNew));
-            }
-        }
-
-        for (RankItem item : rankItems) {
-            Map<String, String> featureMap = new HashMap<>();
-            Map<String, Double> featureMapDouble = item.featureMapDouble;
-
-            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
-                String name = entry.getKey();
-                Double score = entry.getValue();
-                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
-                    Double bucketNum = this.bucketsLen.get(name);
-                    double[] buckets = this.bucketsMap.get(name);
-                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                    featureMap.put(name, String.valueOf(scoreNew));
-                }
-            }
-            item.featureMap = featureMap;
-        }
-
-        // TODO 3 排序
-        Map<String, String> sceneFeatureMap = new HashMap<>(0);
-
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240609.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
-        List<Video> result = new ArrayList<>();
-        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
-        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
-
-        for (RankItem item : items) {
-            double score = 0.0;
-            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
-                    .getOrDefault(hasReturnRovKey, "0"));
-            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double fmRov = item.getScoreRov();
-            item.getScoresMap().put("fmRov", fmRov);
-            if (chooseFunction == 0){
-                score = fmRov * (1 + hasReturnRovScore);
-            }else if (chooseFunction == 1){
-                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
-            }else {
-                score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
-            }
-
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoresMap(item.getScoresMap());
-            video.setAllFeatureMap(item.getAllFeatureMap());
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getVideoFeature())
-                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
-                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
-            }
-            if (feature != null
-                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
-                video.getMetaFeatureMap().putAll(feature.getUserFeature());
-            }
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-
-        return result;
-    }
-
-    private Map<String, Map<String, String>> extractVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
-    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
-    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
-    public void readBucketFile() {
-        InputStream resourceStream = RankStrategy4RegionMergeModelV999.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
-        if (resourceStream != null) {
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
-                Map<String, double[]> bucketsMap = new HashMap<>();
-                Map<String, Double> bucketsLen = new HashMap<>();
-                String line;
-                while ((line = reader.readLine()) != null) {
-                    // 替换空格和换行符，过滤空行
-                    line = line.replace(" ", "").replaceAll("\n", "");
-                    if (!line.isEmpty()) {
-                        String[] rList = line.split("\t");
-                        if (rList.length == 3) {
-                            String key = rList[0];
-                            double value1 = Double.parseDouble(rList[1]);
-                            bucketsLen.put(key, value1);
-                            double[] value2 = Arrays.stream(rList[2].split(","))
-                                    .mapToDouble(Double::valueOf)
-                                    .toArray();
-                            bucketsMap.put(key, value2);
-                        }
-                    }
-                }
-                this.bucketsMap = bucketsMap;
-                this.bucketsLen = bucketsLen;
-            } catch (IOException e) {
-                log.error("something is wrong in parse bucket file:" + e);
-            }
-        } else {
-            log.error("no bucket file");
-        }
-
-    }
-
-    static class Tuple4 {
-        public Map<String, String> first;
-        public Map<String, String> second;
-        public Map<String, String> third;
-
-        public String name;
-
-        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name) {
-            this.first = first;
-            this.second = second;
-            this.third = third;
-            this.name = name;
-        }
-
-    }
-
-    static class Tuple2 {
-        public Map<String, String> first;
-
-        public String name;
-
-        public Tuple2(Map<String, String> first, String name) {
-            this.first = first;
-            this.name = name;
-        }
-
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4ShareDeepAndWidth.java

@@ -1,298 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4ShareDeepAndWidth extends RankStrategy4RegionMergeModelBasic {
-    @ApolloJsonValue("${rank.score.merge.weightv645:}")
-    private Map<String, Double> mergeWeight;
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        Set<Long> setVideo = new HashSet<>();
-        //-------------------老地域召回-------------------
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        removeDuplicate(oldRovs);
-        int sizeReturn = param.getSize();
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        rovRecallRank.addAll(v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------sim相似召回------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        v5 = v5.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v5 = v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size()));
-        rovRecallRank.addAll(v5);
-        setVideo.addAll(v5.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------return相似召回------------------
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------节日特殊召回-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        v7 = v7.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v7 = v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size()));
-        rovRecallRank.addAll(v7);
-        setVideo.addAll(v7.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------基于分享宽度 召回-------------------
-        List<Video> v8 = extractAndSort(param, ShareWidthRecallStrategy.PUSH_FORM);
-        v8 = v8.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v8 = v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 5.0).intValue(), v8.size()));
-        rovRecallRank.addAll(v8);
-        setVideo.addAll(v8.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------基于分享深度 召回-------------------
-        List<Video> v9 = extractAndSort(param, ShareDeepRecallStrategy.PUSH_FORM);
-        v9 = v9.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v9 = v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size()));
-        rovRecallRank.addAll(v9);
-        setVideo.addAll(v9.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-
-            Map<String, Double> shareUvList1h = itemRealMap.getOrDefault("share_uv_list_1h", new HashMap<>());
-            Double sumShareCount = shareUvList1h.values().stream().mapToDouble(Double::doubleValue).sum();
-            item.scoresMap.put("sumShareCount", sumShareCount);
-
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 1.0);
-        double b = mergeWeight.getOrDefault("b", 1.0);
-
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
-        for (RankItem item : items) {
-            Map<String, String> featureMap = item.getFeatureMap();
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double score = 0.0;
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > 50) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            } else {
-                score += (f * share2allreturnScore + g * view2allreturnScore) * 0.01;
-            }
-            double sumShareCount = item.scoresMap.getOrDefault("sumShareCount", 0.0);
-            double daySharedepthMaxAvg = Double.parseDouble(featureMap.getOrDefault("i_1day_sharedepth_max_avg", "0.0"));
-            double daySharewidthMaxAvg = Double.parseDouble(featureMap.getOrDefault("i_1day_sharewidth_max_avg", "0.0"));
-
-            if (sumShareCount > 30) {
-                score += a * daySharedepthMaxAvg + b * daySharewidthMaxAvg;
-                item.getScoresMap().put("dayShareDepthMaxAvg", daySharedepthMaxAvg);
-                item.getScoresMap().put("dayShareWidthMaxAvg", daySharewidthMaxAvg);
-            }
-
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                if (j > videoRtFeatures.size() - 1) {
-                    break;
-                }
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-
-                String k1 = "sharedepth_max_avg_list_1day";
-                String k2 = "sharewidth_max_avg_list_1day";
-                if (vfMapNew.containsKey(k1)) {
-                    Double shareDepth = vfMapNew.get(k1).getOrDefault(rtFeaPart1day, 0.0);
-                    item.getFeatureMap().put("i_1day_sharedepth_max_avg", shareDepth + "");
-                }
-                if (vfMapNew.containsKey(k2)) {
-                    Double shareWidth = vfMapNew.get(k2).getOrDefault(rtFeaPart1day, 0.0);
-                    item.getFeatureMap().put("i_1day_sharewidth_max_avg", shareWidth + "");
-                }
-
-            }
-            for (RankItem item : rankItems) {
-                if (j > videoRtFeatures.size() - 1) {
-                    break;
-                }
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-
-        return rankItems;
-    }
-
-    public static void main(String[] args) {
-    }
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategyFlowThompsonModel.java

@@ -1,92 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-
-/**
- * @author 孙铭泽
- * @desc 对流量池增加排序Thompson sampling 策略
- * @date 2024-01-17
- */
-@Service
-@Slf4j
-public class RankStrategyFlowThompsonModel extends RankService {
-
-    public List<Video> mergeAndRankFlowPoolRecall(RankParam param) {
-        if (param.getAppType() == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
-                || param.getAppType() == AppTypeEnum.ZUI_JING_QI.getCode()) {
-            if (param.getAbCode().equals("60054")
-                    || param.getAbCode().equals("60068")
-                    || param.getAbCode().equals("60081")
-                    || param.getAbCode().equals("60084")) {
-                return extractAndSort(param, FlowPoolConstants.QUICK_PUSH_FORM);
-            } else {
-                return Collections.emptyList();
-            }
-        } else {
-            List<Video> quickFlowPoolVideos = sortFlowPoolByThompson(param, FlowPoolConstants.QUICK_PUSH_FORM);
-            if (CollectionUtils.isNotEmpty(quickFlowPoolVideos)) {
-                return quickFlowPoolVideos;
-            } else {
-                return sortFlowPoolByThompson(param, FlowPoolConstants.PUSH_FORM);
-            }
-        }
-    }
-
-    public List<Video> sortFlowPoolByThompson(RankParam param, String pushFrom) {
-
-        //初始化 userid
-        UserFeature userFeature = new UserFeature();
-        userFeature.setMid(param.getMid());
-
-        // 初始化RankItem
-        Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
-                .filter(d -> d.getPushFrom().equals(pushFrom))
-                .findFirst();
-        if (!data.isPresent()){
-            return Collections.emptyList();
-        }
-        List<Video> videoList = data.get().getVideos();
-        if (videoList == null) {
-            return Collections.emptyList();
-        }
-        List<RankItem> rankItems = new ArrayList<>();
-        for (int i = 0; i < videoList.size(); i++) {
-            RankItem rankItem = new RankItem(videoList.get(i));
-            rankItems.add(rankItem);
-        }
-
-        // 初始化上下文参数
-        ScoreParam scoreParam = convert(param);
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.FLOWPOOL_CONF)
-                .scoring(scoreParam, userFeature, rankItems);
-
-        if (rovRecallScore == null) {
-            return Collections.emptyList();
-        }
-
-        return CommonCollectionUtils.toList(rovRecallScore, i -> {
-            // hard code 将排序分数 赋值给video的sortScore
-            Video v = i.getVideo();
-            v.setSortScore(i.getScore());
-            return v;
-        });
-    }
-
-}