feat:解决冲突

recommend-server-service/pom.xml

@@ -73,7 +73,29 @@
             <artifactId>fastutil</artifactId>
             <version>7.0.12</version>
         </dependency>
-
+        <dependency>
+            <groupId>org.xm</groupId>
+            <artifactId>similarity</artifactId>
+            <version>1.1</version>
+            <exclusions>
+                <!--                <exclusion>-->
+                <!--                    <artifactId>logback-access</artifactId>-->
+                <!--                    <groupId>ch.qos.logback</groupId>-->
+                <!--                </exclusion>-->
+                <!--                <exclusion>-->
+                <!--                    <artifactId>logback-classic</artifactId>-->
+                <!--                    <groupId>ch.qos.logback</groupId>-->
+                <!--                </exclusion>-->
+                <!--                <exclusion>-->
+                <!--                    <artifactId>logback-core</artifactId>-->
+                <!--                    <groupId>ch.qos.logback</groupId>-->
+                <!--                </exclusion>-->
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -158,7 +180,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.1.16</version>
+            <version>1.1.20</version>
         </dependency>
         <dependency>
             <groupId>com.google.protobuf</groupId>
@@ -221,6 +243,28 @@
             <artifactId>abtest-client</artifactId>
             <version>1.0.0</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>2.12.15</version>
+        </dependency>
+        <dependency>
+            <groupId>ml.dmlc</groupId>
+            <artifactId>xgboost4j-spark_2.12</artifactId>
+            <version>1.7.6</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>scala-library</artifactId>
+                    <groupId>org.scala-lang</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-mllib_2.12</artifactId>
+            <version>3.3.1</version>
+        </dependency>
     </dependencies>
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -2,8 +2,9 @@ package com.tzld.piaoquan.recommend.server;
 
 // import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
 
-//import com.tzld.piaoquan.abtest.client.ABTestClient;
+import com.tzld.piaoquan.abtest.client.ABTestClient;
 import com.tzld.piaoquan.recommend.feature.client.FeatureClient;
+import com.tzld.piaoquan.recommend.feature.client.FeatureV2Client;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
@@ -22,12 +23,13 @@ import org.springframework.scheduling.annotation.EnableScheduling;
         RedisReactiveAutoConfiguration.class
 })
 @ComponentScan({
+        "com.tzld.piaoquan.recommend.server.util",
+        "com.tzld.piaoquan.recommend.server.config",
         "com.tzld.piaoquan.recommend.server.service",
         "com.tzld.piaoquan.recommend.server.implement",
         "com.tzld.piaoquan.recommend.server.framework.utils",
         "com.tzld.piaoquan.recommend.server.grpcservice",
         "com.tzld.piaoquan.recommend.server.remote",
-        "com.tzld.piaoquan.recommend.server.config",
         "com.tzld.piaoquan.recommend.server.web",
         "com.tzld.piaoquan.recommend.server.xxl",
 })
@@ -45,10 +47,15 @@ public class Application {
         return new FeatureClient();
     }
 
-//    @Bean
-//    public ABTestClient abTestClient() {
-//        return new ABTestClient();
-//    }
+    @Bean
+    public FeatureV2Client featureV2Client() {
+        return new FeatureV2Client();
+    }
+
+    @Bean
+    public ABTestClient abTestClient() {
+        return new ABTestClient();
+    }
 
     @Bean
     public ProtobufHttpMessageConverter protobufHttpMessageConverter() {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -17,6 +17,9 @@ public class RankItem implements Comparable<RankItem> {
 
     // featureMap中保存所有的特征
     public Map<String, String> featureMap = new HashMap<>();
+    // 所有特征，包括视频、用户等等
+    public Map<String, String> allFeatureMap = new HashMap<>();
+    public Map<String, Double> featureMapDouble = new HashMap<>();
     public String id;
     public Map<String, Double> scoresMap = new HashMap<>();
     public Map<String, String> itemBasicFeature = new HashMap<>();
@@ -27,6 +30,7 @@ public class RankItem implements Comparable<RankItem> {
     private Video video;
     private double scoreRos; // 记录ros的score
     private double scoreStr; // 记录str的score
+    private double scoreRov; // 记录rov的score
 
     // 记录Item侧用到的特征
     private ItemFeature itemFeature;
@@ -51,10 +55,10 @@ public class RankItem implements Comparable<RankItem> {
     // 排序侧信息
     private Map<String, Double> rankerScore = Maps.newHashMap();
     private Map<String, Integer> rankerIndex = Maps.newHashMap();
-    public RankItem(){
 
-    }
+    public RankItem() {
 
+    }
 
 
     public RankItem(Video video) {
@@ -146,7 +150,4 @@ public class RankItem implements Comparable<RankItem> {
     }
 
 
-
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/SparkConfig.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.config;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+
+@Component
+public class SparkConfig {
+
+    @PostConstruct
+    public void init() {
+        SparkConf sparkConf = new SparkConf()
+                .setMaster("local")
+                .setAppName("XGBoostPredict");
+        JavaSparkContext jsc = new JavaSparkContext(sparkConf);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/Video.java

@@ -39,5 +39,9 @@ public class Video {
     public Map<String, Double> scoresMap = new HashMap<>();
     public Map<String, List<String>> pushFromIndex = new HashMap<>();
     public Map<String, Integer> pushFromRank = new HashMap<>();
+    // 处理后，传给模型的特征
+    public Map<String, String> allFeatureMap = new HashMap<>();
+    // 原始特征
+    public Map<String, Map<String, String>> metaFeatureMap = new HashMap<>();
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/remote/FeatureV2RemoteService.java

@@ -0,0 +1,34 @@
+package com.tzld.piaoquan.recommend.server.remote;
+
+import com.tzld.piaoquan.recommend.feature.client.FeatureV2Client;
+import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class FeatureV2RemoteService {
+
+    @Autowired
+    private FeatureV2Client client;
+
+    public Map<String, String> getFeature(List<FeatureKeyProto> protos) {
+        if (CollectionUtils.isEmpty(protos)) {
+            return Collections.emptyMap();
+        }
+        Map<String, String> result = client.multiGetFeature(protos);
+        return result;
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/FeatureService.java

@@ -0,0 +1,179 @@
+package com.tzld.piaoquan.recommend.server.service;
+
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import com.tzld.piaoquan.recommend.server.remote.FeatureV2RemoteService;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+
+/**
+ * @author dyp
+ */
+@Service
+@Slf4j
+public class FeatureService {
+
+    @Autowired
+    private FeatureV2RemoteService remoteService;
+
+    /**
+     * @return k1:视频、k2:表、k3:特征、v:特征值
+     */
+    public Feature getFeature(String mid, List<String> vidList, String appType,
+                              String province, String headVid) {
+
+
+        List<FeatureKeyProto> protos = new ArrayList<>();
+
+        for (String vid : vidList) {
+            // TODO 补充其他特征
+            // vid
+            // protos.add(genWithVid("alg_vid_feature_all_exp", vid));
+            protos.add(genWithVid("alg_vid_feature_all_exp_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_all_share", vid));
+            protos.add(genWithVid("alg_vid_feature_all_return", vid));
+            // protos.add(genWithVid("alg_vid_feature_exp2share", vid));
+            protos.add(genWithVid("alg_vid_feature_exp2share_v2", vid));
+            protos.add(genWithVid("alg_vid_feature_share2return", vid));
+            // protos.add(genWithVid("alg_vid_feature_feed_noflow_exp", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_exp_v2", vid));
+            // protos.add(genWithVid("alg_vid_feature_feed_noflow_root_share", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_root_share_v2", vid));
+            // protos.add(genWithVid("alg_vid_feature_feed_noflow_root_return", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_noflow_root_return_v2", vid));
+            // protos.add(genWithVid("alg_vid_feature_feed_flow_exp", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_exp_v2", vid));
+            // protos.add(genWithVid("alg_vid_feature_feed_flow_root_share", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_root_share_v2", vid));
+            // protos.add(genWithVid("alg_vid_feature_feed_flow_root_return", vid));
+            protos.add(genWithVid("alg_vid_feature_feed_flow_root_return_v2", vid));
+
+            protos.add(genWithVid("alg_vid_feature_basic_info", vid));
+            // vid + apptype
+
+            // vid + province
+            // protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_exp", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_exp_v2", vid, province));
+            // protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_share", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_share_v2", vid, province));
+            // protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_return", vid, province));
+            protos.add(genWithVidAndProvince("alg_vid_feature_feed_province_root_return_v2", vid, province));
+
+            // vid + headvid
+            protos.add(genWithVidAndHeadVid("alg_recsys_feature_cf_i2i_new", vid, headVid));
+            protos.add(genWithVidAndHeadVid("alg_recsys_feature_cf_i2i_new_v2", vid, headVid));
+        }
+
+
+        // user
+        protos.add(genWithMid("alg_mid_feature_play", mid));
+        protos.add(genWithMid("alg_mid_feature_share_and_return", mid));
+        protos.add(genWithMid("alg_mid_feature_play_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_return_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_share_tags", mid));
+        // protos.add(genWithMid("alg_mid_feature_feed_exp_share_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_feed_exp_share_tags_v2", mid));
+        // protos.add(genWithMid("alg_mid_feature_feed_exp_return_tags", mid));
+        protos.add(genWithMid("alg_mid_feature_feed_exp_return_tags_v2", mid));
+        protos.add(genWithMid("alg_mid_feature_sharecf", mid));
+        protos.add(genWithMid("alg_mid_feature_returncf", mid));
+
+
+        Map<String, String> result = remoteService.getFeature(protos);
+
+        Feature feature = new Feature();
+
+        result.entrySet().forEach(e -> {
+
+            String[] uk = StringUtils.split(e.getKey(), ":");
+            String prefix = uk[0];
+            String table = uk[1];
+            Map<String, String> colMap = JSONUtils.fromJson(e.getValue(), new TypeToken<Map<String, String>>() {
+            }, Collections.emptyMap());
+            String featureStr = colMap.get("feature");
+
+            switch (prefix) {
+                case "v":
+                    String vid = uk[2];
+                    Map<String, Map<String, String>> tableFeatureMap = feature.getVideoFeature().getOrDefault(vid, new HashMap<>());
+                    tableFeatureMap.put(table, JSONUtils.fromJson(featureStr, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getVideoFeature().put(vid, tableFeatureMap);
+                    break;
+                case "u":
+                    feature.getUserFeature().put(table, JSONUtils.fromJson(featureStr, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    break;
+                default:
+                    break;
+            }
+
+        });
+
+
+        return feature;
+    }
+
+    private final String videoUkFormat = "v:%s:%s";
+    private final String userUkFormat = "u:%s";
+
+    private FeatureKeyProto genWithVid(String table, String vid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(videoUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithVidAndAppType(String table, String vid, String appType) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(videoUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .putFieldValue("apptype", appType)
+                .build();
+    }
+
+    private FeatureKeyProto genWithVidAndProvince(String table, String vid, String province) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(videoUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .putFieldValue("province", province)
+                .build();
+    }
+
+    private FeatureKeyProto genWithVidAndHeadVid(String table, String vid, String headVid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(videoUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .putFieldValue("headVid", headVid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithMid(String table, String mid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(userUkFormat, table))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .build();
+    }
+
+    @Data
+    public static class Feature {
+        // k1:视频、k2:表、k3:特征、v:特征值
+        private Map<String, Map<String, Map<String, String>>> videoFeature = new HashMap<>();
+
+        // k1:mid、k2:特征、v:特征值
+        private Map<String, Map<String, String>> userFeature = new HashMap<>();
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -173,6 +173,7 @@ public class RecommendService {
                 Map<String, String> map = new HashMap<>();
 
                 map.put("traceId", String.valueOf(TraceUtils.currentTraceId()));
+                map.put("recommendTraceId", Strings.nullToEmpty(request.getRecommendTraceId()));
 
                 // TODO user
                 map.put("sessionId", request.getSessionId());
@@ -202,6 +203,8 @@ public class RecommendService {
                 map.put("scoreStr", String.valueOf(v.getScoreStr()));
                 map.put("score", String.valueOf(v.getScore()));
                 map.put("scoresMap", JSONUtils.toJson(v.getScoresMap()));
+                map.put("allFeatureMap", JSONUtils.toJson(v.getAllFeatureMap()));
+                map.put("metaFeatureMap", JSONUtils.toJson(v.getMetaFeatureMap()));
 
                 map.put("pushFromRank", JSONUtils.toJson(v.getPushFromRank()));
                 map.put("abExpCode", JSONUtils.toJson(param.getAbExpCodes()));
@@ -599,6 +602,7 @@ public class RecommendService {
         rankParam.setAbExpCodes(param.getAbExpCodes());
         rankParam.setExpIdMap(param.getExpIdMap());
         rankParam.setCategoryId(param.getCategoryId());
+        rankParam.setHeadVid(param.getVideoId());
         return rankParam;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -127,6 +127,11 @@ public abstract class AbstractFilterService {
                 break;
         }
 
+        // VOV过滤实验
+        if (param.getAbExpCodes().contains("689")) {
+            strategies.add(ServiceBeanFactory.getBean(VovLowerStrategy.class));
+        }
+
         return strategies;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/AppletVideoStatusStrategy.java

@@ -89,7 +89,8 @@ public class AppletVideoStatusStrategy implements FilterStrategy {
                 // 数据库有数据
                 for (VideoAppTypeStatus videoAppTypeStatus : videoAppTypeStatusList) {
                     activeStatusMap.put(videoAppTypeStatus.getVideoId(), videoAppTypeStatus.getVideoStatus());
-                    updateRedisStatus.put(String.format(videoAppTypeStatusKeyFormat, appType, videoAppTypeStatus.getVideoId()), "1");
+                    updateRedisStatus.put(String.format(videoAppTypeStatusKeyFormat, appType,
+                            videoAppTypeStatus.getVideoId()), videoAppTypeStatus.getVideoStatus().toString());
                     cacheMissVideoIdSet.remove(videoAppTypeStatus.getVideoId());
                 }
             }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/BlacklistContainer.java

@@ -155,7 +155,7 @@ public class BlacklistContainer {
     }
 
     public void refreshVideoTagCache() {
-        LOG.info("同步本地标签ID与视频列表的缓存任务开始");
+        // LOG.info("同步本地标签ID与视频列表的缓存任务开始");
         Map<Long, Set<Long>> tmpMap = new ConcurrentHashMap<>();
 
         if (MapUtils.isNotEmpty(tagFilterConfigMap)) {
@@ -179,13 +179,13 @@ public class BlacklistContainer {
             for (Long tagId : tagIdSet) {
                 List<WxVideoTagRel> wxVideoTagRels = wxVideoTagRelRepository.findAllByTagId(tagId);
                 Set<Long> videoIdSet = wxVideoTagRels.stream().map(WxVideoTagRel::getVideoId).collect(Collectors.toSet());
-                LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, videoIdSet.size());
+                // LOG.info("同步本地标签ID与视频列表缓存任务 -- tagId: {}, videoIdSize: {}", tagId, videoIdSet.size());
                 tmpMap.put(tagId, videoIdSet);
             }
         }
         videoTagCache = tmpMap;
 
-        LOG.info("同步本地标签ID与视频列表的缓存任务结束");
+        // LOG.info("同步本地标签ID与视频列表的缓存任务结束");
     }
 
     public List<Long> filterUnsafeVideoByUser(List<Long> videoIds, String uid, Long hotSceneType, String cityCode, String clientIP, String mid, String usedScene, Integer appType) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/strategy/VovLowerStrategy.java

@@ -0,0 +1,105 @@
+package com.tzld.piaoquan.recommend.server.service.filter.strategy;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Component
+public class VovLowerStrategy implements FilterStrategy {
+
+    @ApolloJsonValue("${vov.filter.condition:{}}")
+    private Map<String, Double> vovFilterCondition;
+
+    @Autowired
+    @Qualifier("longVideoRedisTemplate")
+    private RedisTemplate<String, String> longVideoRedisTemplate;
+
+    private static final String KEY_FORMAT = "redis:vid_vov_daily4filter:%s";
+
+    @Override
+    public List<Long> filter(FilterParam param) {
+
+        List<Long> result = new ArrayList<>();
+
+        List<Long> videoIds = param.getVideoIds();
+        if (CollectionUtils.isEmpty(videoIds)) {
+            log.info("VOV过滤 -- videoIds为空，跳过");
+            return result;
+        }
+
+        List<String> redisKeys = videoIds.stream()
+                .map(r -> String.format(KEY_FORMAT, r))
+                .collect(Collectors.toList());
+        List<String> vovInfos = longVideoRedisTemplate.opsForValue().multiGet(redisKeys);
+        if (CollectionUtils.isEmpty(vovInfos) || vovInfos.size() != videoIds.size()) {
+            log.info("VOV过滤 -- 获取到的视频VOV信息为空或长度与videoIds长度不一致，跳过: {}, {}", videoIds.size(), vovInfos.size());
+            return result;
+        }
+
+        List<Long> removeIds = new ArrayList<>();
+
+        for (int i = 0; i < videoIds.size(); i++) {
+            Long videoId = videoIds.get(i);
+            String vovInfo = vovInfos.get(i);
+            if (isFilter(vovInfo)) {
+                removeIds.add(videoId);
+                continue;
+            }
+
+            result.add(videoId);
+        }
+
+        log.info("VOV -- 本次过滤的ID列表为: {}", removeIds);
+
+        return result;
+    }
+
+    private boolean isFilter(String vovInfo) {
+        try {
+            Map<String, Double> vovInfoMap = JSON.parseObject(vovInfo, new TypeReference<Map<String, Double>>() {
+            });
+
+            double t0ViewPvCondition = vovFilterCondition.getOrDefault("t_0_view_pv", 1000d);
+            double t1ViewPvCondition = vovFilterCondition.getOrDefault("t_1_view_pv", 1000d);
+            double t2ViewPvCondition = vovFilterCondition.getOrDefault("t_2_view_pv", 1000d);
+
+
+            double t2TodayDistViewPv = vovInfoMap.getOrDefault("t_2_today_dist_view_pv", 0d);
+            double t1TodayDistViewPv = vovInfoMap.getOrDefault("t_1_today_dist_view_pv", 0d);
+            double t0TodayDistViewPv = vovInfoMap.getOrDefault("t_0_today_dist_view_pv", 0d);
+
+
+            double t0VovCondition = vovFilterCondition.getOrDefault("t_0_vov", 0d);
+            double t1VovCondition = vovFilterCondition.getOrDefault("t_1_vov", 0d);
+            double t2VovCondition = vovFilterCondition.getOrDefault("t_1_vov", 0d);
+
+            double t0AllVov = vovInfoMap.getOrDefault("t_0_all_vov", 0d);
+            double t1AllVov = vovInfoMap.getOrDefault("t_1_all_vov", 0d);
+            double t2AllVov = vovInfoMap.getOrDefault("t_2_all_vov", 0d);
+
+            boolean viewResult = (t2TodayDistViewPv > t2ViewPvCondition) && (t1TodayDistViewPv > t1ViewPvCondition) && (t0TodayDistViewPv > t0ViewPvCondition);
+
+            boolean vovResult = (t0AllVov < t0VovCondition) && (t1AllVov < t1VovCondition) && (t2AllVov < t2VovCondition);
+
+            return viewResult && vovResult;
+        } catch (Exception e) {
+            log.info("VOV过滤 -- 异常: ", e);
+        }
+        return false;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankParam.java

@@ -25,6 +25,7 @@ public class RankParam {
     private String city;
     private MachineInfo machineInfo;
     private Set<String> abExpCodes;
+    private Long headVid=0L;
 
     // 层 - 实验
     private Map<String, String> expIdMap;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -20,8 +20,6 @@ public class RankRouter {
     @Autowired
     private RankStrategy4RegionMergeModelV536 rankStrategy4RegionMergeModelV536;
     @Autowired
-    private RankStrategy4RegionMergeModelV551 rankStrategy4RegionMergeModelV3;
-    @Autowired
     private RankStrategy4RegionMergeModelV546 rankStrategy4RegionMergeModelV546;
     @Autowired
     private RankStrategy4RegionMergeModelV547 rankStrategy4RegionMergeModelV547;
@@ -32,8 +30,6 @@ public class RankRouter {
     @Autowired
     private RankStrategy4RegionMergeModelV552 rankStrategy4RegionMergeModelV552;
     @Autowired
-    private RankStrategy4RegionMergeModelV553 rankStrategy4RegionMergeModelV553;
-    @Autowired
     private RankStrategy4RegionMergeModelV561 rankStrategy4RegionMergeModelV561;
     @Autowired
     private RankStrategy4RegionMergeModelV562 rankStrategy4RegionMergeModelV562;
@@ -48,7 +44,7 @@ public class RankRouter {
     @Autowired
     private RankStrategy4RegionMergeModelV567 rankStrategy4RegionMergeModelV567;
     @Autowired
-    private RankStrategy4RegionMergeModelV568 rankStrategy4RegionMergeModelV568;
+    private RankStrategy4RegionMergeModelV999 rankStrategy4RegionMergeModelV999;
     @Autowired
     private RankStrategy4RegionMergeModelV569 rankStrategy4RegionMergeModelV569;
     @Autowired
@@ -58,6 +54,8 @@ public class RankRouter {
     @Autowired
     private RankStrategy4RegionMergeModelV655 rankStrategy4RegionMergeModelV655;
     @Autowired
+    private RankStrategy4RegionMergeModelV656 rankStrategy4RegionMergeModelV656;
+    @Autowired
     private FestivalStrategy4RankModel festivalStrategy4RankModel;
 
     @Autowired
@@ -73,11 +71,12 @@ public class RankRouter {
             return rankService.rank(param);
         }
         switch (abCode) {
-            case "60097":
-            case "60098": // 533
-            case "60111": // 561
+            case "60105": // 551
+                return rankStrategy4RegionMergeModelV551.rank(param);
+            case "60106": // 552
+                return rankStrategy4RegionMergeModelV552.rank(param);
             case "60112": // 562
-                return rankStrategy4Density.rank(param);
+                return rankStrategy4RegionMergeModelV562.rank(param);
             case "60101":
                 return rankStrategy4RankModel.rank(param);
             case "60113": // 563
@@ -91,7 +90,7 @@ public class RankRouter {
             case "60117": // 567
                 return rankStrategy4RegionMergeModelV567.rank(param);
             case "60118": // 568
-                return rankStrategy4RegionMergeModelV568.rank(param);
+                return rankStrategy4RegionMergeModelV999.rank(param);
             case "60119": // 569
                 return rankStrategy4RegionMergeModelV569.rank(param);
             case "60120": // 576
@@ -100,20 +99,12 @@ public class RankRouter {
                 return rankStrategy4RegionMergeModelV536.rank(param);
             case "60122": // 537
                 return rankStrategy4RegionMergeModelV546.rank(param);
-            case "60123": // 541
-                return rankStrategy4RegionMergeModelV3.rank(param);
             case "60124": // 546
                 return rankStrategy4RegionMergeModelV546.rank(param);
             case "60125": // 547
                 return rankStrategy4RegionMergeModelV547.rank(param);
             case "60126": // 548
                 return rankStrategy4RegionMergeModelV548.rank(param);
-            case "60105": // 551
-                return rankStrategy4RegionMergeModelV551.rank(param);
-            case "60106": // 552
-                return rankStrategy4RegionMergeModelV552.rank(param);
-            case "60107": // 553
-                return rankStrategy4RegionMergeModelV553.rank(param);
             case "60130":
             case "60131":
             case "60132":
@@ -126,6 +117,8 @@ public class RankRouter {
                 return rankStrategy4RegionMergeModelV654.rank(param);
             case "60655": // 655
                 return rankStrategy4RegionMergeModelV655.rank(param);
+            case "60656": // 656
+                return rankStrategy4RegionMergeModelV656.rank(param);
             default:
                 break;
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -209,7 +209,6 @@ public class RankService {
                 || param.getAbCode().equals("60097")
                 || param.getAbCode().equals("60098")
                 || param.getAbCode().equals("60111")
-                || param.getAbCode().equals("60112")
                 || param.getAbCode().equals("60103")
                 || param.getAbCode().equals("60104")
                 || param.getAbCode().equals("60110")
@@ -227,9 +226,6 @@ public class RankService {
             // merge sim recall 和 return recall
             rovRecallRank.addAll(extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM));
             rovRecallRank.addAll(extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM));
-            if (param.getAbCode().equals("60112")) {
-                rovRecallRank.addAll(extractAndSort(param, RegionRealtimeRecallStrategyV5Hand.PUSH_FORM));
-            }
             if (param.getAbCode().equals("60150")) {
                 rovRecallRank.addAll(extractAndSort(param, ShareDeepRecallStrategy.PUSH_FORM));
                 rovRecallRank.addAll(extractAndSort(param, ShareWidthRecallStrategy.PUSH_FORM));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/extractor/ExtractorUtils.java

@@ -5,9 +5,72 @@ import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-
+import org.xm.Similarity;
 public class ExtractorUtils {
 
+    public static double sigmoid(double x) {
+        return 1.0 / (1.0 + Math.exp(-x));
+    }
+    public static int findInsertPosition(double[] sortedArray, double target) {
+        int low = 0;
+        int high = sortedArray.length - 1;
+
+        while (low <= high) {
+            int mid = low + (high - low) / 2;
+            double midValue = sortedArray[mid];
+
+            if (midValue < target) {
+                low = mid + 1;
+            } else if (midValue > target) {
+                high = mid - 1;
+            } else {
+                // 找到相等的值，尝试在右侧寻找插入点
+                while (mid < sortedArray.length - 1 && sortedArray[mid + 1] == target) {
+                    mid++;
+                }
+                return mid + 1; // 返回当前mid的下一个位置作为插入点
+            }
+        }
+
+        return low; // 返回low作为插入点
+    }
+    public static Double[] funcC34567ForTags(String tags, String title) {
+        String[] tagsList = tags.split(",");
+        int d1 = 0;
+        List<String> d2 = new ArrayList<>();
+        double d3 = 0.0;
+        double d4 = 0.0;
+
+        for (String tag : tagsList) {
+            if (title.contains(tag)) {
+                d1++;
+                d2.add(tag);
+            }
+            double score = Similarity.conceptSimilarity(tag, title);
+            if (score > d3) {
+                d3 = score;
+            }
+            d4 += score;
+        }
+
+        d4 = (tagsList.length > 0) ? d4 / tagsList.length : d4;
+
+        // 使用数组来返回多个值
+        Double[] result = {(double) d1, d3, d4};
+        return result;
+    }
+    public static Double calDiv(double a, double b){
+        if (a == 0 || b == 0){
+            return 0D;
+        }
+        return a / b;
+    }
+    public static Double calLog(double a){
+        if (a <= 0){
+            return 0D;
+        }
+        return Math.log(a + 1.0);
+    }
     public static Double division(String s1, String s2, Map<String, String> maps){
         double rate = 0.0;
         if (maps.containsKey(s1) && maps.containsKey(s2)){

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelBasic.java

@@ -36,7 +36,7 @@ public class RankStrategy4RegionMergeModelBasic extends RankService {
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
     private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
+    String CLASS_NAME = this.getClass().getSimpleName();
 
     public void duplicate(Set<Long> setVideo, List<Video> videos) {
         Iterator<Video> iterator = videos.iterator();
@@ -298,6 +298,62 @@ public class RankStrategy4RegionMergeModelBasic extends RankService {
         }
     }
 
+    public Map<String, Map<String, String>> getVideoRedisFeature(List<String> vids, String redisKeyPrefix){
+        List<String> keys = vids.stream().map(r -> redisKeyPrefix + r).collect(Collectors.toList());
+        List<String> key2Values = this.redisTemplate.opsForValue().multiGet(keys);
+        Map<String, Map<String, String>> result = new HashMap<>(vids.size());
+        if (key2Values != null) {
+            int j = 0;
+            for (String vid : vids) {
+                String vF = key2Values.get(j);
+                ++j;
+                if (vF == null) {
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                try {
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                } catch (Exception e) {
+                    log.error(String.format("parse video json is wrong on redisKeyPrefix in {} with {}", this.CLASS_NAME, redisKeyPrefix));
+                }
+                result.put(vid, vfMap);
+            }
+        }
+        return result;
+    }
+
+    static class Tuple4 {
+        public Map<String, String> first;
+        public Map<String, String> second;
+        public Map<String, String> third;
+
+        public String name;
+
+        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name) {
+            this.first = first;
+            this.second = second;
+            this.third = third;
+            this.name = name;
+        }
+
+    }
+
+    static class Tuple2 {
+        public Map<String, String> first;
+
+        public String name;
+
+        public Tuple2(Map<String, String> first, String name) {
+            this.first = first;
+            this.name = name;
+        }
+
+    }
+    protected double restoreScore(double score){
+        return (0.1 * score) / (1- 0.9 * score);
+    }
+
+
     public static void main(String[] args) {
 
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV551.java

@@ -1,63 +1,37 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
-import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.text.SimpleDateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.*;
 import java.util.stream.Collectors;
 
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV551 extends RankService {
+public class RankStrategy4RegionMergeModelV551 extends RankStrategy4RegionMergeModelBasic {
     @ApolloJsonValue("${rank.score.merge.weightv551:}")
     private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
@@ -73,548 +47,328 @@ public class RankStrategy4RegionMergeModelV551 extends RankService {
         oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
         removeDuplicate(oldRovs);
+        int sizeReturn = param.getSize();
         List<Video> v0 = oldRovs.size() <= sizeReturn
                 ? oldRovs
                 : oldRovs.subList(0, sizeReturn);
         Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v0);
-
-        //-------------------root rov召回 融合+去重-------------------
-        List<Video> v8 = extractAndSort(param, RegionRealtimeRecallStrategyV6RootRov.PUSH_FORM);
-        this.duplicate(setVideo, v8);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_sort.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2_sort.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3_sort.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        List<Video> rovRecallRank = new ArrayList<>(v0);
+        //-------------------return相似召回------------------
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 10.0).intValue(), v8.size())));
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 30.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 20.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
+        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+        // TODO 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()) {
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
         }
-        for (RankItem item : items) {
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
+        if (!c2.isEmpty()) {
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
         }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.6);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 240.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5) {
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            } else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
 
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()) {
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
+                }
             }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
         }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
 
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
+        );
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String[] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
+                        }
+                    }
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
+                }
             }
         }
-        return data;
-    }
 
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
 
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
 
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
                 }
             }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
 
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()) {
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
                         }
                     }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
             }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
 
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
             }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
+            if (!d1.isEmpty()) {
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
             }
+            item.featureMapDouble = featureMap;
         }
 
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                Double bucketNum = this.bucketsLen.get(name);
+                double[] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
             }
         }
 
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        for (RankItem item : rankItems) {
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
+            }
+            item.featureMap = featureMap;
         }
 
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+        // TODO 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
 
-        //7 流量池按比例强插
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240806.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
         List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
+        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
+
+        for (RankItem item : items) {
+            double score = 0.0;
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault(hasReturnRovKey, "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double fmRov = item.getScoreRov();
+            item.getScoresMap().put("fmRov", fmRov);
+            if (chooseFunction == 0){
+                score = fmRov * (1 + hasReturnRovScore);
+            }else if (chooseFunction == 1){
+                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
+            }else {
+                score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
             }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
+
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
             }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
             }
+            result.add(video);
         }
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
+
+        return result;
+    }
 
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
+    private void readBucketFile() {
+        InputStream resourceStream = RankStrategy4RegionMergeModelV551.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
+                        }
+                    }
                 }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
             }
+        } else {
+            log.error("no bucket file");
         }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
     }
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV552.java

@@ -1,62 +1,37 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.text.SimpleDateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.*;
 import java.util.stream.Collectors;
 
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV552 extends RankService {
+public class RankStrategy4RegionMergeModelV552 extends RankStrategy4RegionMergeModelBasic {
     @ApolloJsonValue("${rank.score.merge.weightv552:}")
     private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
@@ -72,677 +47,332 @@ public class RankStrategy4RegionMergeModelV552 extends RankService {
         oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
         removeDuplicate(oldRovs);
+        int sizeReturn = param.getSize();
         List<Video> v0 = oldRovs.size() <= sizeReturn
                 ? oldRovs
                 : oldRovs.subList(0, sizeReturn);
         Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v0);
-
-        //-------------------root rov召回 融合+去重-------------------
-        List<Video> v8 = extractAndSort(param, RegionRealtimeRecallStrategyV6RootRov.PUSH_FORM);
-        this.duplicate(setVideo, v8);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_sort.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2_sort.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3_sort.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        List<Video> rovRecallRank = new ArrayList<>(v0);
+        //-------------------return相似召回------------------
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 10.0).intValue(), v8.size())));
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 30.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 20.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition",
-                "item_rt_fea_1h_partition",
-                "item_rt_fea_1hroot_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        String rtFeaPart1hRoot = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-            if (rtFeaPartKeyResult.get(2) != null) {
-                rtFeaPart1hRoot = rtFeaPartKeyResult.get(2);
-            }
+        // 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+        // 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()) {
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
         }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
+        if (!c2.isEmpty()) {
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
         }
-        cur = rtFeaPart1hRoot;
-        List<String> datehoursRoot = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehoursRoot.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        // 2.1 item特征提取
-        this.getVideoFeatureFromRedis(items);
-
 
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealRootMap = item.getItemRealTimeRootFeature();
-            List<Double> return_20240408 = getStaticData(itemRealRootMap, datehoursRoot, "return");
-            Double returnScore_20240408 = calScoreWeightNoTimeDecay(return_20240408);
-            item.scoresMap.put("returnScore_20240408", returnScore_20240408);
-            List<Double> views_20240408 = getStaticData(itemRealRootMap, datehoursRoot, "view");
-            List<Double> share_20240408 = getStaticData(itemRealRootMap, datehoursRoot, "share");
-            List<Double> return1_20240408 = getStaticData(itemRealRootMap, datehoursRoot, "return1");
-            List<Double> return2_20240408 = getStaticData(itemRealRootMap, datehoursRoot, "return2");
-            List<Double> return3_20240408 = getStaticData(itemRealRootMap, datehoursRoot, "return3");
-            List<Double> return4_20240408 = getStaticData(itemRealRootMap, datehoursRoot, "return4");
-            List<Double> return12_20240408 = this.merge2List(return1_20240408, return2_20240408);
-            List<Double> rov_20240408 = getRateData(return12_20240408, views_20240408, 0.0, 0.0);
-            Double rovScore_20240408 = calScoreWeightNoTimeDecay(rov_20240408);
-            List<Double> ros_20240408 = getRateData(return12_20240408, share_20240408, 1.0, 10.0);
-            Double rosScore_20240408 = calScoreWeightNoTimeDecay(ros_20240408);
-            item.scoresMap.put("rovScore_20240408", rovScore_20240408);
-            item.scoresMap.put("rosScore_20240408", rosScore_20240408);
-
-
-            List<Double> fissionList =  getRateData(
-                    this.merge2List(return2_20240408, return4_20240408),
-                    this.merge2List(return1_20240408, return3_20240408),
-                    1.0, 1000.0
-            );
-            Double fissionScore = calScoreWeightNoTimeDecay(fissionList);
-            item.scoresMap.put("fissionScore", fissionScore);
-
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.01);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.001);
-        double d = mergeWeight.getOrDefault("d", 0.01);
-        double e = mergeWeight.getOrDefault("e", 0.03);
-        double f = mergeWeight.getOrDefault("f", 0.2);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 240.0);
-        double m = mergeWeight.getOrDefault("m", 0.1);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double returnScore_20240408 = Math.log(1 + item.scoresMap.getOrDefault("returnScore_20240408", 0.0));
-            double rovScore_20240408 = item.scoresMap.getOrDefault("rovScore_20240408", 0.0);
-            double rosScore_20240408 = item.scoresMap.getOrDefault("rosScore_20240408", 0.0);
-            double score = 0.0;
-            score = a * strScore + b * rosScore + c * returnScore_20240408;
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (d * trendScore + e * newVideoScore);
-                score += (f * rosScore_20240408 + g * rovScore_20240408);
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()) {
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
+                }
             }
-            double fissionScore = item.scoresMap.getOrDefault("fissionScore", 0.0);
-            score += m * fissionScore;
-
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays >= 10) {
-            return 0.0;
-        }
-        double score = 1 / (existenceDays + 1.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
         }
-        return down > 1E-8 ? up / down : 0.0;
-    }
 
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
+        );
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String[] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
+                        }
+                    }
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
+                }
             }
         }
-        return data;
-    }
 
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
 
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
 
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
                 }
             }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
 
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()) {
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
                         }
                     }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
             }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
 
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
             }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
+            if (!d1.isEmpty()) {
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
             }
+            item.featureMapDouble = featureMap;
         }
 
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                Double bucketNum = this.bucketsLen.get(name);
+                double[] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
             }
         }
 
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        for (RankItem item : rankItems) {
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
+            }
+            item.featureMap = featureMap;
         }
 
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+        // 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
 
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240807.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
         List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
+        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
+
+        for (RankItem item : items) {
+            double score = 0.0;
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault(hasReturnRovKey, "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double fmRovOrigin = item.getScoreRov();
+            item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
+            double fmRov = restoreScore(fmRovOrigin);
+            item.getScoresMap().put("fmRov", fmRov);
+            if (chooseFunction == 0){
+                score = fmRov * (1 + hasReturnRovScore);
+            }else if (chooseFunction == 1){
+                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
+            }else {
+                score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
             }
-        }
 
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
+            }
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
             }
+            result.add(video);
         }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
 
-        return new RankResult(resultWithDensity);
+        return result;
     }
 
-    private void getVideoFeatureFromRedis(List<RankItem> items){
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getVideoId);
-        List<String> videoKeys = videoIds.stream().map(r -> "item_rt_fea_1hroot_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoKeys);
-        int j = 0;
-        if (videoRtFeatures != null) {
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+    private void readBucketFile() {
+        InputStream resourceStream = RankStrategy4RegionMergeModelV552.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                    item.setItemRealTimeRootFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1hroot_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
             }
+        } else {
+            log.error("no bucket file");
         }
     }
-    private List<Double> merge2List(List<Double> a, List<Double> b){
-        int size = a.size();
-        List<Double> result = new ArrayList<>(size);
-        for (int i = 0; i < size; ++i){
-            result.add(a.get(i) + b.get(i));
-        }
-        return result;
-    }
-
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV552 job = new RankStrategy4RegionMergeModelV552();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
 
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
-    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV553.java

@@ -1,719 +0,0 @@
-package com.tzld.piaoquan.recommend.server.service.rank.strategy;
-
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
-import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
-import org.springframework.stereotype.Service;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
-@Service
-@Slf4j
-public class RankStrategy4RegionMergeModelV553 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv553:}")
-    private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
-
-    @Override
-    public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
-        //-------------------融-------------------
-        //-------------------合-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        List<Video> oldRovs = new ArrayList<>();
-        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
-        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
-        removeDuplicate(oldRovs);
-        List<Video> v0 = oldRovs.size() <= sizeReturn
-                ? oldRovs
-                : oldRovs.subList(0, sizeReturn);
-        Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v0);
-
-        //-------------------root rov召回 融合+去重-------------------
-        List<Video> v8 = extractAndSort(param, RegionRealtimeRecallStrategyV6RootRov.PUSH_FORM);
-        this.duplicate(setVideo, v8);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_sort.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2_sort.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3_sort.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v8.subList(0, Math.min(mergeWeight.getOrDefault("v8", 10.0).intValue(), v8.size())));
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 30.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 20.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
-
-        //-------------------排-------------------
-        //-------------------序-------------------
-        //-------------------逻-------------------
-        //-------------------辑-------------------
-
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        List<String> datehoursRoot = new LinkedList<>();
-        for (int i = 0; i < 24; ++i) {
-            datehoursRoot.add(String.valueOf(i+1));
-        }
-        // 2.1 item特征提取
-        this.getVideoFeatureFromRedis(items);
-
-
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealRootMap = item.getItemRealTimeRootFeature();
-            List<Double> views_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "exp");
-            List<Double> share_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "share");
-            List<Double> return_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "return");
-            List<Double> rov_20240410 = getRateData(return_20240410, views_20240410, 0.0, 0.0);
-            Double rovScore_20240410 = calScoreWeightNoTimeDecay(rov_20240410);
-            List<Double> ros_20240410 = getRateData(return_20240410, share_20240410, 1.0, 10.0);
-            Double rosScore_20240410 = calScoreWeightNoTimeDecay(ros_20240410);
-            item.scoresMap.put("rovScore_20240410", rovScore_20240410);
-            item.scoresMap.put("rosScore_20240410", rosScore_20240410);
-
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
-        double h = mergeWeight.getOrDefault("h", 240.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double rovScore_20240410 = item.scoresMap.getOrDefault("rovScore_20240410", 0.0);
-            double rosScore_20240410 = item.scoresMap.getOrDefault("rosScore_20240410", 0.0);
-
-            double score = 0.0;
-            if (ifAdd < 0.5) {
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            } else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-
-            }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * rosScore_20240410 + g * rovScore_20240410);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
-    }
-
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
-
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
-                        }
-                    }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
-            }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
-            }
-        }
-
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
-
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-    private void getVideoFeatureFromRedis(List<RankItem> items){
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getVideoId);
-        List<String> videoKeys = videoIds.stream().map(r -> "item_rt_fea_1hrootall_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoKeys);
-        int j = 0;
-        if (videoRtFeatures != null) {
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeRootFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1hrootall_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-    }
-
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV553 job = new RankStrategy4RegionMergeModelV553();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
-
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
-
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
-        }
-        return result;
-    }
-
-}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV562.java

@@ -1,585 +1,382 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
-
-import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
-import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeatureV2;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeatureV2;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
-import com.tzld.piaoquan.recommend.server.service.recall.RecallResult;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.service.score.ScoreParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.text.SimpleDateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.*;
 import java.util.stream.Collectors;
 
-/**
- * @author zhangbo
- * @desc 地域召回融合
- */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV562 extends RankService {
+public class RankStrategy4RegionMergeModelV562 extends RankStrategy4RegionMergeModelBasic {
     @ApolloJsonValue("${rank.score.merge.weightv562:}")
     private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos){
-        Iterator<Video> iterator = videos.iterator();
-        while(iterator.hasNext()){
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())){
-                iterator.remove();
-            }else{
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
+
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
-        Map<String, Double> mergeWeight = this.mergeWeight != null? this.mergeWeight: new HashMap<>(0);
+        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
         //-------------------融-------------------
         //-------------------合-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        List<Video> oldRovs = new ArrayList<>();
+        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        removeDuplicate(oldRovs);
+        int sizeReturn = param.getSize();
+        List<Video> v0 = oldRovs.size() <= sizeReturn
+                ? oldRovs
+                : oldRovs.subList(0, sizeReturn);
         Set<Long> setVideo = new HashSet<>();
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v4);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v0);
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        List<Video> rovRecallRank = new ArrayList<>(v0);
+        //-------------------return相似召回------------------
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        this.duplicate(setVideo, v7);
-
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 20.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 10.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 10.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 10.0).intValue(), v7.size())));
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<RankItem> items = model(rovRecallRank, param);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i=0; i<24; ++i){
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
+        // 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+        // 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()) {
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
         }
-        for (RankItem item : items){
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeight(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeight(returns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
+        if (!c2.isEmpty()) {
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
         }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 1.0);
-        double b = mergeWeight.getOrDefault("b", 1.0);
-        double c = mergeWeight.getOrDefault("c", 0.0002);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 0.0);
-        for (RankItem item : items){
-            double trendScore =  item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore =  item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScoreModel = item.getScoreRos();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double score = 0.0;
-            if (ifAdd < 0.5){
-                score = Math.pow(strScore, a) * Math.pow(rosScoreModel, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-            }else {
-                score = a * strScore + b * rosScoreModel + c * preturnsScore +
-                        (newVideoScore > 1E-8? d * trendScore * (e + newVideoScore): 0.0);
-            }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoreRos(item.getScoreRos());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
 
-    public double calNewVideoScore(Map<String, String> itemBasicMap){
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5){
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-    public double calTrendScore(List<Double> data){
-        double sum = 0.0;
-        int size = data.size();
-        for (int i=0; i<size-4; ++i){
-            sum += data.get(i) - data.get(i+4);
-        }
-        if (sum * 10 > 0.6){
-            sum = 0.6;
-        }else{
-            sum = sum * 10;
-        }
-        if (sum > 0){
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-    public Double calScoreWeight(List<Double> data){
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i=0; i<data.size(); ++i){
-            up += 1.0 / (i + 1) * data.get(i);
-            down += 1.0 / (i + 1);
-        }
-        return down > 1E-8? up / down: 0.0;
-    }
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down){
-        List<Double> data = new LinkedList<>();
-        for(int i=0; i<ups.size(); ++i){
-            data.add(
-                    (ups.get(i) + up) / (downs.get(i) + down)
-            );
-        }
-        return data;
-    }
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key){
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours){
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0: views.get(views.size()-1))
-            );
-        }
-        return views;
-    }
-    public List<RankItem> model(List<Video> videos, RankParam param){
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()){
-            return result;
-        }
-
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()){
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null){
-                try{
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {},
-                            userFeatureMap);
-                }catch (Exception e){
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()) {
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
                 }
-            }else{
-                JSONObject obj = new JSONObject();
-                obj.put("name", "user_key_in_model_is_null");
-                obj.put("class", this.CLASS_NAME);
             }
         }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
-                "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
-        Map<String, String> f1 = RankExtractorUserFeatureV2.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, Double> f2__ = RankExtractorUserFeatureV2.getUserRateFeature(userFeatureMap);
-        Map<String, String> f2 = RankExtractorUserFeatureV2.rateFeatureChange(f2__);
-        Map<String, String> f3 = RankExtractorUserFeatureV2.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
-                        "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt"
-                ))
+
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
         );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
-                "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null){
-            for (int i=0; i<videoFeatures.size(); ++i){
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null){
-                    continue;
-                }
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String[] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
                         }
                     }
-                    Map<String, Double> f4__ = RankExtractorItemFeatureV2.getItemRateFeature(vfMap);
-                    Map<String, String> f4 = RankExtractorItemFeatureV2.rateFeatureChange(f4__);
-                    Map<String, String> f5 = RankExtractorItemFeatureV2.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt",
-                                    "i_7day_exp_cnt", "i_7day_click_cnt", "i_7day_share_cnt", "i_7day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                }catch (Exception e){
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
                 }
             }
         }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null){
-            if (rtFeaPartKeyResult.get(0) != null){
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null){
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
 
-        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-        if (videoRtFeatures != null){
-            int j = 0;
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
+
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
                 }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+            }
+
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()) {
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
-                item.getFeatureMap().putAll(f8);
             }
-            for (RankItem item: rankItems){
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null){
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try{
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
-                        String value = entry.getValue();
-                        if (value == null){
-                            continue;
-                        }
-                        String [] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1){
-                            String [] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                    item.setItemRealTimeFeature(vfMapNew);
-                }catch (Exception e){
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, Double> f8__ = RankExtractorItemFeatureV2.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                Map<String, String> f8 = RankExtractorItemFeatureV2.rateFeatureChange(f8__);
-                item.getFeatureMap().putAll(f8);
             }
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
+            if (!d1.isEmpty()) {
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
+            }
+            item.featureMapDouble = featureMap;
         }
 
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline("feeds_score_config_20240228.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        JSONObject obj = new JSONObject();
-        obj.put("name", "user_key_in_model_is_not_null");
-        obj.put("class", this.CLASS_NAME);
-        return rovRecallScore;
-    }
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ){
-            ;
-        }else{
-            city = city.replaceAll("市$", "");
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                Double bucketNum = this.bucketsLen.get(name);
+                double[] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
+            }
         }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
 
-        return sceneFeatureMap;
-    }
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
+        for (RankItem item : rankItems) {
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
             }
+            item.featureMap = featureMap;
         }
 
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()){
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
+        // 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
 
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240807.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
+        Map<String, Map<String, String>> vid2VovFeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vov_1d3d:");
+        double alpha_vov = mergeWeight.getOrDefault("alpha_vov", 2.0);
+        List<Video> result = new ArrayList<>();
+//        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+//        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
 
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+        for (RankItem item : items) {
+            double score = 0.0;
+            double recommend_rate_1d = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("recommend_rate_1d", "0"));
+            double recommend_exp_per_1d = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("recommend_exp_per_1d", "0"));
+            double vorScore =  recommend_rate_1d * recommend_exp_per_1d;
+            item.getScoresMap().put("recommend_rate_1d", recommend_rate_1d);
+            item.getScoresMap().put("recommend_exp_per_1d", recommend_exp_per_1d);
+            item.getScoresMap().put("vorScore", vorScore);
+            item.getScoresMap().put("alpha_vov", alpha_vov);
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("rate_n", "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double fmRovOrigin = item.getScoreRov();
+            item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
+            double fmRov = restoreScore(fmRovOrigin);
+            item.getScoresMap().put("fmRov", fmRov);
+            score = fmRov * (1 + hasReturnRovScore) * (1.0 + alpha_vov * recommend_rate_1d);
 
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
             }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
             }
+            result.add(video);
         }
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
 
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
+        return result;
+    }
+
+    private void readBucketFile() {
+        InputStream resourceStream = RankStrategy4RegionMergeModelV562.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
+                        }
+                    }
                 }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
             }
+        } else {
+            log.error("no bucket file");
         }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
     }
 
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV563.java

@@ -1,62 +1,37 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.text.SimpleDateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.*;
 import java.util.stream.Collectors;
 
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV563 extends RankService {
+public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeModelBasic {
     @ApolloJsonValue("${rank.score.merge.weightv563:}")
     private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
@@ -72,620 +47,336 @@ public class RankStrategy4RegionMergeModelV563 extends RankService {
         oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
         removeDuplicate(oldRovs);
+        int sizeReturn = param.getSize();
         List<Video> v0 = oldRovs.size() <= sizeReturn
                 ? oldRovs
                 : oldRovs.subList(0, sizeReturn);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2_sort.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        //-------------------流量池回捞-------------------
-        List<Video> v9 = extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM);
-
-        // 去重
         Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v0);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        this.duplicate(setVideo, v4);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v9);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v7);
-
-
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 25.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
-        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size())));
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        List<Video> rovRecallRank = new ArrayList<>(v0);
+        //-------------------return相似召回------------------
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
+        // 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+        // 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()) {
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
         }
-        List<String> datehoursRoot = new LinkedList<>();
-        for (int i = 0; i < 24; ++i) {
-            datehoursRoot.add(String.valueOf(i+1));
+        if (!c2.isEmpty()) {
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
         }
-        // 2.1 item特征提取
-        this.getVideoFeatureFromRedis(items);
 
-
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealRootMap = item.getItemRealTimeRootFeature();
-            List<Double> views_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "exp");
-            List<Double> share_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "share");
-            List<Double> return_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "return");
-            List<Double> rov_20240410 = getRateData(return_20240410, views_20240410, 0.0, 0.0);
-            Double rovScore_20240410 = calScoreWeightNoTimeDecay(rov_20240410);
-            List<Double> ros_20240410 = getRateData(return_20240410, share_20240410, 1.0, 10.0);
-            Double rosScore_20240410 = calScoreWeightNoTimeDecay(ros_20240410);
-            item.scoresMap.put("rovScore_20240410", rovScore_20240410);
-            item.scoresMap.put("rosScore_20240410", rosScore_20240410);
-
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 50.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double rovScore_20240410 = item.scoresMap.getOrDefault("rovScore_20240410", 0.0);
-            double rosScore_20240410 = item.scoresMap.getOrDefault("rosScore_20240410", 0.0);
-
-            double score = 0.0;
-            if (ifAdd < 0.5) {
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            } else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-
-            }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * rosScore_20240410 + g * rovScore_20240410);
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()) {
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
+                }
             }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
         }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
 
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
-
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
+        );
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String[] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
+                        }
+                    }
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
+                }
             }
         }
-        return data;
-    }
 
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
 
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
 
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
                 }
             }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
 
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()) {
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
                         }
                     }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
             }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
 
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
                         }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
             }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
+            if (!d1.isEmpty()) {
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
             }
+            item.featureMapDouble = featureMap;
         }
 
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                Double bucketNum = this.bucketsLen.get(name);
+                double[] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
+            }
         }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
 
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
+        for (RankItem item : rankItems) {
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
             }
+            item.featureMap = featureMap;
         }
 
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+        // 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
 
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_xgb_20240828.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
+        Map<String, Map<String, String>> vid2VovFeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vov_1d3d:");
+        double alpha_vov = mergeWeight.getOrDefault("alpha_vov", 2.0);
+        List<Video> result = new ArrayList<>();
+//        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+//        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
 
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+        for (RankItem item : items) {
+            double score = 0.0;
+            double recommend_rate_1d = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("recommend_rate_1d", "0"));
+            double recommend_exp_per_1d = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("recommend_exp_per_1d", "0"));
+            double vorScore =  recommend_rate_1d * recommend_exp_per_1d;
+            item.getScoresMap().put("recommend_rate_1d", recommend_rate_1d);
+            item.getScoresMap().put("recommend_exp_per_1d", recommend_exp_per_1d);
+            item.getScoresMap().put("vorScore", vorScore);
+            item.getScoresMap().put("alpha_vov", alpha_vov);
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("rate_n", "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double rovRateOrigin = item.getScoreRov();
+            item.getScoresMap().put("rovRateOrigin", rovRateOrigin);
+            double rovRate = restoreScore(rovRateOrigin);
+            item.getScoresMap().put("rovRate", rovRate);
+            score = rovRate * (1 + hasReturnRovScore) * (1.0 + alpha_vov * recommend_rate_1d);
 
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
             }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
-
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
             }
+            result.add(video);
         }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
 
-        return new RankResult(resultWithDensity);
+        return result;
     }
 
-    private void getVideoFeatureFromRedis(List<RankItem> items){
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getVideoId);
-        List<String> videoKeys = videoIds.stream().map(r -> "item_rt_fea_1hrootall_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoKeys);
-        int j = 0;
-        if (videoRtFeatures != null) {
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+    private void readBucketFile() {
+        InputStream resourceStream = RankStrategy4RegionMergeModelV563.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                    item.setItemRealTimeRootFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1hrootall_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
             }
+        } else {
+            log.error("no bucket file");
         }
     }
 
 
-    public static void main(String[] args) {
-
-
-    }
-
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV564.java

@@ -1,62 +1,37 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.text.SimpleDateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.*;
 import java.util.stream.Collectors;
 
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV564 extends RankService {
+public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeModelBasic {
     @ApolloJsonValue("${rank.score.merge.weightv564:}")
     private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
@@ -72,620 +47,378 @@ public class RankStrategy4RegionMergeModelV564 extends RankService {
         oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
         removeDuplicate(oldRovs);
+        int sizeReturn = param.getSize();
         List<Video> v0 = oldRovs.size() <= sizeReturn
                 ? oldRovs
                 : oldRovs.subList(0, sizeReturn);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2_sort.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        //-------------------流量池回捞-------------------
-        List<Video> v9 = extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM);
-
-        // 去重
         Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v0);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        this.duplicate(setVideo, v4);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v9);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v7);
-
-
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 25.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
-        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size())));
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        List<Video> rovRecallRank = new ArrayList<>(v0);
+        //-------------------return相似召回------------------
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
+        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+        // TODO 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()) {
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
         }
-        List<String> datehoursRoot = new LinkedList<>();
-        for (int i = 0; i < 24; ++i) {
-            datehoursRoot.add(String.valueOf(i+1));
+        if (!c2.isEmpty()) {
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
         }
-        // 2.1 item特征提取
-        this.getVideoFeatureFromRedis(items);
 
-
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealRootMap = item.getItemRealTimeRootFeature();
-            List<Double> views_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "exp");
-            List<Double> share_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "share");
-            List<Double> return_20240410 = getStaticData(itemRealRootMap, datehoursRoot, "return");
-            List<Double> rov_20240410 = getRateData(return_20240410, views_20240410, 0.0, 0.0);
-            Double rovScore_20240410 = calScoreWeightNoTimeDecay(rov_20240410);
-            List<Double> ros_20240410 = getRateData(return_20240410, share_20240410, 1.0, 10.0);
-            Double rosScore_20240410 = calScoreWeightNoTimeDecay(ros_20240410);
-            item.scoresMap.put("rovScore_20240410", rovScore_20240410);
-            item.scoresMap.put("rosScore_20240410", rosScore_20240410);
-
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> preturns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(preturns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeightNoTimeDecay(share2return);
-            List<Double> view2return = getRateData(preturns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeightNoTimeDecay(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeightNoTimeDecay(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeightNoTimeDecay(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 1.0, 10.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
-
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // 平台回流
-            Double preturnsScore = calScoreWeightNoTimeDecay(preturns);
-            item.scoresMap.put("preturnsScore", preturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double a = mergeWeight.getOrDefault("a", 0.1);
-        double b = mergeWeight.getOrDefault("b", 0.0);
-        double c = mergeWeight.getOrDefault("c", 0.000001);
-        double d = mergeWeight.getOrDefault("d", 1.0);
-        double e = mergeWeight.getOrDefault("e", 1.0);
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 2.0);
-        double h = mergeWeight.getOrDefault("h", 5.0);
-        double ifAdd = mergeWeight.getOrDefault("ifAdd", 1.0);
-        for (RankItem item : items) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 1E-8 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double strScore = item.getScoreStr();
-            double rosScore = item.scoresMap.getOrDefault("share2returnScore", 0.0);
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
-            double preturnsScore = Math.log(1 + item.scoresMap.getOrDefault("preturnsScore", 0.0));
-            double rovScore_20240410 = item.scoresMap.getOrDefault("rovScore_20240410", 0.0);
-            double rosScore_20240410 = item.scoresMap.getOrDefault("rosScore_20240410", 0.0);
-
-            double score = 0.0;
-            if (ifAdd < 0.5) {
-                score = Math.pow(strScore, a) * Math.pow(rosScore, b) + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-            } else {
-                score = a * strScore + b * rosScore + c * preturnsScore +
-                        (newVideoScore > 1E-8 ? d * trendScore * (e + newVideoScore) : 0.0);
-
-            }
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > h) {
-                score += (f * rosScore_20240410 + g * rovScore_20240410);
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()) {
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
+                }
             }
-            Video video = item.getVideo();
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
-    public double calNewVideoScore(Map<String, String> itemBasicMap) {
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 5) {
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 10.0);
-        return score;
-    }
-
-    public double calTrendScore(List<Double> data) {
-        double sum = 0.0;
-        int size = data.size();
-        for (int i = 0; i < size - 4; ++i) {
-            sum += data.get(i) - data.get(i + 4);
-        }
-        if (sum * 10 > 0.6) {
-            sum = 0.6;
-        } else {
-            sum = sum * 10;
-        }
-        if (sum > 0) {
-            // 为了打断点
-            sum = sum;
         }
-        return sum;
-    }
-
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
-    }
 
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
+        );
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String[] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
+                        }
+                    }
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
+                }
             }
         }
-        return data;
-    }
 
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
-    }
 
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
 
-        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
-        redisSC.setPort(6379);
-        redisSC.setPassword("Wqsd@2019");
-        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
-        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
-        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
-        redisTemplate.setConnectionFactory(connectionFactory);
-        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
-        redisTemplate.afterPropertiesSet();
-
-        // 0: 场景特征处理
-        Map<String, String> sceneFeatureMap = this.getSceneFeature(param);
-
-        // 1: user特征处理
-        Map<String, String> userFeatureMap = new HashMap<>();
-        if (param.getMid() != null && !param.getMid().isEmpty()) {
-            String midKey = "user_info_4video_" + param.getMid();
-            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
-            if (userFeatureStr != null) {
-                try {
-                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
-                            new TypeToken<Map<String, String>>() {
-                            },
-                            userFeatureMap);
-                } catch (Exception e) {
-                    log.error(String.format("parse user json is wrong in {} with {}", this.CLASS_NAME, e));
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+                    double f6 = ExtractorUtils.calDiv(returns, share);
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+                    String key6 = tuple4.name + "_" + prefix2 + "_" + "ROS";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
+                    featureMap.put(key6, f6);
                 }
             }
-        }
-        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
-                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
-                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-        ));
-        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
-        while (iterator.hasNext()) {
-            Map.Entry<String, String> entry = iterator.next();
-            if (!userFeatureSet.contains(entry.getKey())) {
-                iterator.remove();
-            }
-        }
 
-        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
-                ))
-        );
-        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
-        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
-                new HashSet<String>(Arrays.asList(
-                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
-                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
-                ))
-        );
-        f1.putAll(f2);
-        f1.putAll(f3);
-
-        // 2-1: item特征处理
-        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
-                "total_time", "play_count_total",
-                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
-        ));
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-        List<String> videoFeatureKeys = videoIds.stream().map(r -> "video_info_" + r)
-                .collect(Collectors.toList());
-        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
-        if (videoFeatures != null) {
-            for (int i = 0; i < videoFeatures.size(); ++i) {
-                String vF = videoFeatures.get(i);
-                Map<String, String> vfMap = new HashMap<>();
-                if (vF == null) {
-                    continue;
-                }
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    Map<String, String> vfMapCopy = new HashMap<>(vfMap);
-                    rankItems.get(i).setItemBasicFeature(vfMapCopy);
-                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
-                    while (iteratorIn.hasNext()) {
-                        Map.Entry<String, String> entry = iteratorIn.next();
-                        if (!itemFeatureSet.contains(entry.getKey())) {
-                            iteratorIn.remove();
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()) {
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
                         }
                     }
-                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
-                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
-                            new HashSet<String>(Arrays.asList(
-                                    "total_time", "play_count_total",
-                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
-                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
-                    );
-                    f4.putAll(f5);
-                    rankItems.get(i).setFeatureMap(f4);
-                } catch (Exception e) {
-                    log.error(String.format("parse video json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
             }
-        }
-        // 2-2: item 实时特征处理
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1day = date + hour;
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(0) != null) {
-                rtFeaPart1day = rtFeaPartKeyResult.get(0);
-            }
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
 
-        List<String> videoRtKeys1 = videoIds.stream().map(r -> "item_rt_fea_1day_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        videoRtKeys1.addAll(videoRtKeys2);
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
                         }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
-                item.getFeatureMap().putAll(f8);
             }
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
-                item.getFeatureMap().putAll(f8);
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
+            if (!d1.isEmpty()) {
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
             }
+            item.featureMapDouble = featureMap;
         }
 
-
-        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
-        return rovRecallScore;
-    }
-
-    private Map<String, String> getSceneFeature(RankParam param) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        String provinceCn = param.getProvince();
-        provinceCn = provinceCn.replaceAll("省$", "");
-        sceneFeatureMap.put("ctx_region", provinceCn);
-        String city = param.getCity();
-        if ("台北市".equals(city) |
-                "高雄市".equals(city) |
-                "台中市".equals(city) |
-                "桃园市".equals(city) |
-                "新北市".equals(city) |
-                "台南市".equals(city) |
-                "基隆市".equals(city) |
-                "吉林市".equals(city) |
-                "新竹市".equals(city) |
-                "嘉义市".equals(city)
-        ) {
-        } else {
-            city = city.replaceAll("市$", "");
-        }
-        sceneFeatureMap.put("ctx_city", city);
-
-        Calendar calendar = Calendar.getInstance();
-        sceneFeatureMap.put("ctx_week", (calendar.get(Calendar.DAY_OF_WEEK) + 6) % 7 + "");
-        sceneFeatureMap.put("ctx_hour", new SimpleDateFormat("HH").format(calendar.getTime()));
-
-        return sceneFeatureMap;
-    }
-
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                Double bucketNum = this.bucketsLen.get(name);
+                double[] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
             }
         }
 
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        for (RankItem item : rankItems) {
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
+            }
+            item.featureMap = featureMap;
         }
 
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
-
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+        // TODO 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
 
-        //7 流量池按比例强插
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240711.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
         List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
+        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
+        Double rosDefault = mergeWeight.getOrDefault("rosDefault", 1.0);
+
+        for (RankItem item : items) {
+            double score = 0.0;
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault(hasReturnRovKey, "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double fmRov = item.getScoreRov();
+            item.getScoresMap().put("fmRov", fmRov);
+            if (chooseFunction == 0){
+                score = fmRov * (rosDefault + hasReturnRovScore);
+            }else if (chooseFunction == 1){
+                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
+            }else {
+                score = fmRov * (1 + hasReturnRovScore);
             }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
+
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
             }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
             }
+            result.add(video);
         }
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
 
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
+        return result;
+    }
 
-        return new RankResult(resultWithDensity);
+    private Map<String, Map<String, String>> extractVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
     }
 
-    private void getVideoFeatureFromRedis(List<RankItem> items){
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getVideoId);
-        List<String> videoKeys = videoIds.stream().map(r -> "item_rt_fea_1hrootall_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoKeys);
-        int j = 0;
-        if (videoRtFeatures != null) {
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
+    }
+
+    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
+    }
+
+    private void readBucketFile() {
+        InputStream resourceStream = RankStrategy4RegionMergeModelV564.class.getClassLoader().getResourceAsStream("20240609_bucket_314.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                    item.setItemRealTimeRootFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1hrootall_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
             }
+        } else {
+            log.error("no bucket file");
         }
+
     }
 
+    static class Tuple4 {
+        public Map<String, String> first;
+        public Map<String, String> second;
+        public Map<String, String> third;
 
-    public static void main(String[] args) {
+        public String name;
 
+        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name) {
+            this.first = first;
+            this.second = second;
+            this.third = third;
+            this.name = name;
+        }
 
     }
 
+    static class Tuple2 {
+        public Map<String, String> first;
+
+        public String name;
 
+        public Tuple2(Map<String, String> first, String name) {
+            this.first = first;
+            this.name = name;
+        }
+
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV565.java

@@ -1,62 +1,37 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.text.SimpleDateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.*;
 import java.util.stream.Collectors;
 
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV565 extends RankService {
+public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeModelBasic {
     @ApolloJsonValue("${rank.score.merge.weightv565:}")
     private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
@@ -72,303 +47,375 @@ public class RankStrategy4RegionMergeModelV565 extends RankService {
         oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
-        int sizeReturn = param.getSize();
         removeDuplicate(oldRovs);
+        int sizeReturn = param.getSize();
         List<Video> v0 = oldRovs.size() <= sizeReturn
                 ? oldRovs
                 : oldRovs.subList(0, sizeReturn);
-
-        //-------------------地域相关召回 融合+去重-------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2_sort.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
-        List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
-        //-------------------相关性召回 融合+去重-------------------
-        List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        //-------------------节日扶持召回 融合+去重-------------------
-        List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        //-------------------流量池回捞-------------------
-        List<Video> v9 = extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM);
-
-        // 去重
         Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v0);
-        this.duplicate(setVideo, v5);
-        this.duplicate(setVideo, v6);
-        this.duplicate(setVideo, v4);
-        this.duplicate(setVideo, v3);
-        this.duplicate(setVideo, v2);
-        this.duplicate(setVideo, v9);
-        this.duplicate(setVideo, v1);
-        this.duplicate(setVideo, v7);
-
-
-
-        List<Video> rovRecallRank = new ArrayList<>();
-        rovRecallRank.addAll(v0);
-        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 25.0).intValue(), v1.size())));
-        rovRecallRank.addAll(v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 15.0).intValue(), v2.size())));
-        rovRecallRank.addAll(v3.subList(0, Math.min(mergeWeight.getOrDefault("v3", 10.0).intValue(), v3.size())));
-        rovRecallRank.addAll(v4.subList(0, Math.min(mergeWeight.getOrDefault("v4", 5.0).intValue(), v4.size())));
-        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
-        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
-        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size())));
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        List<Video> rovRecallRank = new ArrayList<>(v0);
+        //-------------------return相似召回------------------
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+        // TODO 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()) {
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
+        }
+        if (!c2.isEmpty()) {
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
+        }
+
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()) {
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
+                }
             }
         }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
+
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
+        );
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String[] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
+                        }
+                    }
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
+                }
+            }
         }
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
 
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
 
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
+
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
+                }
+            }
+
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()) {
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
+                        }
+                    }
+                }
+            }
+
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
+                        }
+                    }
+                }
+            }
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
+            if (!d1.isEmpty()) {
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
+            }
+            item.featureMapDouble = featureMap;
+        }
 
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                Double bucketNum = this.bucketsLen.get(name);
+                double[] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
+            }
+        }
 
+        for (RankItem item : rankItems) {
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
+            }
+            item.featureMap = featureMap;
         }
-        // 3 融合公式
+
+        // TODO 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
+
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240609.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
         List<Video> result = new ArrayList<>();
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
+        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
+        Double rosDefault = mergeWeight.getOrDefault("rosDefault", 0.1);
+
         for (RankItem item : items) {
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
             double score = 0.0;
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > 50) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }else{
-                score += (f * share2allreturnScore + g * view2allreturnScore) * 0.01;
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault(hasReturnRovKey, "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double fmRov = item.getScoreRov();
+            item.getScoresMap().put("fmRov", fmRov);
+            if (chooseFunction == 0){
+                score = fmRov * (rosDefault + hasReturnRovScore);
+            }else if (chooseFunction == 1){
+                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
+            }else {
+                score = fmRov * (1 + hasReturnRovScore);
             }
+
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
             video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
+            }
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
+            }
             result.add(video);
         }
         result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
+
         return result;
     }
 
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
+    private Map<String, Map<String, String>> extractVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
     }
 
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
+    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
     }
 
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
+    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
     }
 
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-
-        // 2-2: item 实时特征处理
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys2);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+    private void readBucketFile() {
+        InputStream resourceStream = RankStrategy4RegionMergeModelV999.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
             }
+        } else {
+            log.error("no bucket file");
         }
 
-        return rankItems;
     }
 
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
+    static class Tuple4 {
+        public Map<String, String> first;
+        public Map<String, String> second;
+        public Map<String, String> third;
 
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
+        public String name;
 
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
+        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name) {
+            this.first = first;
+            this.second = second;
+            this.third = third;
+            this.name = name;
         }
 
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+    }
 
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+    static class Tuple2 {
+        public Map<String, String> first;
 
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
-        }
+        public String name;
 
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
+        public Tuple2(Map<String, String> first, String name) {
+            this.first = first;
+            this.name = name;
         }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
-    }
-
-    public static void main(String[] args) {
-
 
     }
 
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV567.java

@@ -99,7 +99,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankService {
         rovRecallRank.addAll(v0);
         rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
         rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
-        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 5.0).intValue(), v9.size())));
+        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 0.0).intValue(), v9.size())));
         rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size())));
         rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV569.java

@@ -1,62 +1,36 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
-import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
-import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
-import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.text.SimpleDateFormat;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.*;
-import java.util.stream.Collectors;
 
-/**
- * @author zhangbo
- * @desc 地域召回融合 流量池汤姆森
- */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV569 extends RankService {
+public class RankStrategy4RegionMergeModelV569 extends RankStrategy4RegionMergeModelBasic {
     @ApolloJsonValue("${rank.score.merge.weightv569:}")
     private Map<String, Double> mergeWeight;
-    @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
-    final private String CLASS_NAME = this.getClass().getSimpleName();
-
-    public void duplicate(Set<Long> setVideo, List<Video> videos) {
-        Iterator<Video> iterator = videos.iterator();
-        while (iterator.hasNext()) {
-            Video v = iterator.next();
-            if (setVideo.contains(v.getVideoId())) {
-                iterator.remove();
-            } else {
-                setVideo.add(v.getVideoId());
-            }
-        }
-    }
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
 
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
@@ -66,9 +40,6 @@ public class RankStrategy4RegionMergeModelV569 extends RankService {
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        List<Video> rovRecallRank = new ArrayList<>();
-        Set<Long> setVideo = new HashSet<>();
-        //-------------------老地域召回-------------------
         List<Video> oldRovs = new ArrayList<>();
         oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
@@ -80,320 +51,367 @@ public class RankStrategy4RegionMergeModelV569 extends RankService {
         List<Video> v0 = oldRovs.size() <= sizeReturn
                 ? oldRovs
                 : oldRovs.subList(0, sizeReturn);
+        Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v0);
-        rovRecallRank.addAll(v0);
-        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------sim相似召回------------------
+
+
+        //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
-        v5 = v5.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v5 = v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size()));
-        rovRecallRank.addAll(v5);
-        setVideo.addAll(v5.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------return相似召回------------------
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
-        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
-        rovRecallRank.addAll(v6);
-        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
+        this.duplicate(setVideo, v5);
+        this.duplicate(setVideo, v6);
+        //-------------------流量池直接送 融合+去重-------------------
+        List<Video> v9 = extractAndSort(param, FlowPoolLastDayTopRecallStrategy.PUSH_FORM);
+        this.duplicate(setVideo, v9);
+        //-------------------地域相关召回 融合+去重-------------------
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------节日特殊召回-------------------
+        this.duplicate(setVideo, v1);
+        //-------------------节日扶持召回 融合+去重-------------------
         List<Video> v7 = extractAndSort(param, FestivalRecallStrategyV1.PUSH_FORM);
-        v7 = v7.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v7 = v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size()));
-        rovRecallRank.addAll(v7);
-        setVideo.addAll(v7.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-
+        this.duplicate(setVideo, v7);
+        List<Video> rovRecallRank = new ArrayList<>();
+        rovRecallRank.addAll(v0);
+        rovRecallRank.addAll(v5.subList(0, Math.min(mergeWeight.getOrDefault("v5", 5.0).intValue(), v5.size())));
+        rovRecallRank.addAll(v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size())));
+        rovRecallRank.addAll(v9.subList(0, Math.min(mergeWeight.getOrDefault("v9", 0.0).intValue(), v9.size())));
+        rovRecallRank.addAll(v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size())));
+        rovRecallRank.addAll(v7.subList(0, Math.min(mergeWeight.getOrDefault("v7", 5.0).intValue(), v7.size())));
 
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // 1 模型分
-        List<String> rtFeaPart = new ArrayList<>();
-        List<RankItem> items = model(rovRecallRank, param, rtFeaPart);
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+        // TODO 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()) {
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
+        }
+        if (!c2.isEmpty()) {
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
+        }
+
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()) {
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
+                }
             }
         }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
+
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
+        );
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String[] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
+                        }
+                    }
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
+                }
+            }
         }
-        for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
 
-            // 全部回流的rov和ros
-            List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
-            Double share2allreturnScore = calScoreWeightNoTimeDecay(share2allreturn);
-            item.scoresMap.put("share2allreturnScore", share2allreturnScore);
-            List<Double> view2allreturn = getRateData(allreturns, views, 0.0, 0.0);
-            Double view2allreturnScore = calScoreWeightNoTimeDecay(view2allreturn);
-            item.scoresMap.put("view2allreturnScore", view2allreturnScore);
 
-            // 全部回流
-            Double allreturnsScore = calScoreWeightNoTimeDecay(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
+
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
+                }
+            }
 
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()) {
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
+                        }
+                    }
+                }
+            }
 
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
+                        }
+                    }
+                }
+            }
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
+            if (!d1.isEmpty()) {
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
+            }
+            item.featureMapDouble = featureMap;
+        }
+
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                Double bucketNum = this.bucketsLen.get(name);
+                double[] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
+            }
         }
-        // 3 融合公式
+
+        for (RankItem item : rankItems) {
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
+            }
+            item.featureMap = featureMap;
+        }
+
+        // TODO 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
+
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240609.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
         List<Video> result = new ArrayList<>();
-        double f = mergeWeight.getOrDefault("f", 0.1);
-        double g = mergeWeight.getOrDefault("g", 1.0);
+        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 0.0) < 0.5 ? "rate_1" : "rate_n";
         for (RankItem item : items) {
-            double share2allreturnScore = item.scoresMap.getOrDefault("share2allreturnScore", 0.0);
-            double view2allreturnScore = item.scoresMap.getOrDefault("view2allreturnScore", 0.0);
             double score = 0.0;
-            double allreturnsScore = item.scoresMap.getOrDefault("allreturnsScore", 0.0);
-            if (allreturnsScore > 50) {
-                score += (f * share2allreturnScore + g * view2allreturnScore);
-            }else{
-                score += (f * share2allreturnScore + g * view2allreturnScore) * 0.01;
-            }
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault(hasReturnRovKey, "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double fmRov = item.getScoreRov();
+            item.getScoresMap().put("fmRov", fmRov);
+            score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
             video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
+            }
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
+            }
             result.add(video);
         }
         result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
+
         return result;
     }
 
-    public Double calScoreWeightNoTimeDecay(List<Double> data) {
-        Double up = 0.0;
-        Double down = 0.0;
-        for (int i = 0; i < data.size(); ++i) {
-            up += 1.0 * data.get(i);
-            down += 1.0;
-        }
-        return down > 1E-8 ? up / down : 0.0;
+    private Map<String, Map<String, String>> extractVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
     }
 
-    public List<Double> getRateData(List<Double> ups, List<Double> downs, Double up, Double down) {
-        List<Double> data = new LinkedList<>();
-        for (int i = 0; i < ups.size(); ++i) {
-            if (ExtractorUtils.isDoubleEqualToZero(downs.get(i) + down)) {
-                data.add(0.0);
-            } else {
-                data.add(
-                        (ups.get(i) + up) / (downs.get(i) + down)
-                );
-            }
-        }
-        return data;
+    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
     }
 
-    public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
-                                      List<String> datehours, String key) {
-        List<Double> views = new LinkedList<>();
-        Map<String, Double> tmp = itemRealMap.getOrDefault(key, new HashMap<>());
-        for (String dh : datehours) {
-            views.add(tmp.getOrDefault(dh, 0.0D) +
-                    (views.isEmpty() ? 0.0 : views.get(views.size() - 1))
-            );
-        }
-        return views;
+    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
     }
 
-    public List<RankItem> model(List<Video> videos, RankParam param,
-                                List<String> rtFeaPart) {
-        List<RankItem> result = new ArrayList<>();
-        if (videos.isEmpty()) {
-            return result;
-        }
-
-
-        List<RankItem> rankItems = CommonCollectionUtils.toList(videos, RankItem::new);
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(videos, Video::getVideoId);
-
-        // 2-2: item 实时特征处理
-        List<String> videoRtKeys2 = videoIds.stream().map(r -> "item_rt_fea_1h_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys2);
-
-
-        if (videoRtFeatures != null) {
-            int j = 0;
-            for (RankItem item : rankItems) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
+    private void readBucketFile() {
+        InputStream resourceStream = RankStrategy4RegionMergeModelV999.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
                         }
-                        vfMapNew.put(entry.getKey(), tmp);
                     }
-                    item.setItemRealTimeFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}", this.CLASS_NAME, e));
                 }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
             }
+        } else {
+            log.error("no bucket file");
         }
 
-        return rankItems;
     }
 
-    @Override
-    public RankResult mergeAndSort(RankParam param, List<Video> rovVideos, List<Video> flowVideos) {
-
-        //1 兜底策略，rov池子不足时，用冷启池填补。直接返回。
-        if (CollectionUtils.isEmpty(rovVideos)) {
-            if (param.getSize() < flowVideos.size()) {
-                return new RankResult(flowVideos.subList(0, param.getSize()));
-            } else {
-                return new RankResult(flowVideos);
-            }
-        }
-
-        //2 根据实验号解析阿波罗参数。
-        String abCode = param.getAbCode();
-        Map<String, Map<String, String>> rulesMap = this.filterRules.getOrDefault(abCode, new HashMap<>(0));
-
-        //3 标签读取
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankExtractorItemTags extractorItemTags = new RankExtractorItemTags(this.redisTemplate);
-            extractorItemTags.processor(rovVideos, flowVideos);
-        }
-        //6 合并结果时间卡控
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            RankProcessorTagFilter.processor(rovVideos, flowVideos, rulesMap);
-        }
-
-        //4 rov池提权功能
-        RankProcessorBoost.boostByTag(rovVideos, rulesMap);
+    static class Tuple4 {
+        public Map<String, String> first;
+        public Map<String, String> second;
+        public Map<String, String> third;
 
-        //5 rov池强插功能
-        RankProcessorInsert.insertByTag(param, rovVideos, rulesMap);
+        public String name;
 
-        //7 流量池按比例强插
-        List<Video> result = new ArrayList<>();
-        for (int i = 0; i < param.getTopK() && i < rovVideos.size(); i++) {
-            result.add(rovVideos.get(i));
-        }
-        double flowPoolP = getFlowPoolP(param);
-        int flowPoolIndex = 0;
-        int rovPoolIndex = param.getTopK();
-        for (int i = 0; i < param.getSize() - param.getTopK(); i++) {
-            double rand = RandomUtils.nextDouble(0, 1);
-            if (rand < flowPoolP) {
-                if (flowPoolIndex < flowVideos.size()) {
-                    result.add(flowVideos.get(flowPoolIndex++));
-                } else {
-                    break;
-                }
-            } else {
-                if (rovPoolIndex < rovVideos.size()) {
-                    result.add(rovVideos.get(rovPoolIndex++));
-                } else {
-                    break;
-                }
-            }
-        }
-        if (rovPoolIndex >= rovVideos.size()) {
-            for (int i = flowPoolIndex; i < flowVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(flowVideos.get(i));
-            }
-        }
-        if (flowPoolIndex >= flowVideos.size()) {
-            for (int i = rovPoolIndex; i < rovVideos.size() && result.size() < param.getSize(); i++) {
-                result.add(rovVideos.get(i));
-            }
+        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name) {
+            this.first = first;
+            this.second = second;
+            this.third = third;
+            this.name = name;
         }
 
-        //8 合并结果密度控制
-        Map<String, Integer> densityRules = new HashMap<>();
-        if (rulesMap != null && !rulesMap.isEmpty()) {
-            for (Map.Entry<String, Map<String, String>> entry : rulesMap.entrySet()) {
-                String key = entry.getKey();
-                Map<String, String> value = entry.getValue();
-                if (value.containsKey("density")) {
-                    densityRules.put(key, Integer.valueOf(value.get("density")));
-                }
-            }
-        }
-        Set<Long> videosSet = result.stream().map(Video::getVideoId).collect(Collectors.toSet());
-        List<Video> rovRecallRankNew = rovVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> flowPoolRankNew = flowVideos.stream().filter(r -> !videosSet.contains(r.getVideoId())).collect(Collectors.toList());
-        List<Video> resultWithDensity = RankProcessorDensity.mergeDensityControl(result,
-                rovRecallRankNew, flowPoolRankNew, densityRules);
-
-        return new RankResult(resultWithDensity);
     }
 
-    public static void main(String[] args) {
-//        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
-        String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
-        String down1 = "2024031012:2019,2024031013:1676,2024031014:1626,2024031015:1458,2024031016:1508,2024031017:1510,2024031018:1713,2024031019:1972,2024031020:2500,2024031021:2348,2024031022:2061,2024031023:1253,2024031100:659,2024031101:243,2024031102:191,2024031103:282,2024031104:246,2024031105:439,2024031106:1079,2024031107:1911,2024031108:2023,2024031109:1432,2024031110:1632,2024031111:1183,2024031112:1024,2024031113:938,2024031114:701,2024031115:541";
-
-//        String up2 = "2024031012:215,2024031013:242,2024031014:166,2024031015:194,2024031016:209,2024031017:245,2024031018:320,2024031019:332,2024031020:400,2024031021:375,2024031022:636,2024031023:316,2024031100:167,2024031101:45,2024031102:22,2024031103:26,2024031104:12,2024031105:22,2024031106:24,2024031107:143,2024031108:181,2024031109:199,2024031110:194,2024031111:330,2024031112:423,2024031113:421,2024031114:497,2024031115:424";
-        String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
-        String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
-
-        RankStrategy4RegionMergeModelV569 job = new RankStrategy4RegionMergeModelV569();
-        List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
-        Double d1 = job.calScoreWeightNoTimeDecay(l1);
-
-        System.out.println(d1);
+    static class Tuple2 {
+        public Map<String, String> first;
 
-        List<Double> l2 = job.getRateData(job.help(up2, "2024031115", 24), job.help(down2, "2024031115", 24), 1., 10.);
-        Double d2 = job.calScoreWeightNoTimeDecay(l2);
+        public String name;
 
-        System.out.println(d2);
-
-    }
-
-    List<Double> help(String s, String date, Integer h) {
-        Map<String, Double> maps = Arrays.stream(s.split(",")).map(pair -> pair.split(":"))
-                .collect(Collectors.toMap(
-                        arr -> arr[0],
-                        arr -> Double.valueOf(arr[1])
-                ));
-        List<String> datehours = new LinkedList<>(); // 时间是倒叙的
-        List<Double> result = new ArrayList<>();
-        for (int i = 0; i < h; ++i) {
-            Double d = (result.isEmpty() ? 0.0 : result.get(result.size() - 1));
-            result.add(d + maps.getOrDefault(date, 0D));
-            datehours.add(date);
-            date = ExtractorUtils.subtractHours(date, 1);
+        public Tuple2(Map<String, String> first, String name) {
+            this.first = first;
+            this.name = name;
         }
-        return result;
+
     }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV568.java → recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV656.java

@@ -8,17 +8,25 @@ import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemTags;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorBoost;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorDensity;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorInsert;
 import com.tzld.piaoquan.recommend.server.service.rank.processor.RankProcessorTagFilter;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.RandomUtils;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
 
 import java.text.SimpleDateFormat;
@@ -31,8 +39,8 @@ import java.util.stream.Collectors;
  */
 @Service
 @Slf4j
-public class RankStrategy4RegionMergeModelV568 extends RankService {
-    @ApolloJsonValue("${rank.score.merge.weightv568:}")
+public class RankStrategy4RegionMergeModelV656 extends RankService {
+    @ApolloJsonValue("${rank.score.merge.weightv656:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
     private Map<String, Map<String, Map<String, String>>> filterRules = new HashMap<>();
@@ -72,6 +80,7 @@ public class RankStrategy4RegionMergeModelV568 extends RankService {
         Set<Long> setVideo = new HashSet<>();
         this.duplicate(setVideo, v0);
 
+
         //-------------------相关性召回 融合+去重-------------------
         List<Video> v5 = extractAndSort(param, SimHotVideoRecallStrategy.PUSH_FORM);
         List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
@@ -120,19 +129,11 @@ public class RankStrategy4RegionMergeModelV568 extends RankService {
             datehours.add(cur);
             cur = ExtractorUtils.subtractHours(cur, 1);
         }
-        List<String> datehoursRoot = new LinkedList<>();
-        for (int i = 0; i < 24; ++i) {
-            datehoursRoot.add(String.valueOf(i+1));
-        }
-        // 2.1 item特征提取
-        this.getVideoFeatureFromRedis(items);
-
-
         for (RankItem item : items) {
-            Map<String, Map<String, Double>> itemRealRootMap = item.getItemRealTimeRootFeature();
-            List<Double> views = getStaticData(itemRealRootMap, datehoursRoot, "exp");
-            List<Double> shares = getStaticData(itemRealRootMap, datehoursRoot, "share");
-            List<Double> allreturns = getStaticData(itemRealRootMap, datehoursRoot, "return");
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_uv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
 
             // 全部回流的rov和ros
             List<Double> share2allreturn = getRateData(allreturns, shares, 0.0, 0.0);
@@ -350,45 +351,6 @@ public class RankStrategy4RegionMergeModelV568 extends RankService {
         return new RankResult(resultWithDensity);
     }
 
-    private void getVideoFeatureFromRedis(List<RankItem> items){
-        List<Long> videoIds = CommonCollectionUtils.toListDistinct(items, RankItem::getVideoId);
-        List<String> videoKeys = videoIds.stream().map(r -> "item_rt_fea_1hrootall_" + r)
-                .collect(Collectors.toList());
-        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoKeys);
-        int j = 0;
-        if (videoRtFeatures != null) {
-            for (RankItem item : items) {
-                String vF = videoRtFeatures.get(j);
-                ++j;
-                if (vF == null) {
-                    continue;
-                }
-                Map<String, String> vfMap = new HashMap<>();
-                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
-                try {
-                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {
-                    }, vfMap);
-                    for (Map.Entry<String, String> entry : vfMap.entrySet()) {
-                        String value = entry.getValue();
-                        if (value == null) {
-                            continue;
-                        }
-                        String[] var1 = value.split(",");
-                        Map<String, Double> tmp = new HashMap<>();
-                        for (String var2 : var1) {
-                            String[] var3 = var2.split(":");
-                            tmp.put(var3[0], Double.valueOf(var3[1]));
-                        }
-                        vfMapNew.put(entry.getKey(), tmp);
-                    }
-                    item.setItemRealTimeRootFeature(vfMapNew);
-                } catch (Exception e) {
-                    log.error(String.format("parse video item_rt_fea_1hrootall_ json is wrong in {} with {}", this.CLASS_NAME, e));
-                }
-            }
-        }
-    }
-
     public static void main(String[] args) {
 //        String up1 = "2024031012:513,2024031013:456,2024031014:449,2024031015:262,2024031016:414,2024031017:431,2024031018:643,2024031019:732,2024031020:927,2024031021:859,2024031022:866,2024031023:358,2024031100:133,2024031101:28,2024031102:22,2024031103:15,2024031104:21,2024031105:36,2024031106:157,2024031107:371,2024031108:378,2024031109:216,2024031110:269,2024031111:299,2024031112:196,2024031113:186,2024031114:85,2024031115:82";
         String up1 = "2024031012:1167,2024031013:1023,2024031014:947,2024031015:664,2024031016:842,2024031017:898,2024031018:1170,2024031019:1439,2024031020:2010,2024031021:1796,2024031022:1779,2024031023:722,2024031100:226,2024031101:50,2024031102:31,2024031103:30,2024031104:38,2024031105:63,2024031106:293,2024031107:839,2024031108:1250,2024031109:858,2024031110:767,2024031111:697,2024031112:506,2024031113:534,2024031114:381,2024031115:278";
@@ -398,7 +360,7 @@ public class RankStrategy4RegionMergeModelV568 extends RankService {
         String up2 = "2024031012:409,2024031013:464,2024031014:354,2024031015:474,2024031016:436,2024031017:636,2024031018:709,2024031019:741,2024031020:802,2024031021:904,2024031022:1112,2024031023:639,2024031100:378,2024031101:78,2024031102:47,2024031103:37,2024031104:17,2024031105:49,2024031106:103,2024031107:293,2024031108:457,2024031109:488,2024031110:558,2024031111:711,2024031112:785,2024031113:830,2024031114:974,2024031115:850";
         String down2 = "2024031012:748,2024031013:886,2024031014:788,2024031015:1029,2024031016:957,2024031017:1170,2024031018:1208,2024031019:1181,2024031020:1275,2024031021:1265,2024031022:1512,2024031023:1190,2024031100:1127,2024031101:486,2024031102:289,2024031103:254,2024031104:197,2024031105:310,2024031106:344,2024031107:693,2024031108:976,2024031109:1045,2024031110:1039,2024031111:1257,2024031112:1202,2024031113:1454,2024031114:1785,2024031115:1544";
 
-        RankStrategy4RegionMergeModelV568 job = new RankStrategy4RegionMergeModelV568();
+        RankStrategy4RegionMergeModelV656 job = new RankStrategy4RegionMergeModelV656();
         List<Double> l1 = job.getRateData(job.help(up1, "2024031115", 24), job.help(down1, "2024031115", 24), 1., 10.);
         Double d1 = job.calScoreWeightNoTimeDecay(l1);
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV999.java

@@ -0,0 +1,425 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Service
+@Slf4j
+public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeModelBasic {
+    @ApolloJsonValue("${rank.score.merge.weightv568:}")
+    private Map<String, Double> mergeWeight;
+
+    @Autowired
+    private FeatureService featureService;
+
+    Map<String, double[]> bucketsMap = new HashMap<>();
+    Map<String, Double> bucketsLen = new HashMap<>();
+
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
+        //-------------------融-------------------
+        //-------------------合-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        List<Video> oldRovs = new ArrayList<>();
+        oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, Region24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HRecallStrategy.PUSH_FORM));
+        oldRovs.addAll(extractAndSort(param, RegionRelative24HDupRecallStrategy.PUSH_FORM));
+        removeDuplicate(oldRovs);
+        int sizeReturn = param.getSize();
+        List<Video> v0 = oldRovs.size() <= sizeReturn
+                ? oldRovs
+                : oldRovs.subList(0, sizeReturn);
+        Set<Long> setVideo = new HashSet<>();
+        this.duplicate(setVideo, v0);
+        setVideo.addAll(v0.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        List<Video> rovRecallRank = new ArrayList<>(v0);
+        //-------------------return相似召回------------------
+        List<Video> v6 = extractAndSort(param, ReturnVideoRecallStrategy.PUSH_FORM);
+        v6 = v6.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
+        rovRecallRank.addAll(v6);
+        setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------新地域召回------------------
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        v1 = v1.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+        rovRecallRank.addAll(v1);
+        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        //-------------------7天ROVn召回------------------
+        List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
+        v2 = v2.stream().filter(r-> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+        v2 = v2.subList(0, Math.min(mergeWeight.getOrDefault("v2", 0.0).intValue(), v2.size()));
+        rovRecallRank.addAll(v2);
+        setVideo.addAll(v2.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+
+        //-------------------排-------------------
+        //-------------------序-------------------
+        //-------------------逻-------------------
+        //-------------------辑-------------------
+
+        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+
+        // k1:视频、k2:表、k3:特征、v:特征值
+        String provinceCn = param.getProvince().replaceAll("省$", "");
+        String headVid = String.valueOf(param.getHeadVid());
+        FeatureService.Feature feature = featureService.getFeature(param.getMid(), vids,
+                String.valueOf(param.getAppType()), provinceCn, headVid);
+        Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
+        Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
+
+
+        // TODO 2 特征处理
+        Map<String, Double> userFeatureMapDouble = new HashMap<>();
+        String mid = param.getMid();
+        Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
+        Map<String, String> c2 = featureOriginUser.getOrDefault("alg_mid_feature_share_and_return", new HashMap<>());
+        Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
+        Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags", new HashMap<>());
+        Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
+        Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
+
+        if (!c1.isEmpty()) {
+            userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
+            userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
+            userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
+            userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
+        }
+        if (!c2.isEmpty()) {
+            userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
+            userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
+            userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
+            userFeatureMapDouble.put("share_pv_7d", Double.parseDouble(c2.getOrDefault("share_pv_7d", "0")));
+            userFeatureMapDouble.put("return_uv_12h", Double.parseDouble(c2.getOrDefault("return_uv_12h", "0")));
+            userFeatureMapDouble.put("return_uv_1d", Double.parseDouble(c2.getOrDefault("return_uv_1d", "0")));
+            userFeatureMapDouble.put("return_uv_3d", Double.parseDouble(c2.getOrDefault("return_uv_3d", "0")));
+            userFeatureMapDouble.put("return_uv_7d", Double.parseDouble(c2.getOrDefault("return_uv_7d", "0")));
+        }
+
+        Map<String, String> c34567Map = new HashMap<>(15);
+        List<Tuple2> tmpList0 = Arrays.asList(
+                new Tuple2(c3, "c3_feature"),
+                new Tuple2(c4, "c4_feature"),
+                new Tuple2(c5, "c5_feature"),
+                new Tuple2(c6, "c6_feature"),
+                new Tuple2(c7, "c7_feature")
+        );
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                String tags = tuple2.first.getOrDefault(key_time, "");
+                if (!tags.isEmpty()) {
+                    c34567Map.put(tuple2.name + "_" + key_time, tags);
+                }
+            }
+        }
+
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
+        List<Tuple2> tmpList1 = Arrays.asList(
+                new Tuple2(c8, "c8_feature"),
+                new Tuple2(c9, "c9_feature")
+        );
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
+                String cfListStr = tuple2.first.getOrDefault(key_action, "");
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
+                    String[] entries = cfListStr.split(",");
+                    for (String entry : entries) {
+                        String[] rList = entry.split(":");
+                        if (rList.length >= 4) { // 确保分割后有四个元素
+                            String key = rList[0];
+                            String value1 = rList[1];
+                            String value2 = rList[2];
+                            String value3 = rList[3];
+                            String[] strs = {value1, value2, value3};
+                            cfMap.put(key, strs);
+                        }
+                    }
+                    c89Map.put(tuple2.name + "_" + key_action, cfMap);
+                }
+            }
+        }
+
+
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        for (RankItem item : rankItems) {
+            Map<String, Double> featureMap = new HashMap<>();
+            String vid = item.getVideoId() + "";
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp", new HashMap<>());
+            Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
+            Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share", new HashMap<>());
+            Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
+
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return", new HashMap<>());
+
+            List<Tuple4> originData = Arrays.asList(
+                    new Tuple4(b1, b2, b3, "b123"),
+                    new Tuple4(b1, b6, b7, "b167"),
+                    new Tuple4(b8, b9, b10, "b8910"),
+                    new Tuple4(b11, b12, b13, "b111213"),
+                    new Tuple4(b17, b18, b19, "b171819")
+            );
+
+            for (Tuple4 tuple4 : originData) {
+                for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
+
+                    double f1 = ExtractorUtils.calDiv(share, exp);
+                    double f2 = ExtractorUtils.calLog(share);
+                    double f3 = ExtractorUtils.calDiv(returns, exp);
+                    double f4 = ExtractorUtils.calLog(returns);
+                    double f5 = f3 * f4;
+
+                    String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
+                    String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
+                    String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
+                    String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
+                    String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
+
+                    featureMap.put(key1, f1);
+                    featureMap.put(key2, f2);
+                    featureMap.put(key3, f3);
+                    featureMap.put(key4, f4);
+                    featureMap.put(key5, f5);
+                }
+            }
+
+            Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            featureMap.put("total_time", Double.parseDouble(videoInfo.getOrDefault("total_time", "0")));
+            featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
+
+            String title = videoInfo.getOrDefault("title", "");
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+                        String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
+                        if (!tags.isEmpty()) {
+                            Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                            featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
+                            featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
+                            featureMap.put(name + "_" + key_time + "_avgscore", doubles[2]);
+                        }
+                    }
+                }
+            }
+
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
+                            Double score1 = Double.parseDouble(scores[0]);
+                            Double score2 = Double.parseDouble(scores[1]);
+                            Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
+                            featureMap.put(key_feature + "_" + key_action + "_score", score1);
+                            featureMap.put(key_feature + "_" + key_action + "_num", score2);
+                            featureMap.put(key_feature + "_" + key_action + "_rank", score3);
+                        }
+                    }
+                }
+            }
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
+            if (!d1.isEmpty()) {
+                featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
+                featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
+                featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
+            }
+            item.featureMapDouble = featureMap;
+        }
+
+        // 3 连续值特征分桶
+        readBucketFile();
+        Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
+            String name = entry.getKey();
+            Double score = entry.getValue();
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                Double bucketNum = this.bucketsLen.get(name);
+                double[] buckets = this.bucketsMap.get(name);
+                Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                userFeatureMap.put(name, String.valueOf(scoreNew));
+            }
+        }
+
+        for (RankItem item : rankItems) {
+            Map<String, String> featureMap = new HashMap<>();
+            Map<String, Double> featureMapDouble = item.featureMapDouble;
+
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
+                String name = entry.getKey();
+                Double score = entry.getValue();
+                // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
+                    Double bucketNum = this.bucketsLen.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    featureMap.put(name, String.valueOf(scoreNew));
+                }
+            }
+            item.featureMap = featureMap;
+        }
+
+        // TODO 3 排序
+        Map<String, String> sceneFeatureMap = new HashMap<>(0);
+
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240609.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        String redisScoreKey =  mergeWeight.getOrDefault("redisScoreKey", 0.0) < 0.5 ? "redis:vid_hasreturn_rov:" : "redis:vid_hasreturn_rov_7d:";
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
+        List<Video> result = new ArrayList<>();
+        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
+
+        for (RankItem item : items) {
+            double score = 0.0;
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault(hasReturnRovKey, "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double fmRov = item.getScoreRov();
+            item.getScoresMap().put("fmRov", fmRov);
+            if (chooseFunction == 0){
+                score = fmRov * (1 + hasReturnRovScore);
+            }else if (chooseFunction == 1){
+                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
+            }else {
+                score = fmRov * ExtractorUtils.sigmoid(hasReturnRovScore);
+            }
+
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId() + ""));
+            }
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
+            }
+            result.add(video);
+        }
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
+
+        return result;
+    }
+
+    private Map<String, Map<String, String>> extractVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
+    }
+
+    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
+    }
+
+    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
+        // TODO
+        return null;
+    }
+
+    private void readBucketFile() {
+        InputStream resourceStream = RankStrategy4RegionMergeModelV999.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        if (resourceStream != null) {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                Map<String, double[]> bucketsMap = new HashMap<>();
+                Map<String, Double> bucketsLen = new HashMap<>();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    // 替换空格和换行符，过滤空行
+                    line = line.replace(" ", "").replaceAll("\n", "");
+                    if (!line.isEmpty()) {
+                        String[] rList = line.split("\t");
+                        if (rList.length == 3) {
+                            String key = rList[0];
+                            double value1 = Double.parseDouble(rList[1]);
+                            bucketsLen.put(key, value1);
+                            double[] value2 = Arrays.stream(rList[2].split(","))
+                                    .mapToDouble(Double::valueOf)
+                                    .toArray();
+                            bucketsMap.put(key, value2);
+                        }
+                    }
+                }
+                this.bucketsMap = bucketsMap;
+                this.bucketsLen = bucketsLen;
+            } catch (IOException e) {
+                log.error("something is wrong in parse bucket file:" + e);
+            }
+        } else {
+            log.error("no bucket file");
+        }
+
+    }
+
+    static class Tuple4 {
+        public Map<String, String> first;
+        public Map<String, String> second;
+        public Map<String, String> third;
+
+        public String name;
+
+        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name) {
+            this.first = first;
+            this.second = second;
+            this.third = third;
+            this.name = name;
+        }
+
+    }
+
+    static class Tuple2 {
+        public Map<String, String> first;
+
+        public String name;
+
+        public Tuple2(Map<String, String> first, String name) {
+            this.first = first;
+            this.name = name;
+        }
+
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -1,4 +1,5 @@
 package com.tzld.piaoquan.recommend.server.service.recall;
+
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.common.base.Constant;
@@ -7,6 +8,8 @@ import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.filter.strategy.BlacklistContainer;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -57,11 +60,18 @@ public class RecallService implements ApplicationContextAware {
 
     public RecallResult recall(RecallParam param) {
         List<RecallStrategy> strategies = getRecallStrategy(param);
+//        log.info("strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies,
+//        o -> o.getClass().getSimpleName())));
         CountDownLatch cdl = new CountDownLatch(strategies.size());
         List<Future<RecallResult.RecallData>> recallResultFutures = new ArrayList<>();
         for (final RecallStrategy strategy : strategies) {
             Future<RecallResult.RecallData> future = pool.submit(() -> {
-                List<Video> result = strategy.recall(param);
+                List<Video> result = Collections.emptyList();
+                try {
+                    result = strategy.recall(param);
+                } catch (Throwable e) {
+                    log.error("recall error {}", strategy.getClass().getSimpleName(), e);
+                }
                 cdl.countDown();
                 return new RecallResult.RecallData(strategy.pushFrom(), result);
             });
@@ -94,9 +104,9 @@ public class RecallService implements ApplicationContextAware {
             return strategies;
         }
 
-            String matchUserBlacklistTypeEnum = blacklistContainer.matchUserBlacklistTypeEnum(param.getUid(), param.getHotSceneType(), param.getCityCode(),
-                    param.getClientIp(), param.getMid(), "recommend-flow-pool", param.getAppType());
-        boolean hitUserBlacklist =  StringUtils.isNotBlank(matchUserBlacklistTypeEnum);
+        String matchUserBlacklistTypeEnum = blacklistContainer.matchUserBlacklistTypeEnum(param.getUid(), param.getHotSceneType(), param.getCityCode(),
+                param.getClientIp(), param.getMid(), "recommend-flow-pool", param.getAppType());
+        boolean hitUserBlacklist = StringUtils.isNotBlank(matchUserBlacklistTypeEnum);
         boolean isInBlacklist = CollectionUtils.isNotEmpty(blacklistAppTypeSet) && blacklistAppTypeSet.contains(param.getAppType());
 
         String abCode = param.getAbCode();
@@ -114,17 +124,6 @@ public class RecallService implements ApplicationContextAware {
             return strategies;
         }
         switch (abCode) {
-            case "60113": // 563
-            case "60114": // 564
-            case "60115": // 565
-                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
-                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2_sort.class.getSimpleName()));
-                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
-                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
-                strategies.addAll(getRegionRecallStrategy(param));
-                break;
-            case "60105": // 551
-            case "60106": // 552
             case "60107": // 553
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1_sort.class.getSimpleName()));
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2_sort.class.getSimpleName()));
@@ -160,13 +159,27 @@ public class RecallService implements ApplicationContextAware {
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
                 break;
             case "60117": // 567
+            case "60656": // 656
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
+                strategies.addAll(getRegionRecallStrategy(param));
+                break;
             case "60118": // 568
+                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                 strategies.addAll(getRegionRecallStrategy(param));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                break;
+            case "60105": // 551
+            case "60106": // 552
+            case "60112": // 562
+            case "60113": // 563
+            case "60114": // 564
+            case "60115": // 565
             case "60119": // 569
                 strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
-                strategies.add(strategyMap.get(TitleTagRecallStrategyV1.class.getSimpleName()));
                 strategies.addAll(getRegionRecallStrategy(param));
+                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                break;
             default:
                 strategies.addAll(getRegionRecallStrategy(param));
         }
@@ -200,10 +213,10 @@ public class RecallService implements ApplicationContextAware {
                     strategies.add(strategyMap.get(QuickFlowPoolWithLevelRecallStrategy.class.getSimpleName()));
                     if ("60126".equals(abCode) || "60125".equals(abCode) || "60124".equals(abCode)
                             || "60105".equals(abCode) || "60106".equals(abCode) || "60107".equals(abCode)
-                            || "60113".equals(abCode) || "60114".equals(abCode)
+                            || "60112".equals(abCode) || "60113".equals(abCode) || "60114".equals(abCode)
                             || "60115".equals(abCode) || "60117".equals(abCode) || "60118".equals(abCode)
                             || "60119".equals(abCode) || "60150".equals(abCode) || "60151".equals(abCode)
-                            || "60654".equals(abCode) || "60655".equals(abCode)
+                            || "60654".equals(abCode) || "60655".equals(abCode) || "60656".equals(abCode)
                     ) {
                         strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategyTomson.class.getSimpleName()));
                     } else {
@@ -263,33 +276,27 @@ public class RecallService implements ApplicationContextAware {
                 strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
                 strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
                 break;
-            case "60112": // 562
-                strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV5Hand.class.getSimpleName()));
-                break;
             case "60121": // 536
             case "60122": // 537
             case "60124": // 546
             case "60125": // 547
             case "60123": // 541
             case "60126": // 548
-            case "60105": // 551
-            case "60106": // 552
             case "60107": // 553
             case "60116": // 566
-            case "60119": // 569
-                strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+            case "60117": // 567
+            case "60656": // 656
+            case "60104": // 去掉sim的对比实验
                 strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
-                strategies.add(strategyMap.get(FestivalRecallStrategyV1.class.getSimpleName()));
                 break;
+            case "60105": // 551
+            case "60106": // 552
+            case "60112": // 562
             case "60113": // 563
             case "60114": // 564
             case "60115": // 565
-            case "60117": // 567
             case "60118": // 568
-            case "60104": // 去掉sim的对比实验
-                strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+            case "60119": // 569
                 break;
             case "60110": // 新内容的召回（流量池的Top内容）
                 strategies.add(strategyMap.get(TopGoodPerformanceVideoRecallStrategy.class.getSimpleName()));
@@ -364,13 +371,13 @@ public class RecallService implements ApplicationContextAware {
         this.applicationContext = applicationContext;
     }
 
-    private boolean matchSpecialApp(int appId){
+    private boolean matchSpecialApp(int appId) {
         Set<Integer> notSpecialApp = new HashSet<>(Arrays.asList(0, 4, 5));
-        if (notSpecialApp.contains(appId)){
+        if (notSpecialApp.contains(appId)) {
             // vlog 票圈视频 内容精选 不允许走特殊列表，即使配置了也无效。
             return false;
         }
-        if (specialAppVid != null && specialAppVid.getOrDefault("app", new ArrayList<>()).contains((long) appId)){
+        if (specialAppVid != null && specialAppVid.getOrDefault("app", new ArrayList<>()).contains((long) appId)) {
             log.info("This request hits a special logic in matchSpecialApp with appId={}", appId);
             return true;
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/FlowPoolWithLevelRecallStrategyTomson.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service.recall.strategy;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.collect.Lists;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
@@ -10,6 +11,7 @@ import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
 import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -32,7 +34,8 @@ import static com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConsta
 @Service
 @Slf4j
 public class FlowPoolWithLevelRecallStrategyTomson extends AbstractFlowPoolWithLevelRecallStrategy {
-
+    @ApolloJsonValue("${ifOneLevelRandom:true}")
+    private boolean ifOneLevelRandom;
     @Autowired
     private FlowPoolConfigService flowPoolConfigService;
 
@@ -53,6 +56,8 @@ public class FlowPoolWithLevelRecallStrategyTomson extends AbstractFlowPoolWithL
                 availableLevels.add(lw);
             }
         }
+
+        //log.info("availableLevels {}", JSONUtils.toJson(availableLevels));
         if (CollectionUtils.isEmpty(availableLevels)) {
             return Pair.of("", "");
         }
@@ -125,31 +130,43 @@ public class FlowPoolWithLevelRecallStrategyTomson extends AbstractFlowPoolWithL
             videoFlowPoolMap.put(values[0], values[1]);
             videoFlowPoolMap_.put(NumberUtils.toLong(values[0], 0), values[1]);
         }
-        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_tomson.conf");
-        List<List<Pair<Long, Double>>> results = pipeline.recall(videoFlowPoolMap);
-        List<Pair<Long, Double>> result = results.get(0);
-        Map<Long, Double> resultmap = result.stream()
-                .collect(Collectors.toMap(
-                        Pair::getLeft, // 键是Pair的left值
-                        Pair::getRight, // 值是Pair的right值
-                        (existingValue, newValue) -> existingValue, // 如果键冲突，选择保留现有的值（或者你可以根据需要定义其他合并策略）
-                        LinkedHashMap::new // 使用LinkedHashMap来保持插入顺序（如果需要的话）
-                ));
+        Map<Long, Double> resultmap = null;
+        if ("1".equals(level) && ifOneLevelRandom) {
+            // 流量池一层改为全随机
+            int limitSize = 60;
+            List<Long> keyList = new ArrayList<>(videoFlowPoolMap_.keySet());
+            Collections.shuffle(keyList);
+            resultmap = keyList.stream().limit(limitSize).collect(Collectors.toMap(
+                    key -> key,
+                    key -> Math.random()
+            ));
+        } else {
+            ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_tomson.conf");
+            List<List<Pair<Long, Double>>> results = pipeline.recall(videoFlowPoolMap);
+            List<Pair<Long, Double>> result = results.get(0);
+            resultmap = result.stream()
+                    .collect(Collectors.toMap(
+                            Pair::getLeft, // 键是Pair的left值
+                            Pair::getRight, // 值是Pair的right值
+                            (existingValue, newValue) -> existingValue, // 如果键冲突，选择保留现有的值（或者你可以根据需要定义其他合并策略）
+                            LinkedHashMap::new // 使用LinkedHashMap来保持插入顺序（如果需要的话）
+                    ));
+        }
+
         // 3 召回内部过滤
-        FilterParam filterParam = FilterParamFactory.create(param, result.stream()
-                .map(Pair::getLeft)
-                .collect(Collectors.toList()));
+        FilterParam filterParam = FilterParamFactory.create(param, new ArrayList<>(resultmap.keySet()));
         filterParam.setForceTruncation(10000);
         filterParam.setConcurrent(true);
         filterParam.setNotUsePreView(false);
         FilterResult filterResult = filterService.filter(filterParam);
         List<Video> videosResult = new ArrayList<>();
         if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            Map<Long, Double> finalResultmap = resultmap;
             filterResult.getVideoIds().forEach(vid -> {
                 Video recallData = new Video();
                 recallData.setVideoId(vid);
                 recallData.setAbCode(param.getAbCode());
-                recallData.setRovScore(resultmap.getOrDefault(vid, 0.0));
+                recallData.setRovScore(finalResultmap.getOrDefault(vid, 0.0));
                 recallData.setPushFrom(pushFrom());
                 recallData.setFlowPool(videoFlowPoolMap_.get(vid));
                 recallData.setFlowPoolAbtestGroup(param.getFlowPoolAbtestGroup());

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV4.java

@@ -57,10 +57,10 @@ public class RegionRealtimeRecallStrategyV4 implements RecallStrategy {
                 videosResult.add(video);
             });
         }
-        Collections.sort(videosResult, Comparator.comparingDouble(o -> -o.getRovScore()));
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
         return videosResult;
     }
-    public static final String PUSH_FORM = "recall_strategy_noregion_1h";
+    public static final String PUSH_FORM = "recall_strategy_rovn_7d";
     @Override
     public String pushFrom(){
         return PUSH_FORM;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/BaseFMModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+import com.tzld.piaoquan.recommend.server.service.score.model.FMModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseFMModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseFMModelScorer.class);
+
+    public BaseFMModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(FMModel.class);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/BaseLRV2ModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+import com.tzld.piaoquan.recommend.server.service.score.model.LRV2Model;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseLRV2ModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRV2ModelScorer.class);
+
+    public BaseLRV2ModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(LRV2Model.class);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/BaseXGBoostModelScorer.java

@@ -0,0 +1,33 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+import com.google.common.reflect.TypeToken;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import com.tzld.piaoquan.recommend.server.service.score.model.XGBoostModel;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+
+public abstract class BaseXGBoostModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseXGBoostModelScorer.class);
+
+    public BaseXGBoostModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(XGBoostModel.class);
+        XGBoostModel model = (XGBoostModel) this.getModel();
+        ConfigObject paramMap = scorerConfigInfo.getParamMap();
+        if (paramMap != null) {
+            ConfigValue value = paramMap.get("features");
+            List<String> features = (List<String>) value.unwrapped();
+            model.setFeatures(features.toArray(new String[features.size()]));
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerConfig.java

@@ -1,18 +1,17 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
+import com.google.common.reflect.TypeToken;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import com.typesafe.config.ConfigObject;
 import com.typesafe.config.ConfigValue;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 
 
 public class ScorerConfig {
@@ -100,6 +99,12 @@ public class ScorerConfig {
             if (conf.hasPath("disable-switch")) {
                 disableSwitch = conf.getBoolean("disable-switch");
             }
+            ConfigObject paramMap = null;
+            if (conf.hasPath("param")) {
+                paramMap = conf.getObject("param");
+            }
+
+
             Config paramConfig = loadOptionConfig(conf, "param-config");
             // model path
             String modelPath = loadOptionStringConfig(conf, "model-path");
@@ -118,7 +123,8 @@ public class ScorerConfig {
                     disableSwitch,
                     enableQueues,
                     modelPath,
-                    paramConfig
+                    paramConfig,
+                    paramMap
             );
             LOGGER.debug("parse scorer config info [{}]", configInfo);
             // add to ConfigInfoList

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerConfigInfo.java

@@ -2,6 +2,10 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 import com.google.gson.Gson;
 import com.typesafe.config.Config;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+
+import java.util.Map;
 import java.util.Set;
 
 
@@ -14,6 +18,7 @@ public class ScorerConfigInfo {
     private Set<String> enableQueues;
     private String modelPath;
     private Config paramConfig; // param config
+    private ConfigObject paramMap;
 
     public ScorerConfigInfo(String configName,
                             String scorerName,
@@ -21,7 +26,8 @@ public class ScorerConfigInfo {
                             Boolean disableSwitch,
                             Set<String> enableQueues,
                             String modelPath,
-                            Config paramConfig) {
+                            Config paramConfig,
+                            ConfigObject paramMap) {
 
         this.configName = configName;
         this.scorerName = scorerName;
@@ -30,6 +36,7 @@ public class ScorerConfigInfo {
         this.enableQueues = enableQueues;
         this.modelPath = modelPath;
         this.paramConfig = paramConfig;
+        this.paramMap = paramMap;
     }
 
     public Config getParamConfig() {
@@ -70,6 +77,10 @@ public class ScorerConfigInfo {
         return disableSwitch;
     }
 
+    public ConfigObject getParamMap(){
+        return paramMap;
+    }
+
     @Override
     public String toString() {
         return new Gson().toJson(this);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -34,6 +34,11 @@ public final class ScorerUtils {
         ScorerUtils.init(FLOWPOOL_CONF);
         ScorerUtils.init(VIDEO_SCORE_CONF_FOR_AD);
         ScorerUtils.init("feeds_score_config_20240228.conf");
+        ScorerUtils.init("feeds_score_config_20240609.conf");
+        ScorerUtils.init("feeds_score_config_20240711.conf");
+        ScorerUtils.init("feeds_score_config_20240806.conf");
+        ScorerUtils.init("feeds_score_config_20240807.conf");
+        ScorerUtils.init("feeds_score_config_xgb_20240828.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v1.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v2.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v3.conf");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogRovFMScorer.java

@@ -0,0 +1,161 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.FMModel;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class VlogRovFMScorer extends BaseFMModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogRovFMScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public VlogRovFMScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        FMModel model = (FMModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        FMModel model = (FMModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final FMModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final FMModel model,
+                            final RankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        Map<String, String> featureMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+            featureMap.putAll(item.getFeatureMap());
+        }
+        if (MapUtils.isNotEmpty(userFeatureMap)) {
+            featureMap.putAll(userFeatureMap);
+        }
+        if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+            featureMap.putAll(sceneFeatureMap);
+        }
+
+        double pro = 0.0;
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                pro = model.score(featureMap);
+                // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setScoreRov(pro);
+        item.getScoresMap().put("RovFMScore", pro);
+        item.setAllFeatureMap(featureMap);
+        return pro;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogRovLRScorer.java

@@ -0,0 +1,159 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.LRV2Model;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class VlogRovLRScorer extends BaseLRV2ModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogRovLRScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public VlogRovLRScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRV2Model model = (LRV2Model) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        LRV2Model model = (LRV2Model) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final LRV2Model model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final LRV2Model lrModel,
+                            final RankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        Map<String, String> featureMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+            featureMap.putAll(item.getFeatureMap());
+        }
+        if (MapUtils.isNotEmpty(userFeatureMap)) {
+            featureMap.putAll(userFeatureMap);
+        }
+        if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+            featureMap.putAll(sceneFeatureMap);
+        }
+
+        double pro = 0.0;
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                pro = lrModel.score(featureMap);
+                // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setScoreRov(pro);
+        return pro;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/XGBoostScorer.java

@@ -0,0 +1,159 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.XGBoostModel;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class XGBoostScorer extends BaseXGBoostModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(XGBoostScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public XGBoostScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        XGBoostModel model = (XGBoostModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        XGBoostModel model = (XGBoostModel) this.getModel();
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final XGBoostModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final XGBoostModel model,
+                            final RankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        Map<String, String> featureMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+            featureMap.putAll(item.getFeatureMap());
+        }
+        if (MapUtils.isNotEmpty(userFeatureMap)) {
+            featureMap.putAll(userFeatureMap);
+        }
+        if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+            featureMap.putAll(sceneFeatureMap);
+        }
+
+        double pro = 0.0;
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                pro = model.score(featureMap);
+                // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setScoreRov(pro);
+        item.getScoresMap().put("RovFMScore", pro);
+        item.setAllFeatureMap(featureMap);
+        return pro;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/FMModel.java

@@ -0,0 +1,144 @@
+package com.tzld.piaoquan.recommend.server.service.score.model;
+
+
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class FMModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(FMModel.class);
+    private Map<String, List<Float>> model;
+
+    public void putFeature(Map<String, List<Float>> model, String[] items) {
+
+        String featureKey = items[0];
+        List<Float> weights = new ArrayList<>();
+        for (int i = 1; i < items.length; i++) {
+            weights.add(Float.valueOf(items[i]));
+        }
+
+        model.put(featureKey, weights);
+    }
+
+    public float getWeight(Map<String, List<Float>> model, String featureKey, int index) {
+        if (!model.containsKey(featureKey)) {
+            return 0.0f;
+        }
+
+        return model.get(featureKey).get(index);
+
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.model == null)
+            return 0;
+        return model.size();
+    }
+
+    public void cleanModel() {
+        this.model = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+        float sum = 0.0f;
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+            // 计算 sum w*x
+            float sum0 = 0.0f;
+            for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                float x = NumberUtils.toFloat(e.getValue(), 0.0f);
+                float w = getWeight(this.model, e.getKey(), 0);
+                sum0 += w * x;
+            }
+            sum += sum0;
+
+            // 计算 sum v*v*x*X
+            float sum1 = 0.0f;
+            for (int i = 1; i < 9; i++) {
+                float sum10 = 0.0f;
+                float sum11 = 0.0f;
+                for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                    float x = NumberUtils.toFloat(e.getValue(), 0.0f);
+                    float v = getWeight(this.model, e.getKey(), i);
+                    float d = v * x;
+                    sum10 += d;
+                    sum11 += d * d;
+                }
+                sum1 += sum10 * sum10 - sum11;
+            }
+            sum1 = 0.5f * sum1;
+            float biasW = model.get("bias").get(0);
+            sum = biasW + sum0 + sum1;
+        }
+
+        return (float) (1.0f / (1 + Math.exp(-sum)));
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        Map<String, List<Float>> model = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", model.size(), curTime);
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 9) {
+                if (items[0].equals("bias")) {
+                    putFeature(model, items);
+                }
+                continue;
+            }
+
+            putFeature(model, items);
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.model = model;
+
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", model.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 9) {
+                continue;
+            }
+            putFeature(model, items);
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", model.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/LRV2Model.java

@@ -0,0 +1,111 @@
+package com.tzld.piaoquan.recommend.server.service.score.model;
+
+
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class LRV2Model extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(LRV2Model.class);
+    private Map<String, Float> lrModel;
+
+    public void putFeature(Map<String, Float> model, String featureKey, float weight) {
+        model.put(featureKey, weight);
+    }
+
+    public float getWeight(Map<String, Float> model, String featureKey) {
+        return model.getOrDefault(featureKey, 0.0f);
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.lrModel == null)
+            return 0;
+        return lrModel.size();
+    }
+
+    public void cleanModel() {
+        this.lrModel = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+        float sum = 0.0f;
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+            for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                float w = getWeight(this.lrModel, e.getKey());
+                sum += w * NumberUtils.toFloat(e.getValue(), 0.0f);
+            }
+
+            float biasW = lrModel.get("bias");
+            sum += biasW;
+        }
+
+
+        return (float) (1.0f / (1 + Math.exp(-sum)));
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        Map<String, Float> model = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+
+            putFeature(model, items[0], Float.valueOf(items[1].trim()).floatValue());
+            if (cnt++ < 10) {
+                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
+            }
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.lrModel = model;
+
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            putFeature(model, items[0], Float.valueOf(items[1]).floatValue());
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/Model.java

@@ -1,11 +1,15 @@
 package com.tzld.piaoquan.recommend.server.service.score.model;
 
 
+import java.io.InputStream;
 import java.io.InputStreamReader;
 
 abstract public class Model {
     public abstract int getModelSize();
 
     public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
+    public boolean loadFromStream(InputStream is) throws Exception {
+        return loadFromStream(new InputStreamReader(is));
+    }
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/ModelManager.java

@@ -205,14 +205,14 @@ public class ModelManager {
                         loadTask.lastModifyTime, timeStamp);
 
                 Model model = loadTask.modelClass.newInstance();
-                if (model.loadFromStream(new InputStreamReader(ossObj.getObjectContent()))) {
+                if (model.loadFromStream(ossObj.getObjectContent())) {
                     loadTask.model = model;
                     loadTask.lastModifyTime = timeStamp;
                 }
             }
             ossObj.close();
         } catch (Exception e) {
-            log.error("update model fail", e);
+            //log.error("update model fail", e);
             return false;
         } finally {
             loadTask.isLoading = false;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/XGBoostModel.java

@@ -0,0 +1,71 @@
+package com.tzld.piaoquan.recommend.server.service.score.model;
+
+
+import com.tzld.piaoquan.recommend.server.util.CompressUtil;
+import com.tzld.piaoquan.recommend.server.util.PropertiesUtil;
+import ml.dmlc.xgboost4j.scala.DMatrix;
+import ml.dmlc.xgboost4j.scala.spark.XGBoostClassificationModel;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Map;
+
+
+public class XGBoostModel extends Model {
+    private static final Logger LOGGER = LoggerFactory.getLogger(XGBoostModel.class);
+    private XGBoostClassificationModel model;
+
+    private String[] features;
+
+    public void setFeatures(String[] features){
+        this.features = features;
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.model == null)
+            return 0;
+        return 1;
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws Exception {
+        return false;
+    }
+
+    public void cleanModel() {
+        this.model = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+
+        try {
+            float[] values = new float[features.length];
+            for (int i = 0; i < features.length; i++) {
+                float v = NumberUtils.toFloat(featureMap.getOrDefault(features[i], "0.0"), 0.0f);
+                values[i] = v;
+            }
+            DMatrix dm = new DMatrix(values, 1, features.length, 0.0f);
+            float[][] result = model._booster().predict(dm, false, 100);
+            return result[0][0];
+        } catch (Exception e) {
+            return 0f;
+        }
+    }
+
+    @Override
+    public boolean loadFromStream(InputStream in) throws Exception {
+        String modelDir = PropertiesUtil.getString("model.xgboost.path");
+        CompressUtil.decompressGzFile(in, modelDir);
+        String absolutePath =new File(modelDir).getAbsolutePath();
+        XGBoostClassificationModel model2 = XGBoostClassificationModel.load("file://" + absolutePath);
+        model2.setMissing(0.0f);
+        this.model = model2;
+        return true;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/model4recall/Model4RecallList.java

@@ -8,6 +8,7 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 
@@ -39,6 +40,7 @@ public class Model4RecallList extends AbstractModel {
                 LOGGER.error(String.format("something is wrong with parse pair %s %s: ", id, score), e);
             }
         }
+        this.recallList.sort(Comparator.comparingDouble(o -> -o.getRight()));
         input.close();
         in.close();
         return true;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/FlowPoolScorer.java

@@ -54,5 +54,10 @@ public class FlowPoolScorer extends AbstractScorer4Recall {
         return id2BetaScore.subList(0, Math.min(60, id2BetaScore.size()));
     }
 
+    public static void main(String[] args) {
+        BetaDistribution betaSample;
+        betaSample = new BetaDistribution(0.0, 0.0);
+        System.out.println(betaSample.sample());
+    }
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/RegionRecallScorerV4.java

@@ -26,7 +26,7 @@ public class RegionRecallScorerV4 extends AbstractScorer4Recall {
         Model4RecallList model = (Model4RecallList) this.getModel();
         List<Pair<Long, Double>> lists = model.recallList;
         lists.sort(Comparator.comparingDouble(o -> -o.getRight()));
-        return lists.subList(0, Math.min(100, lists.size()));
+        return lists.subList(0, Math.min(400, lists.size()));
     }
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/CompressUtil.java

@@ -0,0 +1,123 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * @author dyp
+ */
+@Slf4j
+public class CompressUtil {
+    public static void compressDirectoryToGzip(String sourceDirPath, String outputFilePath) {
+        // 创建.gz文件的输出流
+        try (OutputStream out = new FileOutputStream(outputFilePath);
+             GzipCompressorOutputStream gzipOut = new GzipCompressorOutputStream(out);
+             TarArchiveOutputStream taos = new TarArchiveOutputStream(gzipOut)) {
+
+            taos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
+
+            // 遍历目录
+            Files.walk(Paths.get(sourceDirPath))
+                    .filter(Files::isRegularFile)
+                    .forEach(filePath -> {
+                        try {
+                            // 为每个文件创建TarEntry
+                            TarArchiveEntry entry = new TarArchiveEntry(filePath.toFile(), filePath.toString().substring(sourceDirPath.length() + 1));
+                            taos.putArchiveEntry(entry);
+
+                            // 读取文件内容并写入TarArchiveOutputStream
+                            try (InputStream is = Files.newInputStream(filePath)) {
+                                byte[] buffer = new byte[1024];
+                                int len;
+                                while ((len = is.read(buffer)) > 0) {
+                                    taos.write(buffer, 0, len);
+                                }
+                            }
+                            // 关闭entry
+                            taos.closeArchiveEntry();
+                        } catch (IOException e) {
+                            log.error("", e);
+                        }
+                    });
+        } catch (Exception e) {
+            log.error("", e);
+        }
+    }
+
+    public static void decompressGzFile(String gzipFilePath, String destDirPath) {
+        try (InputStream gzipIn = new FileInputStream(gzipFilePath);
+             GzipCompressorInputStream gzIn = new GzipCompressorInputStream(gzipIn);
+             TarArchiveInputStream tais = new TarArchiveInputStream(gzIn)) {
+
+            TarArchiveEntry entry;
+            Files.createDirectories(Paths.get(destDirPath));
+            while ((entry = tais.getNextTarEntry()) != null) {
+                if (entry.isDirectory()) {
+                    // 如果是目录，创建目录
+                    Files.createDirectories(Paths.get(destDirPath, entry.getName()));
+                } else {
+                    // 如果是文件，创建文件并写入内容
+                    File outputFile = new File(destDirPath, entry.getName());
+                    if (!outputFile.exists()) {
+                        File parent = outputFile.getParentFile();
+                        if (!parent.exists()) {
+                            parent.mkdirs();
+                        }
+                        outputFile.createNewFile();
+                    }
+                    try (OutputStream out = new FileOutputStream(outputFile)) {
+                        byte[] buffer = new byte[1024];
+                        int len;
+                        while ((len = tais.read(buffer)) > 0) {
+                            out.write(buffer, 0, len);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("", e);
+        }
+    }
+
+    public static void decompressGzFile(InputStream gzipIn, String destDirPath) {
+        try (GzipCompressorInputStream gzIn = new GzipCompressorInputStream(gzipIn);
+             TarArchiveInputStream tais = new TarArchiveInputStream(gzIn)) {
+
+            TarArchiveEntry entry;
+            Files.createDirectories(Paths.get(destDirPath));
+            while ((entry = tais.getNextTarEntry()) != null) {
+                if (entry.isDirectory()) {
+                    // 如果是目录，创建目录
+                    Files.createDirectories(Paths.get(destDirPath, entry.getName()));
+                } else {
+                    // 如果是文件，创建文件并写入内容
+                    File outputFile = new File(destDirPath, entry.getName());
+                    if (!outputFile.exists()) {
+                        File parent = outputFile.getParentFile();
+                        if (!parent.exists()) {
+                            parent.mkdirs();
+                        }
+                        outputFile.createNewFile();
+                    }
+                    try (OutputStream out = new FileOutputStream(outputFile)) {
+                        byte[] buffer = new byte[1024];
+                        int len;
+                        while ((len = tais.read(buffer)) > 0) {
+                            out.write(buffer, 0, len);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("", e);
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/PropertiesUtil.java

@@ -0,0 +1,22 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import org.springframework.context.EnvironmentAware;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+@Component
+public class PropertiesUtil implements EnvironmentAware {
+
+
+    private static Environment environment;
+
+
+    @Override
+    public void setEnvironment(Environment environment) {
+        this.environment = environment;
+    }
+
+    public static String getString(String name) {
+        return environment.getProperty(name);
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/web/RecommendController.java

@@ -1,12 +1,15 @@
 package com.tzld.piaoquan.recommend.server.web;
 
 import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.tzld.piaoquan.recommend.server.client.ProtobufUtils;
 import com.tzld.piaoquan.recommend.server.client.RecommendHttpRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendResponse;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.VideoRecommendService;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
@@ -26,6 +29,15 @@ public class RecommendController {
     @Autowired
     private VideoRecommendService videoRecommendService;
 
+    @Autowired
+    private FeatureService featureService;
+
+    @RequestMapping("/demo")
+    public String demo() {
+//        return JSONUtils.toJson(featureService.getFeature(Lists.newArrayList("10054148", "20637161"), "0", "1"));
+        return "";
+    }
+
     @RequestMapping("/homepage/recommend")
     public String homepageRecommend(@RequestBody RecommendHttpRequest httpRequest) {
         MDC.put("appType", String.valueOf(httpRequest.getAppType()));

recommend-server-service/src/main/resources/application.yml

@@ -58,4 +58,7 @@ feign:
     config:
       default:
         connectTimeout: 2000
-        readTimeout: 10000
+        readTimeout: 10000
+model:
+  xgboost:
+    path: xgboost

recommend-server-service/src/main/resources/feeds_score_config_20240609.conf

@@ -0,0 +1,7 @@
+scorer-config = {
+  rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogRovFMScorer"
+    scorer-priority = 96
+    model-path = "zhangbo/model_aka8.txt"
+  }
+}

recommend-server-service/src/main/resources/feeds_score_config_20240711.conf

@@ -0,0 +1,7 @@
+scorer-config = {
+  rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogRovFMScorer"
+    scorer-priority = 95
+    model-path = "zhangbo/model_nba8.txt"
+  }
+}

recommend-server-service/src/main/resources/feeds_score_config_20240806.conf

@@ -0,0 +1,7 @@
+scorer-config = {
+  rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogRovFMScorer"
+    scorer-priority = 96
+    model-path = "zhangbo/model_aka8_new1.txt"
+  }
+}

recommend-server-service/src/main/resources/feeds_score_config_20240807.conf

@@ -0,0 +1,7 @@
+scorer-config = {
+  rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogRovFMScorer"
+    scorer-priority = 96
+    model-path = "zhangbo/model_aka8_new2.txt"
+  }
+}

recommend-server-service/src/main/resources/feeds_score_config_xgb_20240828.conf

@@ -0,0 +1,286 @@
+scorer-config = {
+  lr-rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.XGBoostScorer"
+    scorer-priority = 99
+    model-path = "zhangbo/model_xgb_for_recsys.tar.gz"
+    param = {
+      features = [
+        "b123_1h_STR",
+        "b123_1h_log(share)",
+        "b123_1h_ROV",
+        "b123_1h_log(return)",
+        "b123_1h_ROV*log(return)",
+        "b123_2h_STR",
+        "b123_2h_log(share)",
+        "b123_2h_ROV",
+        "b123_2h_log(return)",
+        "b123_2h_ROV*log(return)",
+        "b123_3h_STR",
+        "b123_3h_log(share)",
+        "b123_3h_ROV",
+        "b123_3h_log(return)",
+        "b123_3h_ROV*log(return)",
+        "b123_4h_STR",
+        "b123_4h_log(share)",
+        "b123_4h_ROV",
+        "b123_4h_log(return)",
+        "b123_4h_ROV*log(return)",
+        "b123_12h_STR",
+        "b123_12h_log(share)",
+        "b123_12h_ROV",
+        "b123_12h_log(return)",
+        "b123_12h_ROV*log(return)",
+        "b123_1d_STR",
+        "b123_1d_log(share)",
+        "b123_1d_ROV",
+        "b123_1d_log(return)",
+        "b123_1d_ROV*log(return)",
+        "b123_3d_STR",
+        "b123_3d_log(share)",
+        "b123_3d_ROV",
+        "b123_3d_log(return)",
+        "b123_3d_ROV*log(return)",
+        "b123_7d_STR",
+        "b123_7d_log(share)",
+        "b123_7d_ROV",
+        "b123_7d_log(return)",
+        "b123_7d_ROV*log(return)",
+        "b167_1h_STR",
+        "b167_1h_log(share)",
+        "b167_1h_ROV",
+        "b167_1h_log(return)",
+        "b167_1h_ROV*log(return)",
+        "b167_2h_STR",
+        "b167_2h_log(share)",
+        "b167_2h_ROV",
+        "b167_2h_log(return)",
+        "b167_2h_ROV*log(return)",
+        "b167_3h_STR",
+        "b167_3h_log(share)",
+        "b167_3h_ROV",
+        "b167_3h_log(return)",
+        "b167_3h_ROV*log(return)",
+        "b167_4h_STR",
+        "b167_4h_log(share)",
+        "b167_4h_ROV",
+        "b167_4h_log(return)",
+        "b167_4h_ROV*log(return)",
+        "b167_12h_STR",
+        "b167_12h_log(share)",
+        "b167_12h_ROV",
+        "b167_12h_log(return)",
+        "b167_12h_ROV*log(return)",
+        "b167_1d_STR",
+        "b167_1d_log(share)",
+        "b167_1d_ROV",
+        "b167_1d_log(return)",
+        "b167_1d_ROV*log(return)",
+        "b167_3d_STR",
+        "b167_3d_log(share)",
+        "b167_3d_ROV",
+        "b167_3d_log(return)",
+        "b167_3d_ROV*log(return)",
+        "b167_7d_STR",
+        "b167_7d_log(share)",
+        "b167_7d_ROV",
+        "b167_7d_log(return)",
+        "b167_7d_ROV*log(return)",
+        "b8910_1h_STR",
+        "b8910_1h_log(share)",
+        "b8910_1h_ROV",
+        "b8910_1h_log(return)",
+        "b8910_1h_ROV*log(return)",
+        "b8910_2h_STR",
+        "b8910_2h_log(share)",
+        "b8910_2h_ROV",
+        "b8910_2h_log(return)",
+        "b8910_2h_ROV*log(return)",
+        "b8910_3h_STR",
+        "b8910_3h_log(share)",
+        "b8910_3h_ROV",
+        "b8910_3h_log(return)",
+        "b8910_3h_ROV*log(return)",
+        "b8910_4h_STR",
+        "b8910_4h_log(share)",
+        "b8910_4h_ROV",
+        "b8910_4h_log(return)",
+        "b8910_4h_ROV*log(return)",
+        "b8910_12h_STR",
+        "b8910_12h_log(share)",
+        "b8910_12h_ROV",
+        "b8910_12h_log(return)",
+        "b8910_12h_ROV*log(return)",
+        "b8910_1d_STR",
+        "b8910_1d_log(share)",
+        "b8910_1d_ROV",
+        "b8910_1d_log(return)",
+        "b8910_1d_ROV*log(return)",
+        "b8910_3d_STR",
+        "b8910_3d_log(share)",
+        "b8910_3d_ROV",
+        "b8910_3d_log(return)",
+        "b8910_3d_ROV*log(return)",
+        "b8910_7d_STR",
+        "b8910_7d_log(share)",
+        "b8910_7d_ROV",
+        "b8910_7d_log(return)",
+        "b8910_7d_ROV*log(return)",
+        "b111213_1h_STR",
+        "b111213_1h_log(share)",
+        "b111213_1h_ROV",
+        "b111213_1h_log(return)",
+        "b111213_1h_ROV*log(return)",
+        "b111213_2h_STR",
+        "b111213_2h_log(share)",
+        "b111213_2h_ROV",
+        "b111213_2h_log(return)",
+        "b111213_2h_ROV*log(return)",
+        "b111213_3h_STR",
+        "b111213_3h_log(share)",
+        "b111213_3h_ROV",
+        "b111213_3h_log(return)",
+        "b111213_3h_ROV*log(return)",
+        "b111213_4h_STR",
+        "b111213_4h_log(share)",
+        "b111213_4h_ROV",
+        "b111213_4h_log(return)",
+        "b111213_4h_ROV*log(return)",
+        "b111213_12h_STR",
+        "b111213_12h_log(share)",
+        "b111213_12h_ROV",
+        "b111213_12h_log(return)",
+        "b111213_12h_ROV*log(return)",
+        "b111213_1d_STR",
+        "b111213_1d_log(share)",
+        "b111213_1d_ROV",
+        "b111213_1d_log(return)",
+        "b111213_1d_ROV*log(return)",
+        "b111213_3d_STR",
+        "b111213_3d_log(share)",
+        "b111213_3d_ROV",
+        "b111213_3d_log(return)",
+        "b111213_3d_ROV*log(return)",
+        "b111213_7d_STR",
+        "b111213_7d_log(share)",
+        "b111213_7d_ROV",
+        "b111213_7d_log(return)",
+        "b111213_7d_ROV*log(return)",
+        "b171819_1h_STR",
+        "b171819_1h_log(share)",
+        "b171819_1h_ROV",
+        "b171819_1h_log(return)",
+        "b171819_1h_ROV*log(return)",
+        "b171819_2h_STR",
+        "b171819_2h_log(share)",
+        "b171819_2h_ROV",
+        "b171819_2h_log(return)",
+        "b171819_2h_ROV*log(return)",
+        "b171819_3h_STR",
+        "b171819_3h_log(share)",
+        "b171819_3h_ROV",
+        "b171819_3h_log(return)",
+        "b171819_3h_ROV*log(return)",
+        "b171819_4h_STR",
+        "b171819_4h_log(share)",
+        "b171819_4h_ROV",
+        "b171819_4h_log(return)",
+        "b171819_4h_ROV*log(return)",
+        "b171819_12h_STR",
+        "b171819_12h_log(share)",
+        "b171819_12h_ROV",
+        "b171819_12h_log(return)",
+        "b171819_12h_ROV*log(return)",
+        "b171819_1d_STR",
+        "b171819_1d_log(share)",
+        "b171819_1d_ROV",
+        "b171819_1d_log(return)",
+        "b171819_1d_ROV*log(return)",
+        "b171819_3d_STR",
+        "b171819_3d_log(share)",
+        "b171819_3d_ROV",
+        "b171819_3d_log(return)",
+        "b171819_3d_ROV*log(return)",
+        "b171819_7d_STR",
+        "b171819_7d_log(share)",
+        "b171819_7d_ROV",
+        "b171819_7d_log(return)",
+        "b171819_7d_ROV*log(return)",
+        "total_time",
+        "bit_rate",
+        "playcnt_6h",
+        "playcnt_1d",
+        "playcnt_3d",
+        "playcnt_7d",
+        "share_pv_12h",
+        "share_pv_1d",
+        "share_pv_3d",
+        "share_pv_7d",
+        "return_uv_12h",
+        "return_uv_1d",
+        "return_uv_3d",
+        "return_uv_7d",
+        "c3_feature_tags_1d_matchnum",
+        "c3_feature_tags_1d_maxscore",
+        "c3_feature_tags_1d_avgscore",
+        "c3_feature_tags_3d_matchnum",
+        "c3_feature_tags_3d_maxscore",
+        "c3_feature_tags_3d_avgscore",
+        "c3_feature_tags_7d_matchnum",
+        "c3_feature_tags_7d_maxscore",
+        "c3_feature_tags_7d_avgscore",
+        "c4_feature_tags_1d_matchnum",
+        "c4_feature_tags_1d_maxscore",
+        "c4_feature_tags_1d_avgscore",
+        "c4_feature_tags_3d_matchnum",
+        "c4_feature_tags_3d_maxscore",
+        "c4_feature_tags_3d_avgscore",
+        "c4_feature_tags_7d_matchnum",
+        "c4_feature_tags_7d_maxscore",
+        "c4_feature_tags_7d_avgscore",
+        "c5_feature_tags_1d_matchnum",
+        "c5_feature_tags_1d_maxscore",
+        "c5_feature_tags_1d_avgscore",
+        "c5_feature_tags_3d_matchnum",
+        "c5_feature_tags_3d_maxscore",
+        "c5_feature_tags_3d_avgscore",
+        "c5_feature_tags_7d_matchnum",
+        "c5_feature_tags_7d_maxscore",
+        "c5_feature_tags_7d_avgscore",
+        "c6_feature_tags_1d_matchnum",
+        "c6_feature_tags_1d_maxscore",
+        "c6_feature_tags_1d_avgscore",
+        "c6_feature_tags_3d_matchnum",
+        "c6_feature_tags_3d_maxscore",
+        "c6_feature_tags_3d_avgscore",
+        "c6_feature_tags_7d_matchnum",
+        "c6_feature_tags_7d_maxscore",
+        "c6_feature_tags_7d_avgscore",
+        "c7_feature_tags_1d_matchnum",
+        "c7_feature_tags_1d_maxscore",
+        "c7_feature_tags_1d_avgscore",
+        "c7_feature_tags_3d_matchnum",
+        "c7_feature_tags_3d_maxscore",
+        "c7_feature_tags_3d_avgscore",
+        "c7_feature_tags_7d_matchnum",
+        "c7_feature_tags_7d_maxscore",
+        "c7_feature_tags_7d_avgscore",
+        "c8_feature_share_score",
+        "c8_feature_share_num",
+        "c8_feature_share_rank",
+        "c8_feature_return_score",
+        "c8_feature_return_num",
+        "c8_feature_return_rank",
+        "c9_feature_share_score",
+        "c9_feature_share_num",
+        "c9_feature_share_rank",
+        "c9_feature_return_score",
+        "c9_feature_return_num",
+        "c9_feature_return_rank",
+        "d1_exp",
+        "d1_return_n",
+        "d1_rovn"
+      ]
+    }
+  }
+
+}

recommend-server-service/src/main/resources/logback-spring.xml → recommend-server-service/src/main/resources/logback.xml

@@ -316,10 +316,6 @@
 
     <root level="info">
         <appender-ref ref="CONSOLE"/>
-        <appender-ref ref="DEBUG_FILE"/>
-        <appender-ref ref="INFO_FILE"/>
-        <appender-ref ref="WARN_FILE"/>
-        <appender-ref ref="ERROR_FILE"/>
         <appender-ref ref="loghubAppenderInfo"/>
         <appender-ref ref="loghubAppenderWarn"/>
         <appender-ref ref="loghubAppenderError"/>