ADD: 召回

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/GBDTModel.java

@@ -13,7 +13,7 @@ import java.util.*;
 public class GBDTModel extends Model {
     private static final Logger LOGGER = LoggerFactory.getLogger(GBDTModel.class);
     private static final float LINEAR_TRANSFORM_LOWER_BOUND = 0f;
-    private static final float LINEAR_TRANSFORM_SLOPE = 1f / 1f;
+    private static final float LINEAR_TRANSFORM_SLOPE = 1f;
     private int featureCount = 0;
     private Map<String, Integer> featureIdMap = null;
     private List<HashMap<Integer, Node>> boosterModel = null;
@@ -107,7 +107,7 @@ public class GBDTModel extends Model {
             model.add(tree);
         }
 
-        LOGGER.info("Boosted tree model load over and tree number is " + model.size());
+        LOGGER.info("Boosted tree rankByScore load over and tree number is " + model.size());
         input.close();
         in.close();
         if (model != null && model.size() > 0) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/LRModel.java

@@ -131,7 +131,7 @@ public class LRModel extends Model {
         int cnt = 0;
 
         Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
-        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        LOGGER.info("[MODELLOAD] before rankByScore load, key size: {}, current time: {}", lrModel.size(), curTime);
         //first stage
         while ((line = input.readLine()) != null) {
             String[] items = line.split("\t");
@@ -147,9 +147,9 @@ public class LRModel extends Model {
                 break;
             }
         }
-        //model update
+        //rankByScore update
         this.lrModel = model;
-        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        LOGGER.info("[MODELLOAD] after first stage rankByScore load, key size: {}, current time: {}", lrModel.size(), curTime);
         //final stage
         while ((line = input.readLine()) != null) {
             String[] items = line.split("\t");
@@ -158,9 +158,9 @@ public class LRModel extends Model {
             }
             putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
         }
-        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        LOGGER.info("[MODELLOAD] after rankByScore load, key size: {}, current time: {}", lrModel.size(), curTime);
 
-        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        LOGGER.info("[MODELLOAD] rankByScore load over and size " + cnt);
         input.close();
         in.close();
         return true;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ModelManager.java

@@ -29,10 +29,10 @@ public class ModelManager {
     private OSS client;
     private String bucketName;
 
-    private final String modelOssEndpoint = "model.oss.internal.endpoint";
-    private final String modelOssAccessKeyId = "model.oss.accessKeyId";
-    private final String modelOssAccessKeySecret = "model.oss.accessKetSecret";
-    private final String modelOssBucketName = "model.oss.bucketName";
+    private final String modelOssEndpoint = "rankByScore.oss.internal.endpoint";
+    private final String modelOssAccessKeyId = "rankByScore.oss.accessKeyId";
+    private final String modelOssAccessKeySecret = "rankByScore.oss.accessKetSecret";
+    private final String modelOssBucketName = "rankByScore.oss.bucketName";
 
     private ModelManager() {
         // config load
@@ -153,9 +153,9 @@ public class ModelManager {
         final Runnable task = new Runnable() {
             public void run() {
                 // 模型更新开关
-                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.model-update-switch", true);
+                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.rankByScore-update-switch", true);
                 boolean modelUpdateSwitch = true;
-                log.info("model update switch [{}]", modelUpdateSwitch);
+                log.info("rankByScore update switch [{}]", modelUpdateSwitch);
                 if (modelUpdateSwitch) {
                     updateModels(false);
                 }
@@ -170,7 +170,7 @@ public class ModelManager {
     public void updateModels(final boolean isForceLoads) {
         log.info("begin to update: [{}]", loadTasks.keySet().size());
         for (String modelPath : loadTasks.keySet()) {
-            log.debug("load task model path [{}]", modelPath);
+            log.debug("load task rankByScore path [{}]", modelPath);
             ModelLoadTask task = loadTasks.get(modelPath);
             loadModel(task, isForceLoads, false);
         }
@@ -194,7 +194,7 @@ public class ModelManager {
             ossObj = client.getObject(bucketName, loadTask.path);
             long timeStamp = ossObj.getObjectMetadata().getLastModified().getTime();
             if (loadTask.lastModifyTime < timeStamp || isForceLoads) {
-                log.info("model file changed, ready to update, last modify: [{}], current model time: [{}]",
+                log.info("rankByScore file changed, ready to update, last modify: [{}], current rankByScore time: [{}]",
                         loadTask.lastModifyTime, timeStamp);
 
                 Model model = loadTask.modelClass.newInstance();
@@ -205,7 +205,7 @@ public class ModelManager {
             }
             ossObj.close();
         } catch (Exception e) {
-            log.error("update model fail", e);
+            log.error("update rankByScore fail", e);
         } finally {
             loadTask.isLoading = false;
             if (ossObj != null) {
@@ -231,10 +231,10 @@ public class ModelManager {
     private class ModelLoadTask {
 
         private int refCount;
-        private String path;
+        private final String path;
         private long lastModifyTime;
         private boolean isLoading;
-        private Class<? extends Model> modelClass;
+        private final Class<? extends Model> modelClass;
         private Model model;
 
         ModelLoadTask(String path, Class<? extends Model> modelClass) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/model/ThompsonSamplingModel.java

@@ -57,7 +57,7 @@ public class ThompsonSamplingModel extends Model {
         }
 
         this.thompsonSamplingModel = initModel;
-        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        LOGGER.info("[MODELLOAD] rankByScore load over and size " + cnt);
         input.close();
         in.close();
         return true;
@@ -75,8 +75,8 @@ public class ThompsonSamplingModel extends Model {
         double score = 0.0f;
         VideoActionFeature videoActionFeature = this.thompsonSamplingModel.getOrDefault(rankItem.getVideoId(), new VideoActionFeature());
 
-        int alpha = (int) videoActionFeature.getReturns() + this.alpha;
-        int beta = this.beta_returns + (int) videoActionFeature.getView();
+        int alpha = (int) videoActionFeature.getReturns() + ThompsonSamplingModel.alpha;
+        int beta = beta_returns + (int) videoActionFeature.getView();
         score = this.betaSampler(alpha, beta);
         return score;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/AbstractScorer.java

@@ -38,14 +38,14 @@ public abstract class AbstractScorer {
             try {
                 // 使用 modelPath 作为 modelName 注册
                 modelManager.registerModel(modelPath, modelPath, modelClass);
-                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
+                LOGGER.info("register rankByScore success, rankByScore path [{}], rankByScore class [{}]", modelPath, modelClass);
             } catch (ModelManager.ModelRegisterException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+                LOGGER.error("register rankByScore fail [{}]:[{}]", modelPath, e);
             } catch (IOException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+                LOGGER.error("register rankByScore fail [{}]:[{}]", modelPath, e);
             }
         } else {
-            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
+            LOGGER.error("modelpath is null, for rankByScore class [{}]", modelClass);
         }
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerConfig.java

@@ -17,8 +17,8 @@ import java.util.Set;
 
 public class ScorerConfig {
 
-    private static Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
-    private List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
+    private final List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
 
     public ScorerConfig(Config config) {
         this.load(config);
@@ -61,7 +61,7 @@ public class ScorerConfig {
             int pos = 0;
             for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
                 LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
-                        new Object[]{pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName()});
+                        pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName());
             }
             LOGGER.debug("Load scorer config success");
         } catch (Exception e) {
@@ -101,11 +101,11 @@ public class ScorerConfig {
                 disableSwitch = conf.getBoolean("disable-switch");
             }
             Config paramConfig = loadOptionConfig(conf, "param-config");
-            // model path
-            String modelPath = loadOptionStringConfig(conf, "model-path");
+            // rankByScore path
+            String modelPath = loadOptionStringConfig(conf, "rankByScore-path");
             if (modelPath == null) {
-                modelPath = loadOptionStringConfig(conf, "default-model-path");
-                LOGGER.debug("model-path is not exists in config file, use default-model-path instead, modelPath={}", modelPath);
+                modelPath = loadOptionStringConfig(conf, "default-rankByScore-path");
+                LOGGER.debug("rankByScore-path is not exists in config file, use default-rankByScore-path instead, modelPath={}", modelPath);
             }
             // enable queues
             Set<String> enableQueues = new HashSet<String>();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/score/ScorerUtils.java

@@ -15,9 +15,9 @@ import java.util.concurrent.ConcurrentHashMap;
 
 @Slf4j
 public final class ScorerUtils {
-    private static Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
 
-    private static Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
+    private static final Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
 
     public static String BASE_CONF_NEW_FEED = "feeds_score_config_new_baseline.conf";
 
@@ -44,7 +44,7 @@ public final class ScorerUtils {
     }
 
     /**
-     * init load model
+     * init load rankByScore
      *
      * @param scorers
      */

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/framework/userattention/UserAttentionExtractorUtils.java

@@ -4,8 +4,6 @@ import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import com.typesafe.config.ConfigObject;
 import com.typesafe.config.ConfigValue;
-
-
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,7 +52,7 @@ public class UserAttentionExtractorUtils {
             // debug log
             if (LOGGER.isDebugEnabled()) {
                 for (UserAttentionExtractorConfig extractorConfig : configList) {
-                    LOGGER.debug("attention extractor config [{}] [{}]", extractorConfig.getName(), extractorConfig.toString());
+                    LOGGER.debug("attention extractor config [{}] [{}]", extractorConfig.getName(), extractorConfig);
                 }
             }
         } catch (Exception e) {
@@ -99,7 +97,7 @@ public class UserAttentionExtractorUtils {
         String attentionExtractorName = extractorConfig.getString("name");
         Integer priority = loadOptionIntConfig(extractorConfig, "attention-priority");
         Config paramConfig = loadOptionConfig(extractorConfig, "param-config");
-        String modelPath = loadOptionStringConfig(extractorConfig, "model-path");
+        String modelPath = loadOptionStringConfig(extractorConfig, "rankByScore-path");
         return new UserAttentionExtractorConfig(attentionExtractorName,
                 paramConfig,
                 priority,

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/TopRecommendPipeline.java

@@ -11,19 +11,23 @@ import com.tzld.piaoquan.recommend.server.framework.merger.MergeUtils;
 import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueue;
 import com.tzld.piaoquan.recommend.server.framework.recaller.BaseRecaller;
 import com.tzld.piaoquan.recommend.server.framework.recaller.provider.RedisBackedQueue;
-import com.tzld.piaoquan.recommend.server.framework.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.framework.utils.RedisSmartClient;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
 import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
@@ -62,92 +66,13 @@ public class TopRecommendPipeline {
         if (logPrint) {
             log.info("traceId = {}, rankItems = {}", requestData.getRequestId(), JSONUtils.toJson(rankItems));
         }
-        List<Video> videos = rankVideos(rankItems, requestData.getRequestId());
+        List<Video> videos = rankItem2Video(rankItems);
         if (logPrint) {
             log.info("traceId = {}, videos = {}", requestData.getRequestId(), JSONUtils.toJson(videos));
         }
         return videos;
     }
 
-    private List<Video> rankVideos(List<RankItem> rankItems, String requestId) {
-        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
-        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
-        Calendar calendar = Calendar.getInstance();
-        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
-        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
-        String rtFeaPart1h = date + hour;
-        if (rtFeaPartKeyResult != null) {
-            if (rtFeaPartKeyResult.get(1) != null) {
-                rtFeaPart1h = rtFeaPartKeyResult.get(1);
-            }
-        }
-        // 2 统计分
-        String cur = rtFeaPart1h;
-        List<String> datehours = new LinkedList<>();
-        for (int i = 0; i < 24; ++i) {
-            datehours.add(cur);
-            cur = ExtractorUtils.subtractHours(cur, 1);
-        }
-        for (RankItem item : rankItems) {
-            Map<String, String> itemBasicMap = item.getItemBasicFeature();
-            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
-            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
-            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
-            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
-            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
-            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
-
-            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
-            Double share2returnScore = calScoreWeight(share2return);
-            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
-            Double view2returnScore = calScoreWeight(view2return);
-            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
-            Double view2playScore = calScoreWeight(view2play);
-            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
-            Double play2shareScore = calScoreWeight(play2share);
-            item.scoresMap.put("share2returnScore", share2returnScore);
-            item.scoresMap.put("view2returnScore", view2returnScore);
-            item.scoresMap.put("view2playScore", view2playScore);
-            item.scoresMap.put("play2shareScore", play2shareScore);
-
-            Double allreturnsScore = calScoreWeight(allreturns);
-            item.scoresMap.put("allreturnsScore", allreturnsScore);
-
-            // rov的趋势
-            double trendScore = calTrendScore(view2return);
-            item.scoresMap.put("trendScore", trendScore);
-
-            // 新视频提取
-            double newVideoScore = calNewVideoScore(itemBasicMap);
-            item.scoresMap.put("newVideoScore", newVideoScore);
-
-        }
-        // 3 融合公式
-        List<Video> result = new ArrayList<>();
-        double alpha = this.mergeWeight.getOrDefault("alpha", 1.0);
-        double beta = this.mergeWeight.getOrDefault("beta", 1.0);
-        for (RankItem item : rankItems) {
-            double trendScore = item.scoresMap.getOrDefault("trendScore", 0.0) > 0.0 ?
-                    item.scoresMap.getOrDefault("trendScore", 0.0) : 0.0;
-            double newVideoScore = item.scoresMap.getOrDefault("newVideoScore", 0.0) > 0.0 ?
-                    item.scoresMap.getOrDefault("newVideoScore", 0.0) : 0.0;
-            double score = item.getScoreStr() *
-                    item.scoresMap.getOrDefault("share2returnScore", 0.0)
-                    + alpha * trendScore
-                    + beta * newVideoScore;
-            Video video = new Video();
-            video.setVideoId(Long.parseLong(item.getId()));
-            video.setPushFrom(item.getQueue());
-            video.setScore(score);
-            video.setSortScore(score);
-            video.setScoreStr(item.getScoreStr());
-            video.setScoresMap(item.getScoresMap());
-            result.add(video);
-        }
-        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
-        return result;
-    }
-
     public List<Double> getStaticData(Map<String, Map<String, Double>> itemRealMap,
                                       List<String> datehours, String key){
         List<Double> views = new LinkedList<>();
@@ -160,32 +85,6 @@ public class TopRecommendPipeline {
         return views;
     }
 
-    public double calNewVideoScore(Map<String, String> itemBasicMap){
-        double existenceDays = Double.valueOf(itemBasicMap.getOrDefault("existence_days", "30"));
-        if (existenceDays > 8){
-            return 0.0;
-        }
-        double score = 1.0 / (existenceDays + 5.0);
-        return score;
-    }
-    public double calTrendScore(List<Double> data){
-        double sum = 0.0;
-        int size = data.size();
-        for (int i=0; i<size-4; ++i){
-            sum += data.get(i) - data.get(i+4);
-        }
-        if (sum * 10 > 0.6){
-            sum = 0.6;
-        }else{
-            sum = sum * 10;
-        }
-        if (sum > 0){
-            // 为了打断点
-            sum = sum;
-        }
-        return sum;
-    }
-
     public Double calScoreWeight(List<Double> data){
         Double up = 0.0;
         Double down = 0.0;
@@ -208,7 +107,7 @@ public class TopRecommendPipeline {
     public List<RankItem> feedByRec(final RecommendRequest requestData,
                                     final int requestIndex,
                                     final User userInfo, Boolean logPrint) {
-        int recallNum = 200;
+        int recallNum = 150;
 
         // Step 1: Attention extraction
 //        long timestamp = System.currentTimeMillis();
@@ -253,6 +152,11 @@ public class TopRecommendPipeline {
 
         // 多样性融合
         List<RankItem> mergeItems = topQueue.getItems();
+        if (CollectionUtils.isEmpty(mergeItems)) {
+            return new ArrayList<>();
+        }
+        duplicate(mergeItems);
+
         if (logPrint) {
             log.info("traceId = {}, mergeItems = {}", requestData.getRequestId(), JSONUtils.toJson(mergeItems));
         }
@@ -260,15 +164,305 @@ public class TopRecommendPipeline {
 
         // Step 6: Global Rank & subList
         // TODO 前置和后置处理逻辑 hardcode，后续优化
-        Map<String, String> sceneFeatureMap = getSceneFeature(requestData);
-        Map<String, String> userFeatureMap = getUserFeatureMap(requestData, mergeItems);
-        List<RankItem> rovRecallRankNewallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF_NEW_FEED)
-                .scoring(sceneFeatureMap, userFeatureMap, mergeItems);
+        List<RankItem> rovRecallRankNewScore = rankByScore(mergeItems, requestData);
         if (logPrint) {
-            log.info("traceId = {}, rovRecallRankNewallScore = {}", requestData.getRequestId(), JSONUtils.toJson(rovRecallRankNewallScore));
+            log.info("traceId = {}, rovRecallRankNewScore = {}", requestData.getRequestId(), JSONUtils.toJson(rovRecallRankNewScore));
         }
 
-        return rovRecallRankNewallScore;
+        return rovRecallRankNewScore;
+    }
+
+    private List<Video> rankItem2Video(List<RankItem> items) {
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+        // 2 统计分
+        String cur = rtFeaPart1h;
+        List<String> datehours = new LinkedList<>();
+        for (int i=0; i<24; ++i){
+            datehours.add(cur);
+            cur = ExtractorUtils.subtractHours(cur, 1);
+        }
+        for (RankItem item : items){
+            Map<String, String> itemBasicMap = item.getItemBasicFeature();
+            Map<String, Map<String, Double>> itemRealMap = item.getItemRealTimeFeature();
+            List<Double> views = getStaticData(itemRealMap, datehours, "view_pv_list_1h");
+            List<Double> plays = getStaticData(itemRealMap, datehours, "play_pv_list_1h");
+            List<Double> shares = getStaticData(itemRealMap, datehours, "share_pv_list_1h");
+            List<Double> returns = getStaticData(itemRealMap, datehours, "p_return_uv_list_1h");
+            List<Double> allreturns = getStaticData(itemRealMap, datehours, "return_uv_list_1h");
+
+            List<Double> share2return = getRateData(returns, shares, 1.0, 1000.0);
+            Double share2returnScore = calScoreWeight(share2return);
+            List<Double> view2return = getRateData(returns, views, 1.0, 1000.0);
+            Double view2returnScore = calScoreWeight(view2return);
+            List<Double> view2play = getRateData(plays, views, 1.0, 1000.0);
+            Double view2playScore = calScoreWeight(view2play);
+            List<Double> play2share = getRateData(shares, plays, 1.0, 1000.0);
+            Double play2shareScore = calScoreWeight(play2share);
+            item.scoresMap.put("share2returnScore", share2returnScore);
+            item.scoresMap.put("view2returnScore", view2returnScore);
+            item.scoresMap.put("view2playScore", view2playScore);
+            item.scoresMap.put("play2shareScore", play2shareScore);
+
+            Double allreturnsScore = calScoreWeight(allreturns);
+            item.scoresMap.put("allreturnsScore", allreturnsScore);
+        }
+        // 3 融合公式
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : items){
+            double score = item.getScoreStr() *
+                    item.scoresMap.getOrDefault("share2returnScore", 0.0) *
+                    Math.log(1 + item.scoresMap.getOrDefault("allreturnsScore", 0.0));
+            Video video = new Video();
+            video.setVideoId(Long.parseLong(item.getId()));
+            video.setPushFrom(item.getQueue());
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoreStr(item.getScoreStr());
+            video.setScoresMap(item.getScoresMap());
+            result.add(video);
+        }
+        Collections.sort(result, Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    private void duplicate(List<RankItem> items) {
+        Set<String> ids = new HashSet<>();
+        List<RankItem> result = new ArrayList<>();
+        for (RankItem item : items) {
+            if (ids.contains(item.getId())) {
+                continue;
+            }
+            ids.add(item.getId());
+            result.add(item);
+        }
+        items.clear();
+        items.addAll(result);
+    }
+
+    public List<RankItem> rankByScore(List<RankItem> rankItems, RecommendRequest param){
+        List<RankItem> result = new ArrayList<>();
+        if (rankItems.isEmpty()){
+            return result;
+        }
+
+        RedisStandaloneConfiguration redisSC = new RedisStandaloneConfiguration();
+        redisSC.setPort(6379);
+        redisSC.setPassword("Wqsd@2019");
+        redisSC.setHostName("r-bp1pi8wyv6lzvgjy5z.redis.rds.aliyuncs.com");
+        RedisConnectionFactory connectionFactory = new JedisConnectionFactory(redisSC);
+        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
+        redisTemplate.setConnectionFactory(connectionFactory);
+        redisTemplate.setDefaultSerializer(new StringRedisSerializer());
+        redisTemplate.afterPropertiesSet();
+
+        // 0: 场景特征处理
+        Map<String, String> sceneFeatureMap =  this.getSceneFeature(param);
+
+        // 1: user特征处理
+        Map<String, String> userFeatureMap = new HashMap<>();
+        if (param.getMid() != null && !param.getMid().isEmpty()){
+            String midKey = "user_info_4video_" + param.getMid();
+            String userFeatureStr = redisTemplate.opsForValue().get(midKey);
+            if (userFeatureStr != null){
+                try{
+                    userFeatureMap = JSONUtils.fromJson(userFeatureStr,
+                            new TypeToken<Map<String, String>>() {},
+                            userFeatureMap);
+                }catch (Exception e){
+                    log.error(String.format("parse user json is wrong in {} with {}", this.getClass().getSimpleName(), e));
+                }
+            }else{
+                JSONObject obj = new JSONObject();
+                obj.put("name", "user_key_in_model_is_null");
+                obj.put("class", this.getClass().getSimpleName());
+                log.info(obj.toString());
+//                return videos;
+            }
+        }
+        final Set<String> userFeatureSet = new HashSet<>(Arrays.asList(
+                "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system",
+                "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+        ));
+        Iterator<Map.Entry<String, String>> iterator = userFeatureMap.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            if (!userFeatureSet.contains(entry.getKey())) {
+                iterator.remove();
+            }
+        }
+        Map<String, String> f1 = RankExtractorUserFeature.getOriginFeature(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_system"
+                ))
+        );
+        Map<String, String> f2 = RankExtractorUserFeature.getUserRateFeature(userFeatureMap);
+        Map<String, String> f3 = RankExtractorUserFeature.cntFeatureChange(userFeatureMap,
+                new HashSet<String>(Arrays.asList(
+                        "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
+                        "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt"
+                ))
+        );
+        f1.putAll(f2);
+        f1.putAll(f3);
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
+
+        // 2-1: item特征处理
+        final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
+                "total_time", "play_count_total",
+                "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"
+        ));
+
+        List<String> videoIds = CommonCollectionUtils.toListDistinct(rankItems, RankItem::getId);
+        List<String> videoFeatureKeys = videoIds.stream().map(r-> "video_info_" + r)
+                .collect(Collectors.toList());
+        List<String> videoFeatures = redisTemplate.opsForValue().multiGet(videoFeatureKeys);
+        if (videoFeatures != null){
+            for (int i=0; i<videoFeatures.size(); ++i){
+                String vF = videoFeatures.get(i);
+                Map<String, String> vfMap = new HashMap<>();
+                if (vF == null){
+                    continue;
+                }
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    rankItems.get(i).setItemBasicFeature(vfMap);
+                    Iterator<Map.Entry<String, String>> iteratorIn = vfMap.entrySet().iterator();
+                    while (iteratorIn.hasNext()) {
+                        Map.Entry<String, String> entry = iteratorIn.next();
+                        if (!itemFeatureSet.contains(entry.getKey())) {
+                            iteratorIn.remove();
+                        }
+                    }
+                    Map<String, String> f4 = RankExtractorItemFeature.getItemRateFeature(vfMap);
+                    Map<String, String> f5 = RankExtractorItemFeature.cntFeatureChange(vfMap,
+                            new HashSet<>(Arrays.asList(
+                                    "total_time", "play_count_total",
+                                    "i_1day_exp_cnt", "i_1day_click_cnt", "i_1day_share_cnt", "i_1day_return_cnt",
+                                    "i_3day_exp_cnt", "i_3day_click_cnt", "i_3day_share_cnt", "i_3day_return_cnt"))
+                    );
+                    f4.putAll(f5);
+                    rankItems.get(i).setFeatureMap(f4);
+                }catch (Exception e){
+                    log.error(String.format("parse video json is wrong in {} with {}", this.getClass().getSimpleName(), e));
+                }
+            }
+        }
+        // 2-2: item 实时特征处理
+        List<String> rtFeaPartKey = new ArrayList<>(Arrays.asList("item_rt_fea_1day_partition", "item_rt_fea_1h_partition"));
+        List<String> rtFeaPartKeyResult = this.redisTemplate.opsForValue().multiGet(rtFeaPartKey);
+        Calendar calendar = Calendar.getInstance();
+        String date = new SimpleDateFormat("yyyyMMdd").format(calendar.getTime());
+        String hour = new SimpleDateFormat("HH").format(calendar.getTime());
+        String rtFeaPart1day = date + hour;
+        String rtFeaPart1h = date + hour;
+        if (rtFeaPartKeyResult != null){
+            if (rtFeaPartKeyResult.get(0) != null){
+                rtFeaPart1day = rtFeaPartKeyResult.get(0);
+            }
+            if (rtFeaPartKeyResult.get(1) != null){
+                rtFeaPart1h = rtFeaPartKeyResult.get(1);
+            }
+        }
+
+        List<String> videoRtKeys1 = videoIds.stream().map(r-> "item_rt_fea_1day_" + r)
+                .collect(Collectors.toList());
+        List<String> videoRtKeys2 = videoIds.stream().map(r-> "item_rt_fea_1h_" + r)
+                .collect(Collectors.toList());
+        videoRtKeys1.addAll(videoRtKeys2);
+        List<String> videoRtFeatures = this.redisTemplate.opsForValue().multiGet(videoRtKeys1);
+
+
+        if (videoRtFeatures != null){
+            int j = 0;
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1day_ json is wrong in {} with {}",
+                            this.getClass().getSimpleName(), e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1day);
+                item.getFeatureMap().putAll(f8);
+            }
+            for (RankItem item: rankItems){
+                String vF = videoRtFeatures.get(j);
+                ++j;
+                if (vF == null){
+                    continue;
+                }
+                Map<String, String> vfMap = new HashMap<>();
+                Map<String, Map<String, Double>> vfMapNew = new HashMap<>();
+                try{
+                    vfMap = JSONUtils.fromJson(vF, new TypeToken<Map<String, String>>() {}, vfMap);
+
+                    for (Map.Entry<String, String> entry : vfMap.entrySet()){
+                        String value = entry.getValue();
+                        if (value == null){
+                            continue;
+                        }
+                        String [] var1 = value.split(",");
+                        Map<String, Double> tmp = new HashMap<>();
+                        for (String var2 : var1){
+                            String [] var3 = var2.split(":");
+                            tmp.put(var3[0], Double.valueOf(var3[1]));
+                        }
+                        vfMapNew.put(entry.getKey(), tmp);
+                    }
+                    item.setItemRealTimeFeature(vfMapNew);
+                }catch (Exception e){
+                    log.error(String.format("parse video item_rt_fea_1h_ json is wrong in {} with {}",
+                            this.getClass().getSimpleName(), e));
+                }
+                Map<String, String> f8 = RankExtractorItemFeature.getItemRealtimeRate(vfMapNew, rtFeaPart1h);
+                item.getFeatureMap().putAll(f8);
+            }
+        }
+
+
+        log.info("ItemFeature = {}", JSONUtils.toJson(videoFeatures));
+
+
+
+        List<RankItem> rovRecallScore = ScorerUtils.getScorerPipeline(ScorerUtils.BASE_CONF)
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        log.info("mergeAndRankRovRecallNew rovRecallScore={}", JSONUtils.toJson(rovRecallScore));
+        JSONObject obj = new JSONObject();
+        obj.put("name", "user_key_in_model_is_not_null");
+        obj.put("class", this.getClass().getSimpleName());
+        log.info(obj.toString());
+        return rovRecallScore;
     }
 
     private Map<String, String> getUserFeatureMap(RecommendRequest param, List<RankItem> rankItems) {
@@ -318,7 +512,7 @@ public class TopRecommendPipeline {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global1hHotCandidate.java

@@ -0,0 +1,62 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Global1hHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "video";
+
+    public Global1hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
+
+
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.45)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.45)).intValue();
+
+
+        QueueName queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "short_rov")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "global1h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
+
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+
+    }
+
+    @Override
+    public String toString() {
+        return "Global24hHotCandidate{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global24hHotCandidate.java

@@ -8,6 +8,7 @@ import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
 import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 
+import java.math.BigDecimal;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -22,29 +23,21 @@ public class Global24hHotCandidate extends IndexCandidateQueue {
     @Override
     public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
         int currRecall = 0;
-        String region = user.getRegion();
         Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
 
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
 
-        int num = recallNum / 3;
 
-        // index key  video:queue:type=global24h:region=北京:ordering=rov
-        QueueName queueName24HGlobal = new QueueName(ItemType, "rov")
+        QueueName queueName = new QueueName(ItemType, "rov")
                 .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName24HGlobal, num, getStrategyQueueInfo().getQueueName());
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
 
-        // index key  video:queue:type=global24h:region=北京:ordering=realplay
-        queueName24HGlobal = new QueueName(ItemType, "realplay")
-                .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName24HGlobal, num, getStrategyQueueInfo().getQueueName());
 
-        // index key  video:queue:type=global24h:region=北京:ordering=str
-        queueName24HGlobal = new QueueName(ItemType, "str")
+        queueName = new QueueName(ItemType, "str")
                 .addMatch("type", "global24h");
-        currRecall += addCandidateKey(simplifiedCandidates, queueName24HGlobal, num, getStrategyQueueInfo().getQueueName());
-
-        // index key  video:queue:type=global24h:region=北京:ordering=realplay_rate
-
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
 
         candidates.putAll(simplifiedCandidates);
         return currRecall;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Global3hHotCandidate.java

@@ -0,0 +1,62 @@
+package com.tzld.piaoquan.recommend.server.implement.candidate;
+
+import com.tzld.piaoquan.recommend.server.framework.candidiate.Candidate;
+import com.tzld.piaoquan.recommend.server.framework.candidiate.QueueName;
+import com.tzld.piaoquan.recommend.server.framework.common.User;
+import com.tzld.piaoquan.recommend.server.framework.merger.IndexCandidateQueue;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
+import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
+import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
+
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class Global3hHotCandidate extends IndexCandidateQueue {
+
+    public static final String ItemType = "video";
+
+    public Global3hHotCandidate(StrategyQueueInfo strategyQueueInfo, StrategyQueueConfig strategyQueueConfig) {
+        super(strategyQueueInfo, strategyQueueConfig);
+    }
+
+    @Override
+    public int addCandidateKey(Map<String, Candidate> candidates, int recallNum, User user, RecommendRequest requestData, int requestIndex, int expId) {
+        int currRecall = 0;
+        Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
+
+
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.45)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.45)).intValue();
+
+
+        QueueName queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "global3h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "short_rov")
+                .addMatch("type", "global3h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum, getStrategyQueueInfo().getQueueName());
+
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "global3h");
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum, getStrategyQueueInfo().getQueueName());
+
+
+        candidates.putAll(simplifiedCandidates);
+        return currRecall;
+
+    }
+
+    @Override
+    public String toString() {
+        return "Global24hHotCandidate{" +
+                "myCandidates=" + myCandidates +
+                ", rankerItemsList=" + rankerItemsList +
+                ", children=" + children +
+                ", items=" + items +
+                '}';
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region1hHotCandidate.java

@@ -8,6 +8,7 @@ import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
 import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 
+import java.math.BigDecimal;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -25,27 +26,30 @@ public class Region1hHotCandidate extends IndexCandidateQueue {
         String region = user.getRegion();
         Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
 
-        int num = recallNum / 3;
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.45)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.45)).intValue();
 
 
-        // index key  video:queue:type=region1h:region=北京:ordering=rov
-        QueueName queueName1HRegion = new QueueName(ItemType, "short_rov")
+        QueueName queueName = new QueueName(ItemType, "short_rov")
                 .addMatch("type", "region1h")
                 .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName1HRegion, num, getStrategyQueueInfo().getQueueName());
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum,
+                getStrategyQueueInfo().getQueueName());
 
 
-        // index key  video:queue:type=region1h:region=北京:ordering=rov
-        queueName1HRegion = new QueueName(ItemType, "rov")
+        queueName = new QueueName(ItemType, "rov")
                 .addMatch("type", "region1h")
                 .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName1HRegion, num, getStrategyQueueInfo().getQueueName());
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
+                getStrategyQueueInfo().getQueueName());
 
-        // index key  video:queue:type=region1h:region=北京:ordering=str
-        queueName1HRegion = new QueueName(ItemType, "str")
+        queueName = new QueueName(ItemType, "str")
                 .addMatch("type", "region1h")
                 .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName1HRegion, num, getStrategyQueueInfo().getQueueName());
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
+                getStrategyQueueInfo().getQueueName());
 
         candidates.putAll(simplifiedCandidates);
         return currRecall;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region24hHotCandidate.java

@@ -8,6 +8,7 @@ import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
 import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 
+import java.math.BigDecimal;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -25,32 +26,23 @@ public class Region24hHotCandidate extends IndexCandidateQueue {
         String region = user.getRegion();
         Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<String, Candidate>();
 
-        int num = recallNum / 4;
 
-        // index key  video:queue:type=region24h:region=北京:ordering=rov
-        QueueName queueName24HRegion = new QueueName(ItemType, "rov")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName24HRegion, num, getStrategyQueueInfo().getQueueName());
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.5)).intValue();
 
-        // index key  video:queue:type=region24h:region=北京:ordering=realplay
-        queueName24HRegion = new QueueName(ItemType, "realplay")
+        // index key  video:queue:type=region24h:region=北京:ordering=rov
+        QueueName queueName = new QueueName(ItemType, "rov")
                 .addMatch("type", "region24h")
                 .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName24HRegion, num, getStrategyQueueInfo().getQueueName());
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum, getStrategyQueueInfo().getQueueName());
 
         // index key  video:queue:type=region24h:region=北京:ordering=str
-        queueName24HRegion = new QueueName(ItemType, "str")
+        queueName = new QueueName(ItemType, "str")
                 .addMatch("type", "region24h")
                 .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName24HRegion, num, getStrategyQueueInfo().getQueueName());
-
-        // index key  video:queue:type=region24h:region=北京:ordering=realplay_rate
-        queueName24HRegion = new QueueName(ItemType, "realplay_ratio")
-                .addMatch("type", "region24h")
-                .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName24HRegion, num, getStrategyQueueInfo().getQueueName());
-
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
+                getStrategyQueueInfo().getQueueName());
 
         candidates.putAll(simplifiedCandidates);
         return currRecall;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/candidate/Region3hHotCandidate.java

@@ -8,6 +8,7 @@ import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueConfig;
 import com.tzld.piaoquan.recommend.server.framework.merger.StrategyQueueInfo;
 import com.tzld.piaoquan.recommend.server.gen.recommend.RecommendRequest;
 
+import java.math.BigDecimal;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -25,14 +26,33 @@ public class Region3hHotCandidate extends IndexCandidateQueue {
         String region = user.getRegion();
         Map<String, Candidate> simplifiedCandidates = new ConcurrentHashMap<>();
 
-        int num = recallNum;
+        BigDecimal recallNumBigDecimal = BigDecimal.valueOf(recallNum);
+        int strNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.1)).intValue();
+        int rovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.45)).intValue();
+        int shortRovNum = recallNumBigDecimal.multiply(BigDecimal.valueOf(0.45)).intValue();
 
 
         // index key video:queue:type=region3h:region=北京:ordering=short_rov
-        QueueName queueName3HRegion = new QueueName(ItemType, "short_rov")
+        QueueName queueName = new QueueName(ItemType, "short_rov")
                 .addMatch("type", "region3h")
                 .addMatch("region", region);
-        currRecall += addCandidateKey(simplifiedCandidates, queueName3HRegion, num, getStrategyQueueInfo().getQueueName());
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, shortRovNum,
+                getStrategyQueueInfo().getQueueName());
+
+        // index key video:queue:type=region3h:region=北京:ordering=rov
+        queueName = new QueueName(ItemType, "rov")
+                .addMatch("type", "region3h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, rovNum,
+                getStrategyQueueInfo().getQueueName());
+
+        // index key video:queue:type=region3h:region=北京:ordering=str
+        queueName = new QueueName(ItemType, "str")
+                .addMatch("type", "region3h")
+                .addMatch("region", region);
+        currRecall += addCandidateKey(simplifiedCandidates, queueName, strNum,
+                getStrategyQueueInfo().getQueueName());
+
 
         candidates.putAll(simplifiedCandidates);
         return currRecall;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogShareLRScorer.java

@@ -46,7 +46,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
 
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(rankItems, param.getRequestContext(),
@@ -63,7 +63,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                                       final UserFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         // userBytes
         UserBytesFeature userInfoBytes = null;
@@ -75,7 +75,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("before enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i));
             }
         }
 
@@ -107,8 +107,8 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
                     new RequestContextBytesFeature(requestContext));
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -117,8 +117,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
             // 增加实时特征后打开在线存储日志逻辑
             //
@@ -185,7 +184,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{requestContext.getRequest_id(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
     }
     @Override
     public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
@@ -197,7 +196,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
 
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(
@@ -215,7 +214,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                                       final List<RankItem> items) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
         // userBytes
         Map<String, byte[]> userFeatureMapByte = new HashMap<>();
         for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
@@ -233,7 +232,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("before enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i));
             }
         }
 
@@ -290,7 +289,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{sceneFeatureMapByte.size(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
     }
 
     public double calcScore(final LRModel lrModel,
@@ -310,8 +309,8 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             }
             lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{"", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -320,8 +319,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
         }
         item.setScoreStr(pro);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogShareLRScorer4Ros.java

@@ -46,7 +46,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
 
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(rankItems, param.getRequestContext(),
@@ -63,7 +63,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
                                       final UserFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         // userBytes
         UserBytesFeature userInfoBytes = null;
@@ -75,7 +75,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("before enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i));
             }
         }
 
@@ -107,8 +107,8 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
             lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
                     new RequestContextBytesFeature(requestContext));
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -117,8 +117,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
             // 增加实时特征后打开在线存储日志逻辑
             //
@@ -185,7 +184,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{requestContext.getRequest_id(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
     }
     @Override
     public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
@@ -197,7 +196,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
 
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(
@@ -215,7 +214,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
                                       final List<RankItem> items) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
         // userBytes
         Map<String, byte[]> userFeatureMapByte = new HashMap<>();
         for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
@@ -233,7 +232,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("before enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i));
             }
         }
 
@@ -290,7 +289,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{sceneFeatureMapByte.size(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
     }
 
     public double calcScore(final LRModel lrModel,
@@ -310,8 +309,8 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
             }
             lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{"", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -320,8 +319,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
         }
         item.setScoreRos(pro);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/implement/score/VlogThompsonScorer.java

@@ -39,7 +39,7 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
 
         long startTime = System.currentTimeMillis();
         ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(rankItems, param.getRequestContext(), userFeature);
@@ -55,7 +55,7 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
                                       final UserFeature user) {
         long startTime = System.currentTimeMillis();
         ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         // 所有都参与打分，按照ROV Thompson排序
         multipleCtrScore(items, model);
@@ -63,7 +63,7 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
+                LOGGER.debug("after enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
             }
         }
 
@@ -83,8 +83,7 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
         try {
             score = model.score(item);
         } catch (Exception e) {
-            LOGGER.error("score error for doc={} exception={}", new Object[]{
-                    item.getVideo(), ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("score error for doc={} exception={}", item.getVideo(), ExceptionUtils.getFullStackTrace(e));
         }
         item.setScore(score);
         return score;
@@ -139,6 +138,6 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
                 }
             }
         }
-        LOGGER.debug("ROV-Thompson Score {}, Total: {}, Cancel: {}", new Object[]{items.size(), cancel});
+        LOGGER.debug("ROV-Thompson Score {}, Total: {}, Cancel: {}", items.size(), cancel);
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/ModelService.java

@@ -32,7 +32,7 @@ public class ModelService {
     @Autowired
     private RedisTemplate<String, String> featureRedisTemplate;
 
-    private Map<String, String> modelNameMap = new HashMap<>();
+    private final Map<String, String> modelNameMap = new HashMap<>();
 
     @PostConstruct
     public void init() {
@@ -83,7 +83,7 @@ public class ModelService {
             }
         }
 
-        log.info("userFeature in model = {}", JSONUtils.toJson(userFeatureMap));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(userFeatureMap));
 
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
                 "total_time", "play_count_total",
@@ -170,7 +170,6 @@ public class ModelService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ) {
-            ;
         } else {
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/VideoRecommendService.java

@@ -14,7 +14,7 @@ import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.remote.FeatureRemoteService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankResult;
-import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4RegionMergeModelV6;
+import com.tzld.piaoquan.recommend.server.service.rank.strategy.RankStrategy4RegionMergeModelV2;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
@@ -43,7 +43,7 @@ public class VideoRecommendService {
     @Resource
     private FlowPoolRecommendPipeline flowPoolRecommendPipeline;
     @Resource
-    private RankStrategy4RegionMergeModelV6 rankStrategy4RegionMergeModelV6;
+    private RankStrategy4RegionMergeModelV2 rankStrategy4RegionMergeModelV2;
 
     public RecommendResponse homepageRecommend(RecommendRequest request) {
         return newRecommendSwitch ? recommendNew(request, 0) : recommend(request, 0);
@@ -80,7 +80,8 @@ public class VideoRecommendService {
         RecommendParam recommendParam = recommendService.genRecommendParam(request, recommendType);
         RankParam rankParam = recommendService.convertToRankParam(recommendParam, null);
 
-        RankResult rankResult = rankStrategy4RegionMergeModelV6.mergeAndSort(rankParam, topRecommendVideoList, flowPoolRecommendVideoList);
+        RankResult rankResult = rankStrategy4RegionMergeModelV2.mergeAndSort(rankParam, topRecommendVideoList,
+                flowPoolRecommendVideoList);
         if (newLogPrint) {
             log.info("traceId = {}, rankResult [{}]", request.getRequestId(), JSONObject.toJSONString(rankResult));
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RankModel.java

@@ -8,6 +8,8 @@ import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
@@ -18,11 +20,10 @@ import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -35,7 +36,7 @@ import java.util.stream.Collectors;
 @Slf4j
 public class RankStrategy4RankModel extends RankService {
 
-    @ApolloJsonValue("${video.model.weightv1:}")
+    @ApolloJsonValue("${video.rankByScore.weightv1:}")
     private Map<String, Double> mergeWeightNew;
     final private String CLASS_NAME = this.getClass().getSimpleName();
 
@@ -200,7 +201,7 @@ public class RankStrategy4RankModel extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -368,7 +369,6 @@ public class RankStrategy4RankModel extends RankService {
             "新竹市".equals(city) |
             "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4Rankv2Model.java

@@ -8,6 +8,8 @@ import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
 import com.tzld.piaoquan.recommend.server.service.rank.RankService;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
@@ -18,11 +20,10 @@ import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
 import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
-import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
 import org.springframework.stereotype.Service;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorUserFeature;
-import com.tzld.piaoquan.recommend.server.service.rank.extractor.RankExtractorItemFeature;
+
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -35,7 +36,7 @@ import java.util.stream.Collectors;
 @Slf4j
 public class RankStrategy4Rankv2Model extends RankService {
 
-    @ApolloJsonValue("${video.model.weightv2:}")
+    @ApolloJsonValue("${video.rankByScore.weightv2:}")
     private Map<String, Double> mergeWeight;
     final private String CLASS_NAME = this.getClass().getSimpleName();
 
@@ -200,7 +201,7 @@ public class RankStrategy4Rankv2Model extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -368,7 +369,6 @@ public class RankStrategy4Rankv2Model extends RankService {
             "新竹市".equals(city) |
             "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV1.java

@@ -2,7 +2,6 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 
 import com.alibaba.fastjson.JSONObject;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
@@ -34,7 +33,7 @@ import java.util.stream.Collectors;
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV1 extends RankService {
-//    @ApolloJsonValue("${video.model.weightv3:}")
+//    @ApolloJsonValue("${video.rankByScore.weightv3:}")
 //    private Map<String, Double> mergeWeight;
     final private String CLASS_NAME = this.getClass().getSimpleName();
     public void duplicate(Set<Long> setVideo, List<Video> videos){
@@ -259,7 +258,7 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -425,7 +424,6 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV2.java

@@ -33,7 +33,7 @@ import java.util.stream.Collectors;
 @Service
 @Slf4j
 public class RankStrategy4RegionMergeModelV2 extends RankService {
-//    @ApolloJsonValue("${video.model.weightv3:}")
+//    @ApolloJsonValue("${video.rankByScore.weightv3:}")
 //    private Map<String, Double> mergeWeight;
     final private String CLASS_NAME = this.getClass().getSimpleName();
     public void duplicate(Set<Long> setVideo, List<Video> videos){
@@ -258,7 +258,7 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -424,7 +424,6 @@ public class RankStrategy4RegionMergeModelV2 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV3.java

@@ -6,7 +6,6 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
-import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
@@ -50,7 +49,7 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
     @ApolloJsonValue("${rank.score.merge.weight:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
+    private final Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
@@ -364,7 +363,7 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -531,7 +530,6 @@ public class RankStrategy4RegionMergeModelV3 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV4.java

@@ -283,7 +283,7 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -449,7 +449,6 @@ public class RankStrategy4RegionMergeModelV4 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV5.java

@@ -301,7 +301,7 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -468,7 +468,6 @@ public class RankStrategy4RegionMergeModelV5 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV6.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 
 import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
@@ -33,7 +34,6 @@ import org.springframework.stereotype.Service;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.stream.Collectors;
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 /**
  * @author zhangbo
  * @desc 地域召回融合
@@ -44,7 +44,7 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
     @ApolloJsonValue("${rank.score.merge.weight:}")
     private Map<String, Double> mergeWeight;
     @ApolloJsonValue("${RankStrategy4DensityFilterV2:}")
-    private Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
+    private final Map<String,Map<String, Map<String, String>>> filterRules = new HashMap<>();
     final private String CLASS_NAME = this.getClass().getSimpleName();
     public void duplicate(Set<Long> setVideo, List<Video> videos){
         Iterator<Video> iterator = videos.iterator();
@@ -313,7 +313,7 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
         );
         f1.putAll(f2);
         f1.putAll(f3);
-        log.info("userFeature in model = {}", JSONUtils.toJson(f1));
+        log.info("userFeature in rankByScore = {}", JSONUtils.toJson(f1));
 
         // 2-1: item特征处理
         final Set<String> itemFeatureSet = new HashSet<>(Arrays.asList(
@@ -480,7 +480,6 @@ public class RankStrategy4RegionMergeModelV6 extends RankService {
                 "新竹市".equals(city) |
                 "嘉义市".equals(city)
         ){
-            ;
         }else{
             city = city.replaceAll("市$", "");
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/AbstractScorer.java

@@ -1,8 +1,8 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
-import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.service.score.model.Model;
 import com.tzld.piaoquan.recommend.server.service.score.model.ModelManager;
 import org.apache.commons.lang3.StringUtils;
@@ -38,14 +38,14 @@ public abstract class AbstractScorer {
             try {
                 // 使用 modelPath 作为 modelName 注册
                 modelManager.registerModel(modelPath, modelPath, modelClass);
-                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
+                LOGGER.info("register rankByScore success, rankByScore path [{}], rankByScore class [{}]", modelPath, modelClass);
             } catch (ModelManager.ModelRegisterException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+                LOGGER.error("register rankByScore fail [{}]:[{}]", modelPath, e);
             } catch (IOException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+                LOGGER.error("register rankByScore fail [{}]:[{}]", modelPath, e);
             }
         } else {
-            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
+            LOGGER.error("modelpath is null, for rankByScore class [{}]", modelClass);
         }
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerConfig.java

@@ -5,7 +5,6 @@ import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import com.typesafe.config.ConfigObject;
 import com.typesafe.config.ConfigValue;
-
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -18,8 +17,8 @@ import java.util.Set;
 
 public class ScorerConfig {
 
-    private static Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
-    private List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerConfig.class);
+    private final List<ScorerConfigInfo> configInfoList = new ArrayList<ScorerConfigInfo>();
 
     public ScorerConfig(Config config) {
         this.load(config);
@@ -62,7 +61,7 @@ public class ScorerConfig {
             int pos = 0;
             for (ScorerConfigInfo scorerConfigInfo : configInfoList) {
                 LOGGER.debug("scorer at position [{}], priority [{}], scorer name [{}]",
-                        new Object[]{pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName()});
+                        pos++, scorerConfigInfo.getScorerPriority(), scorerConfigInfo.getScorerName());
             }
             LOGGER.debug("Load scorer config success");
         } catch (Exception e) {
@@ -102,11 +101,11 @@ public class ScorerConfig {
                 disableSwitch = conf.getBoolean("disable-switch");
             }
             Config paramConfig = loadOptionConfig(conf, "param-config");
-            // model path
-            String modelPath = loadOptionStringConfig(conf, "model-path");
+            // rankByScore path
+            String modelPath = loadOptionStringConfig(conf, "rankByScore-path");
             if (modelPath == null) {
-                modelPath = loadOptionStringConfig(conf, "default-model-path");
-                LOGGER.debug("model-path is not exists in config file, use default-model-path instead, modelPath={}", modelPath);
+                modelPath = loadOptionStringConfig(conf, "default-rankByScore-path");
+                LOGGER.debug("rankByScore-path is not exists in config file, use default-rankByScore-path instead, modelPath={}", modelPath);
             }
             // enable queues
             Set<String> enableQueues = new HashSet<String>();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -17,10 +17,10 @@ import java.util.concurrent.ConcurrentHashMap;
 
 @Slf4j
 public final class ScorerUtils {
-    private static Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScorerUtils.class);
 
-    private static Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
-    private static Map<String, ScorerPipeline4Recall> scorerPipelineCache4Recall = new ConcurrentHashMap<>();
+    private static final Map<String, ScorerPipeline> scorerPipelineCache = new ConcurrentHashMap<>();
+    private static final Map<String, ScorerPipeline4Recall> scorerPipelineCache4Recall = new ConcurrentHashMap<>();
 
     public static String BASE_CONF = "feeds_score_config_baseline.conf";
     public static String VIDEO_SCORE_CONF_FOR_AD = "video_score_config_for_ad.conf";
@@ -63,7 +63,7 @@ public final class ScorerUtils {
     }
 
     /**
-     * init load model
+     * init load rankByScore
      *
      * @param scorers
      */

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareLRScorer.java

@@ -2,12 +2,11 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 
 import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
-import com.tzld.piaoquan.recommend.server.common.base.*;
-import com.tzld.piaoquan.recommend.feature.model.sample.*;
 import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
 import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
@@ -45,7 +44,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
 
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(rankItems, param.getRequestContext(),
@@ -62,7 +61,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                                       final UserFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         // userBytes
         UserBytesFeature userInfoBytes = null;
@@ -74,7 +73,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("before enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i));
             }
         }
 
@@ -106,8 +105,8 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
                     new RequestContextBytesFeature(requestContext));
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -116,8 +115,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
             // 增加实时特征后打开在线存储日志逻辑
             //
@@ -184,7 +182,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{requestContext.getRequest_id(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
     }
     @Override
     public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
@@ -196,7 +194,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
 
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(
@@ -214,7 +212,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                                       final List<RankItem> items) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
         // userBytes
         Map<String, byte[]> userFeatureMapByte = new HashMap<>();
         for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
@@ -232,7 +230,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("before enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i));
             }
         }
 
@@ -289,7 +287,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{sceneFeatureMapByte.size(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
     }
 
     public double calcScore(final LRModel lrModel,
@@ -309,8 +307,8 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             }
             lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{"", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -319,8 +317,7 @@ public class VlogShareLRScorer extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
         }
         item.setScoreStr(pro);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogShareLRScorer4Ros.java

@@ -2,12 +2,11 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 
 import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
-import com.tzld.piaoquan.recommend.server.common.base.*;
-import com.tzld.piaoquan.recommend.feature.model.sample.*;
 import com.tzld.piaoquan.recommend.feature.domain.video.feature.VlogShareLRFeatureExtractor;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.service.rank.strategy.OfflineVlogShareLRFeatureExtractor;
 import com.tzld.piaoquan.recommend.server.service.score.model.LRModel;
-
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
@@ -45,7 +44,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
 
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(rankItems, param.getRequestContext(),
@@ -62,7 +61,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
                                       final UserFeature user) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         // userBytes
         UserBytesFeature userInfoBytes = null;
@@ -74,7 +73,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("before enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i));
             }
         }
 
@@ -106,8 +105,8 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
             lrSamples = bytesFeatureExtractor.single(userInfoBytes, newsInfoBytes,
                     new RequestContextBytesFeature(requestContext));
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{new String(userInfoBytes.getUid()), item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new String(userInfoBytes.getUid()), item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -116,8 +115,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
             // 增加实时特征后打开在线存储日志逻辑
             //
@@ -184,7 +182,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{requestContext.getRequest_id(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", requestContext.getRequest_id(), items.size(), cancel);
     }
     @Override
     public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
@@ -196,7 +194,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
 
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(
@@ -214,7 +212,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
                                       final List<RankItem> items) {
         long startTime = System.currentTimeMillis();
         LRModel model = (LRModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
         // userBytes
         Map<String, byte[]> userFeatureMapByte = new HashMap<>();
         for(Map.Entry<String, String> entry: userFeatureMap.entrySet()){
@@ -232,7 +230,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+                LOGGER.debug("before enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i));
             }
         }
 
@@ -289,7 +287,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
                 }
             }
         }
-        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", new Object[]{sceneFeatureMapByte.size(), items.size(), cancel});
+        LOGGER.debug("Ctr Score {}, Total: {}, Cancel: {}", sceneFeatureMapByte.size(), items.size(), cancel);
     }
 
     public double calcScore(final LRModel lrModel,
@@ -309,8 +307,8 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
             }
             lrSamples = bytesFeatureExtractor.single(userFeatureMapByte, itemFeatureByte, sceneFeatureMapByte);
         } catch (Exception e) {
-            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", new Object[]{"", item.getVideoId(),
-                    ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("extract feature error for imei={}, doc={}, [{}]", "", item.getVideoId(),
+                    ExceptionUtils.getFullStackTrace(e));
         }
 
 
@@ -319,8 +317,7 @@ public class VlogShareLRScorer4Ros extends BaseLRModelScorer {
             try {
                 pro = lrModel.score(lrSamples);
             } catch (Exception e) {
-                LOGGER.error("score error for doc={} exception={}", new Object[]{
-                        item.getVideoId(), ExceptionUtils.getFullStackTrace(e)});
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
         }
         item.setScoreRos(pro);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogThompsonScorer.java

@@ -1,6 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
-import com.tzld.piaoquan.recommend.feature.domain.video.base.*;
+import com.tzld.piaoquan.recommend.feature.domain.video.base.RequestContext;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.service.score.model.ThompsonSamplingModel;
@@ -36,7 +36,7 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
 
         long startTime = System.currentTimeMillis();
         ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         List<RankItem> result = rankItems;
         result = rankByJava(rankItems, param.getRequestContext(), userFeature);
@@ -52,7 +52,7 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
                                       final UserFeature user) {
         long startTime = System.currentTimeMillis();
         ThompsonSamplingModel model = (ThompsonSamplingModel) this.getModel();
-        LOGGER.debug("model size: [{}]", model.getModelSize());
+        LOGGER.debug("rankByScore size: [{}]", model.getModelSize());
 
         // 所有都参与打分，按照ROV Thompson排序
         multipleCtrScore(items, model);
@@ -60,7 +60,7 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
         // debug log
         if (LOGGER.isDebugEnabled()) {
             for (int i = 0; i < items.size(); i++) {
-                LOGGER.debug("after enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
+                LOGGER.debug("after enter feeds rankByScore predict ctr score [{}] [{}]", items.get(i), items.get(i).getScore());
             }
         }
 
@@ -80,8 +80,7 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
         try {
             score = model.score(item);
         } catch (Exception e) {
-            LOGGER.error("score error for doc={} exception={}", new Object[]{
-                    item.getVideo(), ExceptionUtils.getFullStackTrace(e)});
+            LOGGER.error("score error for doc={} exception={}", item.getVideo(), ExceptionUtils.getFullStackTrace(e));
         }
         item.setScore(score);
         return score;
@@ -136,6 +135,6 @@ public class VlogThompsonScorer extends BaseThompsonSamplingScorer {
                 }
             }
         }
-        LOGGER.debug("ROV-Thompson Score {}, Total: {}, Cancel: {}", new Object[]{items.size(), cancel});
+        LOGGER.debug("ROV-Thompson Score {}, Total: {}, Cancel: {}", items.size(), cancel);
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/GBDTModel.java

@@ -6,18 +6,14 @@ import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 
 
 public class GBDTModel extends Model {
     private static final Logger LOGGER = LoggerFactory.getLogger(GBDTModel.class);
     private static final float LINEAR_TRANSFORM_LOWER_BOUND = 0f;
-    private static final float LINEAR_TRANSFORM_SLOPE = 1f / 1f;
+    private static final float LINEAR_TRANSFORM_SLOPE = 1f;
     private int featureCount = 0;
     private Map<String, Integer> featureIdMap = null;
     private List<HashMap<Integer, Node>> boosterModel = null;
@@ -111,7 +107,7 @@ public class GBDTModel extends Model {
             model.add(tree);
         }
 
-        LOGGER.info("Boosted tree model load over and tree number is " + model.size());
+        LOGGER.info("Boosted tree rankByScore load over and tree number is " + model.size());
         input.close();
         in.close();
         if (model != null && model.size() > 0) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/LRModel.java

@@ -1,7 +1,9 @@
 package com.tzld.piaoquan.recommend.server.service.score.model;
 
 
-import com.tzld.piaoquan.recommend.feature.model.sample.*;
+import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
+import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
 import it.unimi.dsi.fastutil.longs.Long2FloatMap;
 import it.unimi.dsi.fastutil.longs.Long2FloatOpenHashMap;
 import org.slf4j.Logger;
@@ -129,7 +131,7 @@ public class LRModel extends Model {
         int cnt = 0;
 
         Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
-        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        LOGGER.info("[MODELLOAD] before rankByScore load, key size: {}, current time: {}", lrModel.size(), curTime);
         //first stage
         while ((line = input.readLine()) != null) {
             String[] items = line.split("\t");
@@ -145,9 +147,9 @@ public class LRModel extends Model {
                 break;
             }
         }
-        //model update
+        //rankByScore update
         this.lrModel = model;
-        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        LOGGER.info("[MODELLOAD] after first stage rankByScore load, key size: {}, current time: {}", lrModel.size(), curTime);
         //final stage
         while ((line = input.readLine()) != null) {
             String[] items = line.split("\t");
@@ -156,9 +158,9 @@ public class LRModel extends Model {
             }
             putFeature(model, new BigInteger(items[0]).longValue(), Float.valueOf(items[1]).floatValue());
         }
-        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        LOGGER.info("[MODELLOAD] after rankByScore load, key size: {}, current time: {}", lrModel.size(), curTime);
 
-        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        LOGGER.info("[MODELLOAD] rankByScore load over and size " + cnt);
         input.close();
         in.close();
         return true;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/ModelManager.java

@@ -29,10 +29,10 @@ public class ModelManager {
     private OSS client;
     private String bucketName;
 
-    private final String modelOssEndpoint = "model.oss.internal.endpoint";
-    private final String modelOssAccessKeyId = "model.oss.accessKeyId";
-    private final String modelOssAccessKeySecret = "model.oss.accessKetSecret";
-    private final String modelOssBucketName = "model.oss.bucketName";
+    private final String modelOssEndpoint = "rankByScore.oss.internal.endpoint";
+    private final String modelOssAccessKeyId = "rankByScore.oss.accessKeyId";
+    private final String modelOssAccessKeySecret = "rankByScore.oss.accessKetSecret";
+    private final String modelOssBucketName = "rankByScore.oss.bucketName";
 
     private ModelManager() {
         // config load
@@ -153,9 +153,9 @@ public class ModelManager {
         final Runnable task = new Runnable() {
             public void run() {
                 // 模型更新开关
-                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.model-update-switch", true);
+                // boolean modelUpdateSwitch = Configuration.getBoolean("recommend-service-framework.rankByScore-update-switch", true);
                 boolean modelUpdateSwitch = true;
-                log.info("model update switch [{}]", modelUpdateSwitch);
+                log.info("rankByScore update switch [{}]", modelUpdateSwitch);
                 if (modelUpdateSwitch) {
                     updateModels(false);
                 }
@@ -170,7 +170,7 @@ public class ModelManager {
     public void updateModels(final boolean isForceLoads) {
         log.info("begin to update: [{}]", loadTasks.keySet().size());
         for (String modelPath : loadTasks.keySet()) {
-            log.debug("load task model path [{}]", modelPath);
+            log.debug("load task rankByScore path [{}]", modelPath);
             ModelLoadTask task = loadTasks.get(modelPath);
             loadModel(task, isForceLoads, false);
         }
@@ -194,7 +194,7 @@ public class ModelManager {
             ossObj = client.getObject(bucketName, loadTask.path);
             long timeStamp = ossObj.getObjectMetadata().getLastModified().getTime();
             if (loadTask.lastModifyTime < timeStamp || isForceLoads) {
-                log.info("model file changed, ready to update, last modify: [{}], current model time: [{}]",
+                log.info("rankByScore file changed, ready to update, last modify: [{}], current rankByScore time: [{}]",
                         loadTask.lastModifyTime, timeStamp);
 
                 Model model = loadTask.modelClass.newInstance();
@@ -205,7 +205,7 @@ public class ModelManager {
             }
             ossObj.close();
         } catch (Exception e) {
-            log.error("update model fail", e);
+            log.error("update rankByScore fail", e);
         } finally {
             loadTask.isLoading = false;
             if (ossObj != null) {
@@ -231,10 +231,10 @@ public class ModelManager {
     private class ModelLoadTask {
 
         private int refCount;
-        private String path;
+        private final String path;
         private long lastModifyTime;
         private boolean isLoading;
-        private Class<? extends Model> modelClass;
+        private final Class<? extends Model> modelClass;
         private Model model;
 
         ModelLoadTask(String path, Class<? extends Model> modelClass) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/ThompsonSamplingModel.java

@@ -2,7 +2,6 @@ package com.tzld.piaoquan.recommend.server.service.score.model;
 
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.common.base.VideoActionFeature;
-import com.tzld.piaoquan.recommend.server.model.Video;
 import org.apache.commons.math3.distribution.BetaDistribution;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -58,7 +57,7 @@ public class ThompsonSamplingModel extends Model {
         }
 
         this.thompsonSamplingModel = initModel;
-        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        LOGGER.info("[MODELLOAD] rankByScore load over and size " + cnt);
         input.close();
         in.close();
         return true;
@@ -76,8 +75,8 @@ public class ThompsonSamplingModel extends Model {
         double score = 0.0f;
         VideoActionFeature videoActionFeature = this.thompsonSamplingModel.getOrDefault(rankItem.getVideoId(), new VideoActionFeature());
 
-        int alpha = (int) videoActionFeature.getReturns() + this.alpha;
-        int beta = this.beta_returns + (int) videoActionFeature.getView();
+        int alpha = (int) videoActionFeature.getReturns() + ThompsonSamplingModel.alpha;
+        int beta = beta_returns + (int) videoActionFeature.getView();
         score = this.betaSampler(alpha, beta);
         return score;
     }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/AbstractScorer4Recall.java

@@ -37,14 +37,14 @@ public abstract class AbstractScorer4Recall {
             try {
                 // 使用 modelPath 作为 modelName 注册
                 modelManager.registerModel(modelPath, modelPath, modelClass);
-                LOGGER.info("register model success, model path [{}], model class [{}]", modelPath, modelClass);
+                LOGGER.info("register rankByScore success, rankByScore path [{}], rankByScore class [{}]", modelPath, modelClass);
             } catch (ModelManager.ModelRegisterException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+                LOGGER.error("register rankByScore fail [{}]:[{}]", modelPath, e);
             } catch (IOException e) {
-                LOGGER.error("register model fail [{}]:[{}]", modelPath, e);
+                LOGGER.error("register rankByScore fail [{}]:[{}]", modelPath, e);
             }
         } else {
-            LOGGER.error("modelpath is null, for model class [{}]", modelClass);
+            LOGGER.error("modelpath is null, for rankByScore class [{}]", modelClass);
         }
     }
     public Model getModel() {

recommend-server-service/src/main/resources/merge_config.conf

@@ -2,7 +2,7 @@ queue-config = {
   top-queue = {
     class = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue"
     children = {
-      hot-queue = {
+      region-hot-queue = {
         class = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue"
         children = {
           region1h-index = {
@@ -11,8 +11,23 @@ queue-config = {
           region3h-index = {
             class = "com.tzld.piaoquan.recommend.server.implement.candidate.Region3hHotCandidate"
           }
+          region24h-index = {
+            class = "com.tzld.piaoquan.recommend.server.implement.candidate.Region24hHotCandidate"
+          }
         }
       }
+      global-hot-queue = {
+          class = "com.tzld.piaoquan.recommend.server.framework.merger.SimpleMergeQueue"
+          global1h-index = {
+              class = "com.tzld.piaoquan.recommend.server.implement.candidate.Global1hHotCandidate"
+          }
+          global3h-index = {
+              class = "com.tzld.piaoquan.recommend.server.implement.candidate.Global3hHotCandidate"
+          }
+          global24h-index = {
+              class = "com.tzld.piaoquan.recommend.server.implement.candidate.Global24hHotCandidate"
+          }
+      }
     }
   }
 }
@@ -23,23 +38,50 @@ rule-config = {
   // 顶层队列
   top-queue = {
     merge-rule = {
-      hot-queue = {
-        recall-percentage = 1
+      global-hot-queue = {
+        recall-percentage = 0.3
+        min-merge-num = 1
+        priority = 1
+      }
+      region-hot-queue = {
+        recall-percentage = 0.6
         min-merge-num = 1
         priority = 1
       }
     }
   }
 
-  // 精选队列
-  hot-queue = {
+  // 全局热门队列
+  global-hot-queue = {
+    merge-rule = {
+      global1h-index = {
+        recall-percentage = 0.3
+        min-merge-num = 1
+      }
+      global3h-index = {
+        recall-percentage = 0.3
+        min-merge-num = 1
+      }
+      global24h-index = {
+        recall-percentage = 0.3
+        min-merge-num = 1
+      }
+    }
+  }
+
+
+  region-hot-queue = {
     merge-rule = {
       region1h-index = {
-        recall-percentage = 0.8
+        recall-percentage = 0.3
         min-merge-num = 1
       }
       region3h-index = {
-        recall-percentage = 0.2
+        recall-percentage = 0.3
+        min-merge-num = 1
+      }
+      region24h-index = {
+        recall-percentage = 0.3
         min-merge-num = 1
       }
     }