Merge branch 'feature/zhangbo_xgb' of algorithm/recommend-server into master

recommend-server-service/pom.xml

@@ -243,6 +243,28 @@
             <artifactId>abtest-client</artifactId>
             <version>1.0.0</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>2.12.15</version>
+        </dependency>
+        <dependency>
+            <groupId>ml.dmlc</groupId>
+            <artifactId>xgboost4j-spark_2.12</artifactId>
+            <version>1.7.6</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>scala-library</artifactId>
+                    <groupId>org.scala-lang</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-mllib_2.12</artifactId>
+            <version>3.3.1</version>
+        </dependency>
     </dependencies>
 
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/Application.java

@@ -23,12 +23,13 @@ import org.springframework.scheduling.annotation.EnableScheduling;
         RedisReactiveAutoConfiguration.class
 })
 @ComponentScan({
+        "com.tzld.piaoquan.recommend.server.util",
+        "com.tzld.piaoquan.recommend.server.config",
         "com.tzld.piaoquan.recommend.server.service",
         "com.tzld.piaoquan.recommend.server.implement",
         "com.tzld.piaoquan.recommend.server.framework.utils",
         "com.tzld.piaoquan.recommend.server.grpcservice",
         "com.tzld.piaoquan.recommend.server.remote",
-        "com.tzld.piaoquan.recommend.server.config",
         "com.tzld.piaoquan.recommend.server.web",
         "com.tzld.piaoquan.recommend.server.xxl",
 })

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/config/SparkConfig.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.recommend.server.config;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+
+@Component
+public class SparkConfig {
+
+    @PostConstruct
+    public void init() {
+        SparkConf sparkConf = new SparkConf()
+                .setMaster("local")
+                .setAppName("XGBoostPredict");
+        JavaSparkContext jsc = new JavaSparkContext(sparkConf);
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV563.java

@@ -1,7 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
-import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.FeatureService;
@@ -10,7 +9,6 @@ import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -35,7 +33,6 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
     Map<String, double[]> bucketsMap = new HashMap<>();
     Map<String, Double> bucketsLen = new HashMap<>();
 
-    String CLASS_NAME = this.getClass().getSimpleName();
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
@@ -72,13 +69,12 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         rovRecallRank.addAll(v1);
         setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
-
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
         //-------------------辑-------------------
 
-        // TODO 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
+        // 1 批量获取特征  省份参数要对齐  headvid  要传递过来！
         List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
 
         // k1:视频、k2:表、k3:特征、v:特征值
@@ -90,7 +86,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
 
 
-        // TODO 2 特征处理
+        // 2 特征处理
         Map<String, Double> userFeatureMapDouble = new HashMap<>();
         String mid = param.getMid();
         Map<String, String> c1 = featureOriginUser.getOrDefault("alg_mid_feature_play", new HashMap<>());
@@ -98,8 +94,8 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         Map<String, String> c3 = featureOriginUser.getOrDefault("alg_mid_feature_play_tags", new HashMap<>());
         Map<String, String> c4 = featureOriginUser.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
         Map<String, String> c5 = featureOriginUser.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
-        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags", new HashMap<>());
-        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags", new HashMap<>());
+        Map<String, String> c6 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_share_tags_v2", new HashMap<>());
+        Map<String, String> c7 = featureOriginUser.getOrDefault("alg_mid_feature_feed_exp_return_tags_v2", new HashMap<>());
         Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
         Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
 
@@ -169,21 +165,21 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         for (RankItem item : rankItems) {
             Map<String, Double> featureMap = new HashMap<>();
             String vid = item.getVideoId() + "";
-            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp", new HashMap<>());
+            Map<String, String> b1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_exp_v2", new HashMap<>());
             Map<String, String> b2 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_share", new HashMap<>());
             Map<String, String> b3 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_all_return", new HashMap<>());
-            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share", new HashMap<>());
+            Map<String, String> b6 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_exp2share_v2", new HashMap<>());
             Map<String, String> b7 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_share2return", new HashMap<>());
 
-            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp", new HashMap<>());
-            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share", new HashMap<>());
-            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return", new HashMap<>());
-            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp", new HashMap<>());
-            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share", new HashMap<>());
-            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return", new HashMap<>());
-            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp", new HashMap<>());
-            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share", new HashMap<>());
-            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return", new HashMap<>());
+            Map<String, String> b8 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_exp_v2", new HashMap<>());
+            Map<String, String> b9 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_share_v2", new HashMap<>());
+            Map<String, String> b10 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_noflow_root_return_v2", new HashMap<>());
+            Map<String, String> b11 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_exp_v2", new HashMap<>());
+            Map<String, String> b12 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_share_v2", new HashMap<>());
+            Map<String, String> b13 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_flow_root_return_v2", new HashMap<>());
+            Map<String, String> b17 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_exp_v2", new HashMap<>());
+            Map<String, String> b18 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_share_v2", new HashMap<>());
+            Map<String, String> b19 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_feed_province_root_return_v2", new HashMap<>());
 
             List<Tuple4> originData = Arrays.asList(
                     new Tuple4(b1, b2, b3, "b123"),
@@ -254,7 +250,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
                     }
                 }
             }
-            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
+            Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new_v2", new HashMap<>());
             if (!d1.isEmpty()) {
                 featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
                 featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
@@ -296,66 +292,37 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
             item.featureMap = featureMap;
         }
 
-        // 排序模型
+        // 3 排序
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240609.conf")
-                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
 
-        // 排序融合分
-        String redisScoreKey = "redis:vid_hasreturn1d7d_rov:";
-        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, redisScoreKey);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_xgb_20240828.conf")
+                .scoring(sceneFeatureMap, userFeatureMap, rankItems);
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_rov:");
+        Map<String, Map<String, String>> vid2VovFeatureMap = this.getVideoRedisFeature(vids, "redis:vid_vov_1d3d:");
+        double alpha_vov = mergeWeight.getOrDefault("alpha_vov", 2.0);
         List<Video> result = new ArrayList<>();
-        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
-        Double rosDefault = mergeWeight.getOrDefault("rosDefault", 0.1);
-        Double has_1d = mergeWeight.getOrDefault("has_1d", 1.0);
-        Double has_2d = mergeWeight.getOrDefault("has_2d", 1.0);
-        Double has_3d = mergeWeight.getOrDefault("has_3d", 1.0);
-        Double has_4d = mergeWeight.getOrDefault("has_4d", 0.0);
-        Double has_5d = mergeWeight.getOrDefault("has_5d", 0.0);
-        Double has_6d = mergeWeight.getOrDefault("has_6d", 0.0);
-        Double has_7d = mergeWeight.getOrDefault("has_7d", 0.0);
+//        String hasReturnRovKey = mergeWeight.getOrDefault("hasReturnRovKey", 1.0) < 0.5 ? "rate_1" : "rate_n";
+//        Double chooseFunction = mergeWeight.getOrDefault("chooseFunction", 0.0);
 
         for (RankItem item : items) {
             double score = 0.0;
-            Map<String, String> hasReturnRovScoreMap = vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>());
-            double hasReturnRovScore_1d = Double.parseDouble(hasReturnRovScoreMap.getOrDefault("rate_n_1d", "0"));
-            double hasReturnRovScore_2d = Double.parseDouble(hasReturnRovScoreMap.getOrDefault("rate_n_2d", "0"));
-            double hasReturnRovScore_3d = Double.parseDouble(hasReturnRovScoreMap.getOrDefault("rate_n_3d", "0"));
-            double hasReturnRovScore_4d = Double.parseDouble(hasReturnRovScoreMap.getOrDefault("rate_n_4d", "0"));
-            double hasReturnRovScore_5d = Double.parseDouble(hasReturnRovScoreMap.getOrDefault("rate_n_5d", "0"));
-            double hasReturnRovScore_6d = Double.parseDouble(hasReturnRovScoreMap.getOrDefault("rate_n_6d", "0"));
-            double hasReturnRovScore_7d = Double.parseDouble(hasReturnRovScoreMap.getOrDefault("rate_n_7d", "0"));
-            double hasReturnRovScore = (
-                    hasReturnRovScore_1d + hasReturnRovScore_2d + hasReturnRovScore_3d +
-                            hasReturnRovScore_4d + hasReturnRovScore_5d + hasReturnRovScore_6d
-                            + hasReturnRovScore_7d
-                    ) / 7;
-            if (has_1d + has_2d + has_3d + has_4d + has_5d + has_6d + has_7d > 0){
-                 hasReturnRovScore = (
-                        hasReturnRovScore_1d*has_1d + hasReturnRovScore_2d*has_2d + hasReturnRovScore_3d*has_3d +
-                                hasReturnRovScore_4d*has_4d + hasReturnRovScore_5d*has_5d + hasReturnRovScore_6d*has_6d
-                                + hasReturnRovScore_7d*has_7d
-                ) / (
-                        has_1d + has_2d + has_3d + has_4d + has_5d + has_6d + has_7d
-                );
-            }
-            item.getScoresMap().put("hasReturnRovScore_1d", hasReturnRovScore_1d);
-            item.getScoresMap().put("hasReturnRovScore_2d", hasReturnRovScore_2d);
-            item.getScoresMap().put("hasReturnRovScore_3d", hasReturnRovScore_3d);
-            item.getScoresMap().put("hasReturnRovScore_4d", hasReturnRovScore_4d);
-            item.getScoresMap().put("hasReturnRovScore_5d", hasReturnRovScore_5d);
-            item.getScoresMap().put("hasReturnRovScore_6d", hasReturnRovScore_6d);
-            item.getScoresMap().put("hasReturnRovScore_7d", hasReturnRovScore_7d);
+            double recommend_rate_1d = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("recommend_rate_1d", "0"));
+            double recommend_exp_per_1d = Double.parseDouble(vid2VovFeatureMap.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("recommend_exp_per_1d", "0"));
+            double vorScore =  recommend_rate_1d * recommend_exp_per_1d;
+            item.getScoresMap().put("recommend_rate_1d", recommend_rate_1d);
+            item.getScoresMap().put("recommend_exp_per_1d", recommend_exp_per_1d);
+            item.getScoresMap().put("vorScore", vorScore);
+            item.getScoresMap().put("alpha_vov", alpha_vov);
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>())
+                    .getOrDefault("rate_n", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
-            double fmRov = item.getScoreRov();
-            item.getScoresMap().put("fmRov", fmRov);
-            if (chooseFunction == 0){
-                score = fmRov * (rosDefault + hasReturnRovScore);
-            }else if (chooseFunction == 1){
-                score = fmRov * (1 + Math.log(hasReturnRovScore + 1));
-            }else {
-                score = fmRov * (1 + hasReturnRovScore);
-            }
+            double rovRateOrigin = item.getScoreRov();
+            item.getScoresMap().put("rovRateOrigin", rovRateOrigin);
+            double rovRate = restoreScore(rovRateOrigin);
+            item.getScoresMap().put("rovRate", rovRate);
+            score = rovRate * (1 + hasReturnRovScore) * (1.0 + alpha_vov * recommend_rate_1d);
 
             Video video = item.getVideo();
             video.setScore(score);
@@ -378,23 +345,8 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         return result;
     }
 
-    private Map<String, Map<String, String>> extractVideoFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
-    private Map<String, String> extractSceneFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
-    private Map<String, String> extractUserFeature(Map<String, Map<String, Map<String, String>>> featureMap) {
-        // TODO
-        return null;
-    }
-
     private void readBucketFile() {
-        InputStream resourceStream = RankStrategy4RegionMergeModelBasic.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
+        InputStream resourceStream = RankStrategy4RegionMergeModelV563.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
         if (resourceStream != null) {
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
                 Map<String, double[]> bucketsMap = new HashMap<>();
@@ -424,7 +376,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         } else {
             log.error("no bucket file");
         }
-
     }
 
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/BaseXGBoostModelScorer.java

@@ -0,0 +1,33 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+import com.google.common.reflect.TypeToken;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+import com.tzld.piaoquan.recommend.server.service.score.model.XGBoostModel;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+
+public abstract class BaseXGBoostModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseXGBoostModelScorer.class);
+
+    public BaseXGBoostModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(XGBoostModel.class);
+        XGBoostModel model = (XGBoostModel) this.getModel();
+        ConfigObject paramMap = scorerConfigInfo.getParamMap();
+        if (paramMap != null) {
+            ConfigValue value = paramMap.get("features");
+            List<String> features = (List<String>) value.unwrapped();
+            model.setFeatures(features.toArray(new String[features.size()]));
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerConfig.java

@@ -1,18 +1,17 @@
 package com.tzld.piaoquan.recommend.server.service.score;
 
 
+import com.google.common.reflect.TypeToken;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import com.typesafe.config.ConfigObject;
 import com.typesafe.config.ConfigValue;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 
 
 public class ScorerConfig {
@@ -100,6 +99,12 @@ public class ScorerConfig {
             if (conf.hasPath("disable-switch")) {
                 disableSwitch = conf.getBoolean("disable-switch");
             }
+            ConfigObject paramMap = null;
+            if (conf.hasPath("param")) {
+                paramMap = conf.getObject("param");
+            }
+
+
             Config paramConfig = loadOptionConfig(conf, "param-config");
             // model path
             String modelPath = loadOptionStringConfig(conf, "model-path");
@@ -118,7 +123,8 @@ public class ScorerConfig {
                     disableSwitch,
                     enableQueues,
                     modelPath,
-                    paramConfig
+                    paramConfig,
+                    paramMap
             );
             LOGGER.debug("parse scorer config info [{}]", configInfo);
             // add to ConfigInfoList

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerConfigInfo.java

@@ -2,6 +2,10 @@ package com.tzld.piaoquan.recommend.server.service.score;
 
 import com.google.gson.Gson;
 import com.typesafe.config.Config;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigValue;
+
+import java.util.Map;
 import java.util.Set;
 
 
@@ -14,6 +18,7 @@ public class ScorerConfigInfo {
     private Set<String> enableQueues;
     private String modelPath;
     private Config paramConfig; // param config
+    private ConfigObject paramMap;
 
     public ScorerConfigInfo(String configName,
                             String scorerName,
@@ -21,7 +26,8 @@ public class ScorerConfigInfo {
                             Boolean disableSwitch,
                             Set<String> enableQueues,
                             String modelPath,
-                            Config paramConfig) {
+                            Config paramConfig,
+                            ConfigObject paramMap) {
 
         this.configName = configName;
         this.scorerName = scorerName;
@@ -30,6 +36,7 @@ public class ScorerConfigInfo {
         this.enableQueues = enableQueues;
         this.modelPath = modelPath;
         this.paramConfig = paramConfig;
+        this.paramMap = paramMap;
     }
 
     public Config getParamConfig() {
@@ -70,6 +77,10 @@ public class ScorerConfigInfo {
         return disableSwitch;
     }
 
+    public ConfigObject getParamMap(){
+        return paramMap;
+    }
+
     @Override
     public String toString() {
         return new Gson().toJson(this);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -38,6 +38,7 @@ public final class ScorerUtils {
         ScorerUtils.init("feeds_score_config_20240711.conf");
         ScorerUtils.init("feeds_score_config_20240806.conf");
         ScorerUtils.init("feeds_score_config_20240807.conf");
+        ScorerUtils.init("feeds_score_config_xgb_20240828.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v1.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v2.conf");
         ScorerUtils.init4Recall("feeds_recall_config_region_v3.conf");

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/XGBoostScorer.java

@@ -0,0 +1,159 @@
+package com.tzld.piaoquan.recommend.server.service.score;
+
+
+import com.tzld.piaoquan.recommend.feature.domain.video.base.UserFeature;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.service.score.model.XGBoostModel;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class XGBoostScorer extends BaseXGBoostModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(XGBoostScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public XGBoostScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<RankItem> scoring(final ScoreParam param,
+                                  final UserFeature userFeature,
+                                  final List<RankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    @Override
+    public List<RankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                  final Map<String, String> userFeatureMap,
+                                  final List<RankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        XGBoostModel model = (XGBoostModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<RankItem> result = rankItems;
+        result = rankByJava(
+                sceneFeatureMap, userFeatureMap, rankItems
+        );
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<RankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<RankItem> items) {
+        long startTime = System.currentTimeMillis();
+        XGBoostModel model = (XGBoostModel) this.getModel();
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<RankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final XGBoostModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).videoId, ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        //等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final XGBoostModel model,
+                            final RankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        Map<String, String> featureMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+            featureMap.putAll(item.getFeatureMap());
+        }
+        if (MapUtils.isNotEmpty(userFeatureMap)) {
+            featureMap.putAll(userFeatureMap);
+        }
+        if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+            featureMap.putAll(sceneFeatureMap);
+        }
+
+        double pro = 0.0;
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                pro = model.score(featureMap);
+                // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setScoreRov(pro);
+        item.getScoresMap().put("RovFMScore", pro);
+        item.setAllFeatureMap(featureMap);
+        return pro;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/Model.java

@@ -1,11 +1,15 @@
 package com.tzld.piaoquan.recommend.server.service.score.model;
 
 
+import java.io.InputStream;
 import java.io.InputStreamReader;
 
 abstract public class Model {
     public abstract int getModelSize();
 
     public abstract boolean loadFromStream(InputStreamReader in) throws Exception;
+    public boolean loadFromStream(InputStream is) throws Exception {
+        return loadFromStream(new InputStreamReader(is));
+    }
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/ModelManager.java

@@ -205,7 +205,7 @@ public class ModelManager {
                         loadTask.lastModifyTime, timeStamp);
 
                 Model model = loadTask.modelClass.newInstance();
-                if (model.loadFromStream(new InputStreamReader(ossObj.getObjectContent()))) {
+                if (model.loadFromStream(ossObj.getObjectContent())) {
                     loadTask.model = model;
                     loadTask.lastModifyTime = timeStamp;
                 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/model/XGBoostModel.java

@@ -0,0 +1,71 @@
+package com.tzld.piaoquan.recommend.server.service.score.model;
+
+
+import com.tzld.piaoquan.recommend.server.util.CompressUtil;
+import com.tzld.piaoquan.recommend.server.util.PropertiesUtil;
+import ml.dmlc.xgboost4j.scala.DMatrix;
+import ml.dmlc.xgboost4j.scala.spark.XGBoostClassificationModel;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Map;
+
+
+public class XGBoostModel extends Model {
+    private static final Logger LOGGER = LoggerFactory.getLogger(XGBoostModel.class);
+    private XGBoostClassificationModel model;
+
+    private String[] features;
+
+    public void setFeatures(String[] features){
+        this.features = features;
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.model == null)
+            return 0;
+        return 1;
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws Exception {
+        return false;
+    }
+
+    public void cleanModel() {
+        this.model = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+
+        try {
+            float[] values = new float[features.length];
+            for (int i = 0; i < features.length; i++) {
+                float v = NumberUtils.toFloat(featureMap.getOrDefault(features[i], "0.0"), 0.0f);
+                values[i] = v;
+            }
+            DMatrix dm = new DMatrix(values, 1, features.length, 0.0f);
+            float[][] result = model._booster().predict(dm, false, 100);
+            return result[0][0];
+        } catch (Exception e) {
+            return 0f;
+        }
+    }
+
+    @Override
+    public boolean loadFromStream(InputStream in) throws Exception {
+        String modelDir = PropertiesUtil.getString("model.xgboost.path");
+        CompressUtil.decompressGzFile(in, modelDir);
+        String absolutePath =new File(modelDir).getAbsolutePath();
+        XGBoostClassificationModel model2 = XGBoostClassificationModel.load("file://" + absolutePath);
+        model2.setMissing(0.0f);
+        this.model = model2;
+        return true;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/CompressUtil.java

@@ -0,0 +1,123 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * @author dyp
+ */
+@Slf4j
+public class CompressUtil {
+    public static void compressDirectoryToGzip(String sourceDirPath, String outputFilePath) {
+        // 创建.gz文件的输出流
+        try (OutputStream out = new FileOutputStream(outputFilePath);
+             GzipCompressorOutputStream gzipOut = new GzipCompressorOutputStream(out);
+             TarArchiveOutputStream taos = new TarArchiveOutputStream(gzipOut)) {
+
+            taos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
+
+            // 遍历目录
+            Files.walk(Paths.get(sourceDirPath))
+                    .filter(Files::isRegularFile)
+                    .forEach(filePath -> {
+                        try {
+                            // 为每个文件创建TarEntry
+                            TarArchiveEntry entry = new TarArchiveEntry(filePath.toFile(), filePath.toString().substring(sourceDirPath.length() + 1));
+                            taos.putArchiveEntry(entry);
+
+                            // 读取文件内容并写入TarArchiveOutputStream
+                            try (InputStream is = Files.newInputStream(filePath)) {
+                                byte[] buffer = new byte[1024];
+                                int len;
+                                while ((len = is.read(buffer)) > 0) {
+                                    taos.write(buffer, 0, len);
+                                }
+                            }
+                            // 关闭entry
+                            taos.closeArchiveEntry();
+                        } catch (IOException e) {
+                            log.error("", e);
+                        }
+                    });
+        } catch (Exception e) {
+            log.error("", e);
+        }
+    }
+
+    public static void decompressGzFile(String gzipFilePath, String destDirPath) {
+        try (InputStream gzipIn = new FileInputStream(gzipFilePath);
+             GzipCompressorInputStream gzIn = new GzipCompressorInputStream(gzipIn);
+             TarArchiveInputStream tais = new TarArchiveInputStream(gzIn)) {
+
+            TarArchiveEntry entry;
+            Files.createDirectories(Paths.get(destDirPath));
+            while ((entry = tais.getNextTarEntry()) != null) {
+                if (entry.isDirectory()) {
+                    // 如果是目录，创建目录
+                    Files.createDirectories(Paths.get(destDirPath, entry.getName()));
+                } else {
+                    // 如果是文件，创建文件并写入内容
+                    File outputFile = new File(destDirPath, entry.getName());
+                    if (!outputFile.exists()) {
+                        File parent = outputFile.getParentFile();
+                        if (!parent.exists()) {
+                            parent.mkdirs();
+                        }
+                        outputFile.createNewFile();
+                    }
+                    try (OutputStream out = new FileOutputStream(outputFile)) {
+                        byte[] buffer = new byte[1024];
+                        int len;
+                        while ((len = tais.read(buffer)) > 0) {
+                            out.write(buffer, 0, len);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("", e);
+        }
+    }
+
+    public static void decompressGzFile(InputStream gzipIn, String destDirPath) {
+        try (GzipCompressorInputStream gzIn = new GzipCompressorInputStream(gzipIn);
+             TarArchiveInputStream tais = new TarArchiveInputStream(gzIn)) {
+
+            TarArchiveEntry entry;
+            Files.createDirectories(Paths.get(destDirPath));
+            while ((entry = tais.getNextTarEntry()) != null) {
+                if (entry.isDirectory()) {
+                    // 如果是目录，创建目录
+                    Files.createDirectories(Paths.get(destDirPath, entry.getName()));
+                } else {
+                    // 如果是文件，创建文件并写入内容
+                    File outputFile = new File(destDirPath, entry.getName());
+                    if (!outputFile.exists()) {
+                        File parent = outputFile.getParentFile();
+                        if (!parent.exists()) {
+                            parent.mkdirs();
+                        }
+                        outputFile.createNewFile();
+                    }
+                    try (OutputStream out = new FileOutputStream(outputFile)) {
+                        byte[] buffer = new byte[1024];
+                        int len;
+                        while ((len = tais.read(buffer)) > 0) {
+                            out.write(buffer, 0, len);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("", e);
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/PropertiesUtil.java

@@ -0,0 +1,22 @@
+package com.tzld.piaoquan.recommend.server.util;
+
+import org.springframework.context.EnvironmentAware;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+@Component
+public class PropertiesUtil implements EnvironmentAware {
+
+
+    private static Environment environment;
+
+
+    @Override
+    public void setEnvironment(Environment environment) {
+        this.environment = environment;
+    }
+
+    public static String getString(String name) {
+        return environment.getProperty(name);
+    }
+}

recommend-server-service/src/main/resources/application.yml

@@ -58,4 +58,7 @@ feign:
     config:
       default:
         connectTimeout: 2000
-        readTimeout: 10000
+        readTimeout: 10000
+model:
+  xgboost:
+    path: xgboost

recommend-server-service/src/main/resources/feeds_score_config_xgb_20240828.conf

@@ -0,0 +1,286 @@
+scorer-config = {
+  lr-rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.XGBoostScorer"
+    scorer-priority = 99
+    model-path = "zhangbo/model_xgb_for_recsys.tar.gz"
+    param = {
+      features = [
+        "b123_1h_STR",
+        "b123_1h_log(share)",
+        "b123_1h_ROV",
+        "b123_1h_log(return)",
+        "b123_1h_ROV*log(return)",
+        "b123_2h_STR",
+        "b123_2h_log(share)",
+        "b123_2h_ROV",
+        "b123_2h_log(return)",
+        "b123_2h_ROV*log(return)",
+        "b123_3h_STR",
+        "b123_3h_log(share)",
+        "b123_3h_ROV",
+        "b123_3h_log(return)",
+        "b123_3h_ROV*log(return)",
+        "b123_4h_STR",
+        "b123_4h_log(share)",
+        "b123_4h_ROV",
+        "b123_4h_log(return)",
+        "b123_4h_ROV*log(return)",
+        "b123_12h_STR",
+        "b123_12h_log(share)",
+        "b123_12h_ROV",
+        "b123_12h_log(return)",
+        "b123_12h_ROV*log(return)",
+        "b123_1d_STR",
+        "b123_1d_log(share)",
+        "b123_1d_ROV",
+        "b123_1d_log(return)",
+        "b123_1d_ROV*log(return)",
+        "b123_3d_STR",
+        "b123_3d_log(share)",
+        "b123_3d_ROV",
+        "b123_3d_log(return)",
+        "b123_3d_ROV*log(return)",
+        "b123_7d_STR",
+        "b123_7d_log(share)",
+        "b123_7d_ROV",
+        "b123_7d_log(return)",
+        "b123_7d_ROV*log(return)",
+        "b167_1h_STR",
+        "b167_1h_log(share)",
+        "b167_1h_ROV",
+        "b167_1h_log(return)",
+        "b167_1h_ROV*log(return)",
+        "b167_2h_STR",
+        "b167_2h_log(share)",
+        "b167_2h_ROV",
+        "b167_2h_log(return)",
+        "b167_2h_ROV*log(return)",
+        "b167_3h_STR",
+        "b167_3h_log(share)",
+        "b167_3h_ROV",
+        "b167_3h_log(return)",
+        "b167_3h_ROV*log(return)",
+        "b167_4h_STR",
+        "b167_4h_log(share)",
+        "b167_4h_ROV",
+        "b167_4h_log(return)",
+        "b167_4h_ROV*log(return)",
+        "b167_12h_STR",
+        "b167_12h_log(share)",
+        "b167_12h_ROV",
+        "b167_12h_log(return)",
+        "b167_12h_ROV*log(return)",
+        "b167_1d_STR",
+        "b167_1d_log(share)",
+        "b167_1d_ROV",
+        "b167_1d_log(return)",
+        "b167_1d_ROV*log(return)",
+        "b167_3d_STR",
+        "b167_3d_log(share)",
+        "b167_3d_ROV",
+        "b167_3d_log(return)",
+        "b167_3d_ROV*log(return)",
+        "b167_7d_STR",
+        "b167_7d_log(share)",
+        "b167_7d_ROV",
+        "b167_7d_log(return)",
+        "b167_7d_ROV*log(return)",
+        "b8910_1h_STR",
+        "b8910_1h_log(share)",
+        "b8910_1h_ROV",
+        "b8910_1h_log(return)",
+        "b8910_1h_ROV*log(return)",
+        "b8910_2h_STR",
+        "b8910_2h_log(share)",
+        "b8910_2h_ROV",
+        "b8910_2h_log(return)",
+        "b8910_2h_ROV*log(return)",
+        "b8910_3h_STR",
+        "b8910_3h_log(share)",
+        "b8910_3h_ROV",
+        "b8910_3h_log(return)",
+        "b8910_3h_ROV*log(return)",
+        "b8910_4h_STR",
+        "b8910_4h_log(share)",
+        "b8910_4h_ROV",
+        "b8910_4h_log(return)",
+        "b8910_4h_ROV*log(return)",
+        "b8910_12h_STR",
+        "b8910_12h_log(share)",
+        "b8910_12h_ROV",
+        "b8910_12h_log(return)",
+        "b8910_12h_ROV*log(return)",
+        "b8910_1d_STR",
+        "b8910_1d_log(share)",
+        "b8910_1d_ROV",
+        "b8910_1d_log(return)",
+        "b8910_1d_ROV*log(return)",
+        "b8910_3d_STR",
+        "b8910_3d_log(share)",
+        "b8910_3d_ROV",
+        "b8910_3d_log(return)",
+        "b8910_3d_ROV*log(return)",
+        "b8910_7d_STR",
+        "b8910_7d_log(share)",
+        "b8910_7d_ROV",
+        "b8910_7d_log(return)",
+        "b8910_7d_ROV*log(return)",
+        "b111213_1h_STR",
+        "b111213_1h_log(share)",
+        "b111213_1h_ROV",
+        "b111213_1h_log(return)",
+        "b111213_1h_ROV*log(return)",
+        "b111213_2h_STR",
+        "b111213_2h_log(share)",
+        "b111213_2h_ROV",
+        "b111213_2h_log(return)",
+        "b111213_2h_ROV*log(return)",
+        "b111213_3h_STR",
+        "b111213_3h_log(share)",
+        "b111213_3h_ROV",
+        "b111213_3h_log(return)",
+        "b111213_3h_ROV*log(return)",
+        "b111213_4h_STR",
+        "b111213_4h_log(share)",
+        "b111213_4h_ROV",
+        "b111213_4h_log(return)",
+        "b111213_4h_ROV*log(return)",
+        "b111213_12h_STR",
+        "b111213_12h_log(share)",
+        "b111213_12h_ROV",
+        "b111213_12h_log(return)",
+        "b111213_12h_ROV*log(return)",
+        "b111213_1d_STR",
+        "b111213_1d_log(share)",
+        "b111213_1d_ROV",
+        "b111213_1d_log(return)",
+        "b111213_1d_ROV*log(return)",
+        "b111213_3d_STR",
+        "b111213_3d_log(share)",
+        "b111213_3d_ROV",
+        "b111213_3d_log(return)",
+        "b111213_3d_ROV*log(return)",
+        "b111213_7d_STR",
+        "b111213_7d_log(share)",
+        "b111213_7d_ROV",
+        "b111213_7d_log(return)",
+        "b111213_7d_ROV*log(return)",
+        "b171819_1h_STR",
+        "b171819_1h_log(share)",
+        "b171819_1h_ROV",
+        "b171819_1h_log(return)",
+        "b171819_1h_ROV*log(return)",
+        "b171819_2h_STR",
+        "b171819_2h_log(share)",
+        "b171819_2h_ROV",
+        "b171819_2h_log(return)",
+        "b171819_2h_ROV*log(return)",
+        "b171819_3h_STR",
+        "b171819_3h_log(share)",
+        "b171819_3h_ROV",
+        "b171819_3h_log(return)",
+        "b171819_3h_ROV*log(return)",
+        "b171819_4h_STR",
+        "b171819_4h_log(share)",
+        "b171819_4h_ROV",
+        "b171819_4h_log(return)",
+        "b171819_4h_ROV*log(return)",
+        "b171819_12h_STR",
+        "b171819_12h_log(share)",
+        "b171819_12h_ROV",
+        "b171819_12h_log(return)",
+        "b171819_12h_ROV*log(return)",
+        "b171819_1d_STR",
+        "b171819_1d_log(share)",
+        "b171819_1d_ROV",
+        "b171819_1d_log(return)",
+        "b171819_1d_ROV*log(return)",
+        "b171819_3d_STR",
+        "b171819_3d_log(share)",
+        "b171819_3d_ROV",
+        "b171819_3d_log(return)",
+        "b171819_3d_ROV*log(return)",
+        "b171819_7d_STR",
+        "b171819_7d_log(share)",
+        "b171819_7d_ROV",
+        "b171819_7d_log(return)",
+        "b171819_7d_ROV*log(return)",
+        "total_time",
+        "bit_rate",
+        "playcnt_6h",
+        "playcnt_1d",
+        "playcnt_3d",
+        "playcnt_7d",
+        "share_pv_12h",
+        "share_pv_1d",
+        "share_pv_3d",
+        "share_pv_7d",
+        "return_uv_12h",
+        "return_uv_1d",
+        "return_uv_3d",
+        "return_uv_7d",
+        "c3_feature_tags_1d_matchnum",
+        "c3_feature_tags_1d_maxscore",
+        "c3_feature_tags_1d_avgscore",
+        "c3_feature_tags_3d_matchnum",
+        "c3_feature_tags_3d_maxscore",
+        "c3_feature_tags_3d_avgscore",
+        "c3_feature_tags_7d_matchnum",
+        "c3_feature_tags_7d_maxscore",
+        "c3_feature_tags_7d_avgscore",
+        "c4_feature_tags_1d_matchnum",
+        "c4_feature_tags_1d_maxscore",
+        "c4_feature_tags_1d_avgscore",
+        "c4_feature_tags_3d_matchnum",
+        "c4_feature_tags_3d_maxscore",
+        "c4_feature_tags_3d_avgscore",
+        "c4_feature_tags_7d_matchnum",
+        "c4_feature_tags_7d_maxscore",
+        "c4_feature_tags_7d_avgscore",
+        "c5_feature_tags_1d_matchnum",
+        "c5_feature_tags_1d_maxscore",
+        "c5_feature_tags_1d_avgscore",
+        "c5_feature_tags_3d_matchnum",
+        "c5_feature_tags_3d_maxscore",
+        "c5_feature_tags_3d_avgscore",
+        "c5_feature_tags_7d_matchnum",
+        "c5_feature_tags_7d_maxscore",
+        "c5_feature_tags_7d_avgscore",
+        "c6_feature_tags_1d_matchnum",
+        "c6_feature_tags_1d_maxscore",
+        "c6_feature_tags_1d_avgscore",
+        "c6_feature_tags_3d_matchnum",
+        "c6_feature_tags_3d_maxscore",
+        "c6_feature_tags_3d_avgscore",
+        "c6_feature_tags_7d_matchnum",
+        "c6_feature_tags_7d_maxscore",
+        "c6_feature_tags_7d_avgscore",
+        "c7_feature_tags_1d_matchnum",
+        "c7_feature_tags_1d_maxscore",
+        "c7_feature_tags_1d_avgscore",
+        "c7_feature_tags_3d_matchnum",
+        "c7_feature_tags_3d_maxscore",
+        "c7_feature_tags_3d_avgscore",
+        "c7_feature_tags_7d_matchnum",
+        "c7_feature_tags_7d_maxscore",
+        "c7_feature_tags_7d_avgscore",
+        "c8_feature_share_score",
+        "c8_feature_share_num",
+        "c8_feature_share_rank",
+        "c8_feature_return_score",
+        "c8_feature_return_num",
+        "c8_feature_return_rank",
+        "c9_feature_share_score",
+        "c9_feature_share_num",
+        "c9_feature_share_rank",
+        "c9_feature_return_score",
+        "c9_feature_return_num",
+        "c9_feature_return_rank",
+        "d1_exp",
+        "d1_return_n",
+        "d1_rovn"
+      ]
+    }
+  }
+
+}