Merge branch 'feature/rank_model_v5' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/FeatureService.java

@@ -1,11 +1,13 @@
 package com.tzld.piaoquan.recommend.server.service;
 
+import com.alibaba.fastjson.JSON;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
 import com.tzld.piaoquan.recommend.server.model.MachineInfo;
 import com.tzld.piaoquan.recommend.server.remote.FeatureV2RemoteService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
@@ -360,6 +362,28 @@ public class FeatureService {
         return this.getFeatureByProto(protos);
     }
 
+    public UserShareReturnProfile getUserProfile(String mid) {
+        try {
+            String table = "alg_recsys_feature_user_share_return_stat";
+            List<FeatureKeyProto> protos = new ArrayList<>();
+            protos.add(genWithMid(table, mid));
+            Feature feature = getFeatureByProto(protos);
+            Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+            if (null != userFeature) {
+                Map<String, String> c9 = userFeature.get(table);
+                if (null != c9 && !c9.isEmpty()) {
+                    String c9Str = JSONUtils.toJson(c9);
+                    if (!c9Str.isEmpty()) {
+                        return JSON.parseObject(c9Str, UserShareReturnProfile.class);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("get user profile error! value=[{}]", mid, e);
+        }
+        return null;
+    }
+
     private Feature getFeatureByProto(List<FeatureKeyProto> protos) {
         Map<String, String> result = remoteService.getFeature(protos);
         Feature feature = new Feature();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV567.java

@@ -1,17 +1,21 @@
 package com.tzld.piaoquan.recommend.server.service.rank.strategy;
 
+import com.alibaba.fastjson.JSON;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.common.base.RankItem;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.bo.UserSRBO;
+import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
 import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
+import com.tzld.piaoquan.recommend.server.service.rank.tansform.NORFeature;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
-import com.tzld.piaoquan.recommend.server.util.SimilarityUtils;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.math3.util.Pair;
@@ -32,15 +36,6 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
     @Autowired
     private FeatureService featureService;
 
-    private static final List<String> shortPeriod = Arrays.asList("1h", "2h", "4h", "6h", "12h", "24h", "7d");
-    private static final List<String> middlePeriod = Arrays.asList("14d", "30d");
-    private static final List<String> longPeriod = Arrays.asList("7d", "35d", "90d", "365d");
-    private static final List<String> cfRosList = Collections.singletonList("rosn");
-    private static final List<String> cfRovList = Collections.singletonList("rovn");
-    private static final List<String> videoSimAttrs = Arrays.asList("cate1_list", "cate2", "cate2_list",
-            "keywords", "style", "theme", "title", "topic", "user_value");
-    private static final double smoothPlus = 5.0;
-
     @Override
     public List<Video> mergeAndRankRovRecall(RankParam param) {
         Map<String, Double> mergeWeight = this.mergeWeight != null ? this.mergeWeight : new HashMap<>(0);
@@ -49,6 +44,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         //-------------------逻-------------------
         //-------------------辑-------------------
 
+        long currentMs = System.currentTimeMillis();
         List<Video> oldRovs = new ArrayList<>();
         oldRovs.addAll(extractAndSort(param, RegionHRecallStrategy.PUSH_FORM));
         oldRovs.addAll(extractAndSort(param, RegionHDupRecallStrategy.PUSH_FORM));
@@ -73,7 +69,7 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         //-------------------新地域召回------------------
         List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
         v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 10.0).intValue(), v1.size()));
+        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
         rovRecallRank.addAll(v1);
         setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
         //-------------------scene cf rovn------------------
@@ -88,6 +84,12 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
         rovRecallRank.addAll(sceneCFRosn);
         setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        // -------------------cate1------------------
+        int cate1RecallN = mergeWeight.getOrDefault("cate1RecallN", 5.0).intValue();
+        addRecall(param, cate1RecallN, UserCate1RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+        // -------------------cate2------------------
+        int cate2RecallN = mergeWeight.getOrDefault("cate2RecallN", 5.0).intValue();
+        addRecall(param, cate2RecallN, UserCate2RecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -98,15 +100,19 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
 
         // k1:视频、k2:表、k3:特征、v:特征值
-        String provinceCn = param.getProvince().replaceAll("省$", "");
         String headVid = String.valueOf(param.getHeadVid());
-        String sceneType = String.valueOf(param.getHotSceneType());
         Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo(headVid, vids);
-        FeatureService.Feature feature = featureService.getNewFeature(provinceCn, param.getMid(), sceneType, headVid, videoBaseInfoMap, vids);
+        FeatureService.Feature feature = featureService.getFeatureV3(param, videoBaseInfoMap, vids);
         Map<String, Map<String, String>> featureOriginUser = feature.getUserFeature();
         Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
         Map<String, String> headVideoInfo = videoBaseInfoMap.getOrDefault(headVid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
 
+        // 用户信息预处理
+        Map<String, Map<String, String[]>> newC7Map = NORFeature.parseUCFScore(featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>()));
+        Map<String, Map<String, String[]>> newC8Map = NORFeature.parseUCFScore(featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>()));
+        UserShareReturnProfile userProfile = parseUserProfile(featureOriginUser);
+        Map<String, Map<String, String>> userBehaviorVideoMap = getUserBehaviorVideoMap(userProfile);
+
         // 2 特征处理
         Map<String, Double> userFeatureMapDouble = new HashMap<>();
         String mid = param.getMid();
@@ -120,9 +126,6 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
         Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
 
-        // time feature
-        addTimeFeature(userFeatureMapDouble);
-
         if (!c1.isEmpty()) {
             userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
             userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
@@ -219,29 +222,23 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
                     double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
                     double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
 
-                    double f0 = ExtractorUtils.calLog(exp);
                     double f1 = ExtractorUtils.calDiv(share, exp);
                     double f2 = ExtractorUtils.calLog(share);
                     double f3 = ExtractorUtils.calDiv(returns, exp);
                     double f4 = ExtractorUtils.calLog(returns);
                     double f5 = f3 * f4;
-                    double f6 = ExtractorUtils.calDiv(returns, share);
 
-                    String key0 = tuple4.name + "_" + prefix2 + "_" + "EXP";
                     String key1 = tuple4.name + "_" + prefix2 + "_" + "STR";
                     String key2 = tuple4.name + "_" + prefix2 + "_" + "log(share)";
                     String key3 = tuple4.name + "_" + prefix2 + "_" + "ROV";
                     String key4 = tuple4.name + "_" + prefix2 + "_" + "log(return)";
                     String key5 = tuple4.name + "_" + prefix2 + "_" + "ROV*log(return)";
-                    String key6 = tuple4.name + "_" + prefix2 + "_" + "ROS";
 
-                    featureMap.put(key0, f0);
                     featureMap.put(key1, f1);
                     featureMap.put(key2, f2);
                     featureMap.put(key3, f3);
                     featureMap.put(key4, f4);
                     featureMap.put(key5, f5);
-                    featureMap.put(key6, f6);
                 }
             }
 
@@ -299,18 +296,17 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
                 featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
                 featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
             }
-            // ******************** new feature ********************
-            addVideoStatFeature(vid, featureOriginVideo, featureMap);
-            //addVideoCFFeature(vid, featureOriginVideo, featureMap);
-            addVideoSimFeature(headVideoInfo, videoInfo, featureMap);
-
             item.featureMapDouble = featureMap;
         }
 
+        // get nor feature
+        Map<String, String> norUserFeatureMap = getNorUserFeature(currentMs, headVideoInfo, userProfile, featureOriginUser);
+        batchGetNorVideoFeature(currentMs, userProfile, headVideoInfo, videoBaseInfoMap,
+                newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, rankItems);
+
         // 3 连续值特征分桶
         readBucketFile();
         Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
-        Map<String, String> norUserFeatureMap = FeatureBucketUtils.noBucketFeature(userFeatureMapDouble);
         for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
             String name = entry.getKey();
             Double score = entry.getValue();
@@ -338,19 +334,12 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
                 }
             }
             item.featureMap = featureMap;
-            item.norFeatureMap = FeatureBucketUtils.noBucketFeature(featureMapDouble);
         }
         // 4 排序模型计算
-        double fmRovLogBase = mergeWeight.getOrDefault("fmRovLogBase", 2.8);
-        double xgbNorScaleType = mergeWeight.getOrDefault("xgbNorScaleType", 0.0);
-        double xgbNorBias = mergeWeight.getOrDefault("xgbNorBias", -1.6945);
-        double xgbNorWeight = mergeWeight.getOrDefault("xgbNorWeight", 1.8968);
-        double xgbNorPowerWeight = mergeWeight.getOrDefault("xgbNorPowerWeight", 1.2216);
-        double xgbNorPowerExp = mergeWeight.getOrDefault("xgbNorPowerExp", 1.3217);
-        double vorLogWeight = mergeWeight.getOrDefault("vorLogWeight", 10.0);
-        double vorLogBase = mergeWeight.getOrDefault("vorLogBase", 2.6);
+        double xgbNorPowerWeight = mergeWeight.getOrDefault("xgbNorPowerWeight", 1.22);
+        double xgbNorPowerExp = mergeWeight.getOrDefault("xgbNorPowerExp", 1.24);
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250208.conf").scoring(sceneFeatureMap, userFeatureMap, norUserFeatureMap, rankItems);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250303.conf").scoring(sceneFeatureMap, userFeatureMap, norUserFeatureMap, rankItems);
         // 5 排序公式特征
         Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
         List<Video> result = new ArrayList<>();
@@ -360,15 +349,13 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
             item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
             double fmRov = restoreScore(fmRovOrigin);
             item.getScoresMap().put("fmRov", fmRov);
-            double newFmRov = logRov(fmRov, fmRovLogBase);
             double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
             double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
-            double newNorXGBScore = norCalibration(xgbNorScaleType, xgbNorBias, xgbNorWeight, xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
+            double newNorXGBScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
             double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
-            double newVor = logVor(vorLogWeight, vor, vorLogBase);
             item.getScoresMap().put("vor", vor);
-            score = newFmRov * (0.1 + newNorXGBScore) * (0.1 + newVor);
+            score = fmRov * (0.1 + newNorXGBScore) * (0.1 + vor);
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
@@ -392,143 +379,126 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         return result;
     }
 
-    private Map<String, String> getVideoOneTypeInfo(String vid, String name,
-                                                    Map<String, Map<String, Map<String, String>>> videoAllInfoMap) {
-        if (null == videoAllInfoMap) {
-            return new HashMap<>();
-        }
-        return videoAllInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault(name, new HashMap<>());
-    }
-
-    private double getVideoOneInfo(String name, Map<String, String> infoMap) {
-        if (null == infoMap) {
-            return 0.0;
-        }
-        return infoMap.isEmpty() ? 0 : Double.parseDouble(infoMap.getOrDefault(name, "0.0"));
-    }
-
-    private void addVideoStatFeature(String vid, Map<String, Map<String, Map<String, String>>> videoAllInfoMap,
-                                     Map<String, Double> featureMap) {
-        List<Tuple3> vidStatInfo = Arrays.asList(
-                new Tuple3("b20", shortPeriod, getVideoOneTypeInfo(vid, "alg_cate2_feature", videoAllInfoMap)),
-                new Tuple3("b21", shortPeriod, getVideoOneTypeInfo(vid, "alg_cate1_feature", videoAllInfoMap)),
-                new Tuple3("b22", shortPeriod, getVideoOneTypeInfo(vid, "alg_vid_source_feature", videoAllInfoMap)),
-                new Tuple3("b28", shortPeriod, getVideoOneTypeInfo(vid, "alg_sence_type_feature", videoAllInfoMap)),
-                new Tuple3("b29", shortPeriod, getVideoOneTypeInfo(vid, "alg_videoid_feature", videoAllInfoMap)),
-                new Tuple3("b23", middlePeriod, getVideoOneTypeInfo(vid, "alg_cate2_feature_day", videoAllInfoMap)),
-                new Tuple3("b24", middlePeriod, getVideoOneTypeInfo(vid, "alg_cate1_feature_day", videoAllInfoMap)),
-                new Tuple3("b25", middlePeriod, getVideoOneTypeInfo(vid, "alg_video_source_feature_day", videoAllInfoMap)),
-                new Tuple3("b26", longPeriod, getVideoOneTypeInfo(vid, "alg_video_unionid_feature_day", videoAllInfoMap)),
-                new Tuple3("b27", longPeriod, getVideoOneTypeInfo(vid, "alg_vid_feature_day", videoAllInfoMap))
-        );
-        for (Tuple3 tuple3 : vidStatInfo) {
-            String infoType = tuple3.first;
-            List<String> infoPeriod = tuple3.second;
-            Map<String, String> infoMap = tuple3.third;
-            for (String period : infoPeriod) {
-                double view = getVideoOneInfo("view_" + period, infoMap);
-                double share = getVideoOneInfo("share_" + period, infoMap);
-                double return_ = getVideoOneInfo("return_" + period, infoMap);
-                double view_hasreturn = getVideoOneInfo("view_hasreturn_" + period, infoMap);
-                double share_hasreturn = getVideoOneInfo("share_hasreturn_" + period, infoMap);
-                double ros = ExtractorUtils.smoothDiv(return_, share, smoothPlus);
-                double rov = ExtractorUtils.smoothDiv(return_, view, smoothPlus);
-                double r_cnt = ExtractorUtils.smoothDiv(return_, view_hasreturn, smoothPlus);
-                double r_rate = ExtractorUtils.smoothDiv(view_hasreturn, view, smoothPlus);
-                double r_cnt4s = ExtractorUtils.smoothDiv(return_, share_hasreturn, smoothPlus);
-                double str = ExtractorUtils.smoothDiv(share, view, smoothPlus);
-
-                featureMap.put(infoType + "_" + period + "_" + "view", ExtractorUtils.calLog(view));
-                featureMap.put(infoType + "_" + period + "_" + "share", ExtractorUtils.calLog(share));
-                featureMap.put(infoType + "_" + period + "_" + "return", ExtractorUtils.calLog(return_));
-                featureMap.put(infoType + "_" + period + "_" + "view_hasreturn", ExtractorUtils.calLog(view_hasreturn));
-                featureMap.put(infoType + "_" + period + "_" + "share_hasreturn", ExtractorUtils.calLog(share_hasreturn));
-                featureMap.put(infoType + "_" + period + "_" + "ros", ros);
-                featureMap.put(infoType + "_" + period + "_" + "rov", rov);
-                featureMap.put(infoType + "_" + period + "_" + "r_cnt", r_cnt);
-                featureMap.put(infoType + "_" + period + "_" + "r_rate", r_rate);
-                featureMap.put(infoType + "_" + period + "_" + "r_cnt4s", r_cnt4s);
-                featureMap.put(infoType + "_" + period + "_" + "str", str);
+    private UserShareReturnProfile parseUserProfile(Map<String, Map<String, String>> userOriginInfo) {
+        if (null != userOriginInfo) {
+            Map<String, String> c9 = userOriginInfo.get("alg_recsys_feature_user_share_return_stat");
+            if (null != c9 && !c9.isEmpty()) {
+                String c9Str = JSONUtils.toJson(c9);
+                if (!c9Str.isEmpty()) {
+                    try {
+                        return JSON.parseObject(c9Str, UserShareReturnProfile.class);
+                    } catch (Exception e) {
+                        log.error("parseObject user profile error! value=[{}]", c9Str, e);
+                    }
+                }
             }
         }
+        return null;
     }
 
-    private void addVideoCFFeature(String vid, Map<String, Map<String, Map<String, String>>> videoAllInfoMap,
-                                   Map<String, Double> featureMap) {
-        List<Tuple3> vidCFInfo = Arrays.asList(
-                new Tuple3("d2", cfRosList, getVideoOneTypeInfo(vid, "alg_recsys_feature_weak_cf_i2i_scene_ros", videoAllInfoMap)),
-                new Tuple3("d3", cfRosList, getVideoOneTypeInfo(vid, "alg_recsys_feature_cf_i2i_scene_ros", videoAllInfoMap)),
-                new Tuple3("d4", cfRovList, getVideoOneTypeInfo(vid, "alg_recsys_feature_weak_cf_i2i_scene_rov", videoAllInfoMap)),
-                new Tuple3("d5", cfRovList, getVideoOneTypeInfo(vid, "alg_recsys_feature_cf_i2i_scene_rov", videoAllInfoMap))
-        );
-        for (Tuple3 tuple3 : vidCFInfo) {
-            String infoType = tuple3.first;
-            List<String> valTypeList = tuple3.second;
-            Map<String, String> infoMap = tuple3.third;
-            if (!infoMap.isEmpty()) {
-                for (String valType : valTypeList) {
-                    double exp = getVideoOneInfo("exp", infoMap);
-                    double return_n = getVideoOneInfo("return_n", infoMap);
-                    double value = getVideoOneInfo(valType, infoMap);
-
-                    featureMap.put(infoType + "_exp", ExtractorUtils.calLog(exp));
-                    featureMap.put(infoType + "_return_n", ExtractorUtils.calLog(return_n));
-                    featureMap.put(infoType + "_" + valType, value);
+    private Map<String, Map<String, String>> getUserBehaviorVideoMap(UserShareReturnProfile userProfile) {
+        Set<String> vidSet = new HashSet<>();
+        if (null != userProfile) {
+            for (List<UserSRBO> list : Arrays.asList(userProfile.getM_s_s(), userProfile.getM_r_s(), userProfile.getL_s_s(), userProfile.getL_r_s())) {
+                if (null != list) {
+                    for (UserSRBO u : list) {
+                        if (null != u) {
+                            vidSet.add(u.getId() + "");
+                        }
+                    }
                 }
             }
         }
-    }
 
-    private void addVideoSimFeature(Map<String, String> headInfo, Map<String, String> rankInfo, Map<String, Double> featureMap) {
-        if (!headInfo.isEmpty() && !rankInfo.isEmpty()) {
-            List<Future<Pair<String, Double>>> futures = new ArrayList<>();
-            for (String attr : videoSimAttrs) {
-                String headAttr = headInfo.getOrDefault(attr, "");
-                String rankAttr = rankInfo.getOrDefault(attr, "");
-                if (!"".equals(headAttr) && !"".equals(rankAttr)) {
-                    String key = "video_sim_" + attr;
-                    Future<Pair<String, Double>> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                        double simScore = SimilarityUtils.word2VecSimilarity(headAttr, rankAttr);
-                        return Pair.create(key, simScore);
-                    });
-                    futures.add(future);
-                }
-            }
-            try {
-                for (Future<Pair<String, Double>> future : futures) {
-                    Pair<String, Double> pair = future.get(1000, TimeUnit.MILLISECONDS);
-                    featureMap.put(pair.getFirst(), pair.getSecond());
+        Map<String, Map<String, String>> historyVideoMap = new HashMap<>();
+        if (!vidSet.isEmpty()) {
+            Map<String, Map<String, Map<String, String>>> videoMap = featureService.getVideoBaseInfo("", new ArrayList<>(vidSet));
+            if (null != videoMap && !videoMap.isEmpty()) {
+                for (Map.Entry<String, Map<String, Map<String, String>>> entry : videoMap.entrySet()) {
+                    String vid = entry.getKey();
+                    Map<String, Map<String, String>> map = entry.getValue();
+                    if (null != map && map.containsKey("alg_vid_feature_basic_info")) {
+                        historyVideoMap.put(vid, map.get("alg_vid_feature_basic_info"));
+                    }
                 }
-            } catch (Exception e) {
-                log.error("video attr similarity error", e);
             }
         }
+        return historyVideoMap;
     }
 
-    private void addTimeFeature(Map<String, Double> featureMap) {
-        Calendar calendar = Calendar.getInstance();
-        calendar.setTimeInMillis(System.currentTimeMillis());
+    private Map<String, String> getNorUserFeature(long currentMs, Map<String, String> headInfo, UserShareReturnProfile userProfile, Map<String, Map<String, String>> userOriginInfo) {
+        Map<String, Double> featMap = new HashMap<>();
+        // context feature
+        NORFeature.getContextFeature(currentMs, featMap);
+
+        // head video feature
+        NORFeature.getVideoBaseFeature("h", currentMs, headInfo, featMap);
 
-        int week = calendar.get(Calendar.DAY_OF_WEEK);
-        int hour = calendar.get(Calendar.HOUR_OF_DAY) + 1;
-        featureMap.put("week", week * 1.0);
-        featureMap.put("hour", hour * 1.0);
+        // user feature
+        NORFeature.getUserFeature(userOriginInfo, featMap);
+        NORFeature.getUserProfileFeature(userProfile, featMap);
+
+        return FeatureBucketUtils.noBucketFeature(featMap);
     }
 
-    private double norCalibration(double scaleType, double polyBias, double polyWeight, double powerWeight, double powerExp, double score) {
-        if (scaleType < 1) {
-            return norPolyCalibration(polyBias, polyWeight, score);
-        } else {
-            return norPowerCalibration(powerWeight, powerExp, score);
-        }
+    private Map<String, String> getNorVideoFeature(long currentMs, String vid,
+                                                   UserShareReturnProfile userProfile,
+                                                   Map<String, String> headInfo, Map<String, String> rankInfo,
+                                                   Map<String, Map<String, String[]>> c7Map,
+                                                   Map<String, Map<String, String[]>> c8Map,
+                                                   Map<String, Map<String, String>> userOriginInfo,
+                                                   Map<String, Map<String, String>> historyVideoMap,
+                                                   Map<String, Map<String, Map<String, String>>> videoOriginInfo) {
+        Map<String, Double> featMap = new HashMap<>();
+        // user & video feature
+        NORFeature.getUserTagsCrossVideoFeature("c5", rankInfo, userOriginInfo.get("alg_mid_feature_return_tags"), featMap);
+        NORFeature.getUserTagsCrossVideoFeature("c6", rankInfo, userOriginInfo.get("alg_mid_feature_share_tags"), featMap);
+        NORFeature.getUserCFFeature("c7", vid, c7Map, featMap);
+        NORFeature.getUserCFFeature("c8", vid, c8Map, featMap);
+
+        // rank video feature
+        NORFeature.getVideoBaseFeature("r", currentMs, rankInfo, featMap);
+        NORFeature.getVideoFeature(vid, videoOriginInfo, featMap);
+
+        // head&rank cross feature
+        NORFeature.getHeadRankVideoCrossFeature(headInfo, rankInfo, featMap);
+
+        // user profile & rank cross
+        NORFeature.getProfileVideoCrossFeature(currentMs, userProfile, rankInfo, historyVideoMap, featMap);
+
+        return FeatureBucketUtils.noBucketFeature(featMap);
     }
 
-    private double norPolyCalibration(double bias, double weight, double score) {
-        double newScore = bias + weight * score;
-        if (newScore < score) {
-            newScore = score;
+    private void batchGetNorVideoFeature(long currentMs,
+                                         UserShareReturnProfile userProfile,
+                                         Map<String, String> headInfo,
+                                         Map<String, Map<String, Map<String, String>>> videoBaseInfoMap,
+                                         Map<String, Map<String, String[]>> c7Map,
+                                         Map<String, Map<String, String[]>> c8Map,
+                                         Map<String, Map<String, String>> userOriginInfo,
+                                         Map<String, Map<String, String>> historyVideoMap,
+                                         Map<String, Map<String, Map<String, String>>> videoOriginInfo,
+                                         List<RankItem> rankItems) {
+        if (null != rankItems && !rankItems.isEmpty()) {
+            List<Future<Integer>> futures = new ArrayList<>();
+            for (RankItem item : rankItems) {
+                String vid = item.getVideoId() + "";
+                Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+                Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
+                    item.norFeatureMap = getNorVideoFeature(currentMs, vid, userProfile, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
+                    return 1;
+                });
+                futures.add(future);
+            }
+
+            try {
+                for (Future<Integer> future : futures) {
+                    future.get(1000, TimeUnit.MILLISECONDS);
+                }
+            } catch (Exception e) {
+                log.error("get nor feature error", e);
+            }
         }
-        return newScore;
     }
 
     private double norPowerCalibration(double weight, double exp, double score) {
@@ -541,19 +511,13 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         return newScore;
     }
 
-    private double logRov(double x, double base) {
-        if (base > 1) {
-            return Math.log(1 + x) / Math.log(base);
-        } else {
-            return x;
-        }
-    }
-
-    private double logVor(double weight, double x, double base) {
-        if (base > 1) {
-            return weight * Math.log(1 + x / 10) / Math.log(base);
-        } else {
-            return x;
+    private void addRecall(RankParam param, int recallNum, String recallName, Set<Long> setVideo, List<Video> rovRecallRank) {
+        if (recallNum > 0) {
+            List<Video> list = extractAndSort(param, recallName);
+            list = list.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+            list = list.subList(0, Math.min(recallNum, list.size()));
+            rovRecallRank.addAll(list);
+            setVideo.addAll(list.stream().map(Video::getVideoId).collect(Collectors.toSet()));
         }
     }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/tansform/NORFeature.java

@@ -154,7 +154,7 @@ public class NORFeature {
         featMap.put(prefix + "@bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
 
         // cate
-        // getVideoCateFeature(prefix, videoInfo, featMap);
+        getVideoCateFeature(prefix, videoInfo, featMap);
         if (videoInfo.containsKey("title")) {
             int id = FeatureUtils.judgeVideoTimeType(videoInfo.get("title"));
             if (id > 0) {
@@ -233,6 +233,7 @@ public class NORFeature {
         if (null == attrMap || attrMap.isEmpty()) {
             return;
         }
+        attr = attr.trim();
         if (attrMap.containsKey(attr)) {
             VideoAttrSRBO bo = attrMap.get(attr);
             if (null != bo) {
@@ -268,6 +269,7 @@ public class NORFeature {
         }
         for (String attr : videoCateAttrs) {
             String attrVal = videoInfo.getOrDefault(attr, "");
+            attrVal = attrVal.trim();
             int attrId = FeatureUtils.getAttrId(attr, attrVal);
             if (attrId > 0) {
                 String key = String.format("%s@%s@%d", prefix, attr, attrId);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -114,6 +114,10 @@ public class RecallService implements ApplicationContextAware {
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV7LongTermV1.class.getSimpleName()));
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyROS.class.getSimpleName()));
         }
+        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("567")) {
+            strategies.add(strategyMap.get(UserCate1RecallStrategy.class.getSimpleName()));
+            strategies.add(strategyMap.get(UserCate2RecallStrategy.class.getSimpleName()));
+        }
 
         // 命中用户黑名单不走流量池
         if (!param.isRiskUser()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/UserCate1RecallStrategy.java

@@ -0,0 +1,148 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
+import com.tzld.piaoquan.recommend.server.service.rank.bo.VideoAttrSRBO;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.math3.util.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class UserCate1RecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private FeatureService featureService;
+
+    public static final int topN = 2;
+    public static final String PUSH_FORM = "recall_strategy_user_cate1";
+    public static final String redisKeyPrefix = "merge_cate_recall:cate1";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            UserShareReturnProfile userProfile = featureService.getUserProfile(param.getMid());
+            if (null != userProfile && null != userProfile.getC1_s()) {
+                List<String> keys = getRedisKey(userProfile.getC1_s());
+                if (!keys.isEmpty()) {
+                    List<String> values = redisTemplate.opsForValue().multiGet(keys);
+                    List<Long> ids = recall(param.getVideoId(), values);
+                    FilterParam filterParam = FilterParamFactory.create(param, ids);
+                    FilterResult filterResult = filterService.filter(filterParam);
+                    if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                        List<Long> filterIds = filterResult.getVideoIds();
+                        int n = filterIds.size();
+                        for (int i = 0; i < n; i++) {
+                            Video video = new Video();
+                            video.setVideoId(filterIds.get(i));
+                            video.setRovScore(n - i);
+                            video.setPushFrom(pushFrom());
+                            videosResult.add(video);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private List<Long> recall(Long headVid, List<String> values) {
+        List<Long> vidList = new ArrayList<>();
+        if (null != values && !values.isEmpty()) {
+            Set<Long> hits = new HashSet<>();
+            hits.add(headVid);
+            List<Pair<Long, Double>> list = new ArrayList<>();
+            for (String value : values) {
+                if (null != value && !value.isEmpty()) {
+                    String[] cells = value.split("\t");
+                    if (2 == cells.length) {
+                        List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                        List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                        if (!ids.isEmpty() && ids.size() == scores.size()) {
+                            for (int i = 0; i < ids.size(); ++i) {
+                                long id = ids.get(i);
+                                double score = scores.get(i);
+                                if (hits.contains(id)) {
+                                    continue;
+                                }
+                                hits.add(id);
+                                list.add(Pair.create(id, score));
+                            }
+                        }
+                    }
+                }
+            }
+            if (!list.isEmpty()) {
+                list.sort(Comparator.comparingDouble(o -> -o.getSecond()));
+                for (Pair<Long, Double> pair : list) {
+                    vidList.add(pair.getFirst());
+                }
+            }
+        }
+        return vidList;
+    }
+
+    private List<String> getRedisKey(Map<String, VideoAttrSRBO> map) {
+        List<String> keys = new ArrayList<>();
+        if (null != map) {
+            List<Pair<String, Double>> cateList = getOrderedList(map);
+            for (Pair<String, Double> pair : cateList) {
+                keys.add(String.format("%s:%s", redisKeyPrefix, pair.getFirst()));
+                if (keys.size() >= topN) {
+                    break;
+                }
+            }
+        }
+        return keys;
+    }
+
+    private List<Pair<String, Double>> getOrderedList(Map<String, VideoAttrSRBO> map) {
+        List<Pair<String, Double>> pairList = new ArrayList<>();
+        if (null != map) {
+            for (Map.Entry<String, VideoAttrSRBO> entry : map.entrySet()) {
+                String name = entry.getKey();
+                VideoAttrSRBO videoAttrSRBO = entry.getValue();
+                if (null != videoAttrSRBO && videoAttrSRBO.getRu() > 0) {
+                    long sharePv = videoAttrSRBO.getSp();
+                    long returnUv = videoAttrSRBO.getRu();
+                    double score = FeatureUtils.plusSmooth(returnUv, sharePv, 5);
+                    Pair<String, Double> pair = Pair.create(name, score);
+                    pairList.add(pair);
+                }
+            }
+            if (pairList.size() > 1) {
+                pairList.sort(Comparator.comparingDouble(o -> -o.getSecond()));
+            }
+        }
+        return pairList;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/UserCate2RecallStrategy.java

@@ -0,0 +1,148 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.rank.bo.UserShareReturnProfile;
+import com.tzld.piaoquan.recommend.server.service.rank.bo.VideoAttrSRBO;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.math3.util.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class UserCate2RecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private FeatureService featureService;
+
+    public static final int topN = 2;
+    public static final String PUSH_FORM = "recall_strategy_user_cate2";
+    public static final String redisKeyPrefix = "merge_cate_recall:cate2";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            UserShareReturnProfile userProfile = featureService.getUserProfile(param.getMid());
+            if (null != userProfile && null != userProfile.getC2_s()) {
+                List<String> keys = getRedisKey(userProfile.getC2_s());
+                if (!keys.isEmpty()) {
+                    List<String> values = redisTemplate.opsForValue().multiGet(keys);
+                    List<Long> ids = recall(param.getVideoId(), values);
+                    FilterParam filterParam = FilterParamFactory.create(param, ids);
+                    FilterResult filterResult = filterService.filter(filterParam);
+                    if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                        List<Long> filterIds = filterResult.getVideoIds();
+                        int n = filterIds.size();
+                        for (int i = 0; i < n; i++) {
+                            Video video = new Video();
+                            video.setVideoId(filterIds.get(i));
+                            video.setRovScore(n - i);
+                            video.setPushFrom(pushFrom());
+                            videosResult.add(video);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private List<Long> recall(Long headVid, List<String> values) {
+        List<Long> vidList = new ArrayList<>();
+        if (null != values && !values.isEmpty()) {
+            Set<Long> hits = new HashSet<>();
+            hits.add(headVid);
+            List<Pair<Long, Double>> list = new ArrayList<>();
+            for (String value : values) {
+                if (null != value && !value.isEmpty()) {
+                    String[] cells = value.split("\t");
+                    if (2 == cells.length) {
+                        List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                        List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                        if (!ids.isEmpty() && ids.size() == scores.size()) {
+                            for (int i = 0; i < ids.size(); ++i) {
+                                long id = ids.get(i);
+                                double score = scores.get(i);
+                                if (hits.contains(id)) {
+                                    continue;
+                                }
+                                hits.add(id);
+                                list.add(Pair.create(id, score));
+                            }
+                        }
+                    }
+                }
+            }
+            if (!list.isEmpty()) {
+                list.sort(Comparator.comparingDouble(o -> -o.getSecond()));
+                for (Pair<Long, Double> pair : list) {
+                    vidList.add(pair.getFirst());
+                }
+            }
+        }
+        return vidList;
+    }
+
+    private List<String> getRedisKey(Map<String, VideoAttrSRBO> map) {
+        List<String> keys = new ArrayList<>();
+        if (null != map) {
+            List<Pair<String, Double>> cateList = getOrderedList(map);
+            for (Pair<String, Double> pair : cateList) {
+                keys.add(String.format("%s:%s", redisKeyPrefix, pair.getFirst()));
+                if (keys.size() >= topN) {
+                    break;
+                }
+            }
+        }
+        return keys;
+    }
+
+    private List<Pair<String, Double>> getOrderedList(Map<String, VideoAttrSRBO> map) {
+        List<Pair<String, Double>> pairList = new ArrayList<>();
+        if (null != map) {
+            for (Map.Entry<String, VideoAttrSRBO> entry : map.entrySet()) {
+                String name = entry.getKey();
+                VideoAttrSRBO videoAttrSRBO = entry.getValue();
+                if (null != videoAttrSRBO && videoAttrSRBO.getRu() > 0) {
+                    long sharePv = videoAttrSRBO.getSp();
+                    long returnUv = videoAttrSRBO.getRu();
+                    double score = FeatureUtils.plusSmooth(returnUv, sharePv, 5);
+                    Pair<String, Double> pair = Pair.create(name, score);
+                    pairList.add(pair);
+                }
+            }
+            if (pairList.size() > 1) {
+                pairList.sort(Comparator.comparingDouble(o -> -o.getSecond()));
+            }
+        }
+        return pairList;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -33,9 +33,8 @@ public final class ScorerUtils {
         ScorerUtils.init(VIDEO_SCORE_CONF_FOR_AD);
         ScorerUtils.init("feeds_score_config_20240609.conf");
         ScorerUtils.init("feeds_score_config_20240807.conf");
-        ScorerUtils.init("feeds_score_config_fm_xgb_20241209.conf");
-        ScorerUtils.init("feeds_score_config_fm_xgb_20250208.conf");
         ScorerUtils.init("feeds_score_config_fm_xgb_20250221.conf");
+        ScorerUtils.init("feeds_score_config_fm_xgb_20250303.conf");
         ScorerUtils.init("feeds_score_config_xgb_20241209.conf");
         ScorerUtils.init("feeds_score_config_xgb_20250109.conf");
         ScorerUtils.init("feeds_score_config_xgb_rov_20241209.conf");

recommend-server-service/src/main/resources/feeds_score_config_fm_xgb_20250303.conf

@@ -0,0 +1,462 @@
+scorer-config = {
+  rov-score-config = {
+     scorer-name = "com.tzld.piaoquan.recommend.server.service.score.VlogRovFMScorer"
+     scorer-priority = 96
+     model-path = "zhangbo/model_aka8_new2.txt"
+  }
+  nor-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score.NorXGBRegressionScorer"
+    scorer-priority = 97
+    model-path = "zhangbo/model_xgb_for_recsys_v5_nor.tar.gz"
+    param = {
+      localDir = "xgboost/recsys_v5_nor"
+      features = [
+      "b10_12h_is_share",
+      "b10_12h_return_n_uv",
+      "b10_12h_ros",
+      "b10_12h_ros_minus",
+      "b10_12h_rovn",
+      "b10_12h_str",
+      "b10_12h_str_plus",
+      "b10_1h_is_share",
+      "b10_1h_return_n_uv",
+      "b10_1h_ros",
+      "b10_1h_ros_minus",
+      "b10_1h_rovn",
+      "b10_1h_str",
+      "b10_1h_str_plus",
+      "b11_12h_is_share",
+      "b11_12h_return_n_uv",
+      "b11_12h_ros",
+      "b11_12h_ros_minus",
+      "b11_12h_rovn",
+      "b11_12h_str",
+      "b11_12h_str_plus",
+      "b11_168h_is_share",
+      "b11_168h_return_n_uv",
+      "b11_168h_ros",
+      "b11_168h_ros_minus",
+      "b11_168h_rovn",
+      "b11_168h_str",
+      "b11_168h_str_plus",
+      "b13_168h_is_share",
+      "b13_168h_return_n_uv",
+      "b13_168h_ros",
+      "b13_168h_ros_minus",
+      "b13_168h_ros_n",
+      "b13_168h_ros_one",
+      "b13_168h_rovn",
+      "b13_168h_str",
+      "b13_168h_str_plus",
+      "b13_24h_is_share",
+      "b13_24h_return_n_uv",
+      "b13_24h_ros",
+      "b13_24h_ros_minus",
+      "b13_24h_ros_n",
+      "b13_24h_ros_one",
+      "b13_24h_rovn",
+      "b13_24h_str",
+      "b13_24h_str_plus",
+      "b1_168h_exp",
+      "b1_168h_is_return_1",
+      "b1_168h_is_share",
+      "b1_168h_return_n_uv",
+      "b1_168h_ros",
+      "b1_168h_ros_minus",
+      "b1_168h_ros_n",
+      "b1_168h_ros_one",
+      "b1_168h_rovn",
+      "b1_168h_share_cnt",
+      "b1_168h_str",
+      "b1_168h_str_plus",
+      "b1_1h_exp",
+      "b1_1h_is_return_1",
+      "b1_1h_is_share",
+      "b1_1h_return_n_uv",
+      "b1_1h_ros",
+      "b1_1h_ros_minus",
+      "b1_1h_ros_n",
+      "b1_1h_ros_one",
+      "b1_1h_rovn",
+      "b1_1h_share_cnt",
+      "b1_1h_str",
+      "b1_1h_str_plus",
+      "b1_24h_exp",
+      "b1_24h_is_return_1",
+      "b1_24h_is_share",
+      "b1_24h_return_n_uv",
+      "b1_24h_ros",
+      "b1_24h_ros_minus",
+      "b1_24h_ros_n",
+      "b1_24h_ros_one",
+      "b1_24h_rovn",
+      "b1_24h_share_cnt",
+      "b1_24h_str",
+      "b1_24h_str_plus",
+      "b1_3h_exp",
+      "b1_3h_is_return_1",
+      "b1_3h_is_share",
+      "b1_3h_return_n_uv",
+      "b1_3h_ros",
+      "b1_3h_ros_minus",
+      "b1_3h_ros_n",
+      "b1_3h_ros_one",
+      "b1_3h_rovn",
+      "b1_3h_share_cnt",
+      "b1_3h_str",
+      "b1_3h_str_plus",
+      "b1_72h_exp",
+      "b1_72h_is_return_1",
+      "b1_72h_is_share",
+      "b1_72h_return_n_uv",
+      "b1_72h_ros",
+      "b1_72h_ros_minus",
+      "b1_72h_ros_n",
+      "b1_72h_ros_one",
+      "b1_72h_rovn",
+      "b1_72h_share_cnt",
+      "b1_72h_str",
+      "b1_72h_str_plus",
+      "b2_1h_is_return_1",
+      "b2_1h_is_share",
+      "b2_1h_return_n_uv",
+      "b2_1h_ros",
+      "b2_1h_ros_minus",
+      "b2_1h_ros_n",
+      "b2_1h_ros_one",
+      "b2_1h_rovn",
+      "b2_1h_share_cnt",
+      "b2_1h_str",
+      "b2_1h_str_plus",
+      "b2_24h_is_return_1",
+      "b2_24h_is_share",
+      "b2_24h_return_n_uv",
+      "b2_24h_ros",
+      "b2_24h_ros_minus",
+      "b2_24h_ros_n",
+      "b2_24h_ros_one",
+      "b2_24h_rovn",
+      "b2_24h_share_cnt",
+      "b2_24h_str",
+      "b2_24h_str_plus",
+      "b2_3h_is_return_1",
+      "b2_3h_is_share",
+      "b2_3h_return_n_uv",
+      "b2_3h_ros",
+      "b2_3h_ros_minus",
+      "b2_3h_ros_n",
+      "b2_3h_ros_one",
+      "b2_3h_rovn",
+      "b2_3h_share_cnt",
+      "b2_3h_str",
+      "b2_3h_str_plus",
+      "b3_168h_is_return_1",
+      "b3_168h_is_share",
+      "b3_168h_return_n_uv",
+      "b3_168h_ros",
+      "b3_168h_ros_minus",
+      "b3_168h_ros_n",
+      "b3_168h_ros_one",
+      "b3_168h_rovn",
+      "b3_168h_share_cnt",
+      "b3_168h_str",
+      "b3_168h_str_plus",
+      "b3_24h_is_return_1",
+      "b3_24h_is_share",
+      "b3_24h_return_n_uv",
+      "b3_24h_ros",
+      "b3_24h_ros_minus",
+      "b3_24h_ros_n",
+      "b3_24h_ros_one",
+      "b3_24h_rovn",
+      "b3_24h_share_cnt",
+      "b3_24h_str",
+      "b3_24h_str_plus",
+      "b4_12h_is_return_1",
+      "b4_12h_is_share",
+      "b4_12h_return_n_uv",
+      "b4_12h_ros",
+      "b4_12h_ros_minus",
+      "b4_12h_ros_n",
+      "b4_12h_ros_one",
+      "b4_12h_rovn",
+      "b4_12h_share_cnt",
+      "b4_12h_str",
+      "b4_12h_str_plus",
+      "b4_1h_is_return_1",
+      "b4_1h_is_share",
+      "b4_1h_return_n_uv",
+      "b4_1h_ros",
+      "b4_1h_ros_minus",
+      "b4_1h_ros_n",
+      "b4_1h_ros_one",
+      "b4_1h_rovn",
+      "b4_1h_share_cnt",
+      "b4_1h_str",
+      "b4_1h_str_plus",
+      "b5_168h_is_share",
+      "b5_168h_return_n_uv",
+      "b5_168h_ros",
+      "b5_168h_ros_minus",
+      "b5_168h_ros_n",
+      "b5_168h_ros_one",
+      "b5_168h_rovn",
+      "b5_168h_str",
+      "b5_168h_str_plus",
+      "b5_72h_is_share",
+      "b5_72h_return_n_uv",
+      "b5_72h_ros",
+      "b5_72h_ros_minus",
+      "b5_72h_ros_n",
+      "b5_72h_ros_one",
+      "b5_72h_rovn",
+      "b5_72h_str",
+      "b5_72h_str_plus",
+      "b6_1h_is_share",
+      "b6_1h_return_n_uv",
+      "b6_1h_ros",
+      "b6_1h_ros_minus",
+      "b6_1h_ros_n",
+      "b6_1h_ros_one",
+      "b6_1h_rovn",
+      "b6_1h_str",
+      "b6_1h_str_plus",
+      "b6_24h_is_share",
+      "b6_24h_return_n_uv",
+      "b6_24h_ros",
+      "b6_24h_ros_minus",
+      "b6_24h_ros_n",
+      "b6_24h_ros_one",
+      "b6_24h_rovn",
+      "b6_24h_str",
+      "b6_24h_str_plus",
+      "b7_168h_is_share",
+      "b7_168h_return_n_uv",
+      "b7_168h_ros",
+      "b7_168h_ros_minus",
+      "b7_168h_rovn",
+      "b7_168h_str",
+      "b7_168h_str_plus",
+      "b7_24h_is_share",
+      "b7_24h_return_n_uv",
+      "b7_24h_ros",
+      "b7_24h_ros_minus",
+      "b7_24h_rovn",
+      "b7_24h_str",
+      "b7_24h_str_plus",
+      "b8_24h_is_share",
+      "b8_24h_return_n_uv",
+      "b8_24h_ros",
+      "b8_24h_ros_minus",
+      "b8_24h_rovn",
+      "b8_24h_str",
+      "b8_24h_str_plus",
+      "b9_24h_is_share",
+      "b9_24h_return_n_uv",
+      "b9_24h_ros",
+      "b9_24h_ros_minus",
+      "b9_24h_rovn",
+      "b9_24h_str",
+      "b9_24h_str_plus",
+      "c1_168h_is_return_1",
+      "c1_168h_is_share",
+      "c1_168h_return_n_uv",
+      "c1_168h_ros",
+      "c1_168h_ros_minus",
+      "c1_168h_ros_n",
+      "c1_168h_ros_one",
+      "c1_168h_rovn",
+      "c1_168h_share_cnt",
+      "c1_168h_str",
+      "c1_168h_str_plus",
+      "c1_72h_is_return_1",
+      "c1_72h_is_share",
+      "c1_72h_return_n_uv",
+      "c1_72h_ros",
+      "c1_72h_ros_minus",
+      "c1_72h_ros_n",
+      "c1_72h_ros_one",
+      "c1_72h_rovn",
+      "c1_72h_share_cnt",
+      "c1_72h_str",
+      "c1_72h_str_plus",
+      "c5_tags_1d_avgscore",
+      "c5_tags_1d_matchnum",
+      "c5_tags_1d_maxscore",
+      "c5_tags_3d_avgscore",
+      "c5_tags_3d_matchnum",
+      "c5_tags_3d_maxscore",
+      "c5_tags_7d_avgscore",
+      "c5_tags_7d_matchnum",
+      "c5_tags_7d_maxscore",
+      "c6_tags_1d_avgscore",
+      "c6_tags_1d_matchnum",
+      "c6_tags_1d_maxscore",
+      "c6_tags_3d_avgscore",
+      "c6_tags_3d_matchnum",
+      "c6_tags_3d_maxscore",
+      "c6_tags_7d_avgscore",
+      "c6_tags_7d_matchnum",
+      "c6_tags_7d_maxscore",
+      "c7_return_num",
+      "c7_return_rank",
+      "c7_return_score",
+      "c7_share_num",
+      "c7_share_rank",
+      "c7_share_score",
+      "c8_return_num",
+      "c8_return_rank",
+      "c8_return_score",
+      "c8_share_num",
+      "c8_share_rank",
+      "c8_share_score",
+      "c9_c1s@mu",
+      "c9_c1s@ros",
+      "c9_c1s@ros_minus",
+      "c9_c1s@ros_one",
+      "c9_c1s@rp",
+      "c9_c1s@ru",
+      "c9_c1s@sp",
+      "c9_c2s@mu",
+      "c9_c2s@ros",
+      "c9_c2s@ros_minus",
+      "c9_c2s@ros_one",
+      "c9_c2s@rp",
+      "c9_c2s@ru",
+      "c9_c2s@sp",
+      "c9_l1s@mu",
+      "c9_l1s@ros",
+      "c9_l1s@ros_minus",
+      "c9_l1s@ros_one",
+      "c9_l1s@rp",
+      "c9_l1s@ru",
+      "c9_l1s@sp",
+      "c9_l2s@mu",
+      "c9_l2s@ros",
+      "c9_l2s@ros_minus",
+      "c9_l2s@ros_one",
+      "c9_l2s@rp",
+      "c9_l2s@ru",
+      "c9_l2s@sp",
+      "c9_lrs@1@ts",
+      "c9_lrs@1@uv",
+      "c9_lrs@1_title",
+      "c9_lrs@2@ts",
+      "c9_lrs@2@uv",
+      "c9_lrs@2_title",
+      "c9_lss@1@cnt",
+      "c9_lss@1@ts",
+      "c9_lss@1_title",
+      "c9_lss@2@cnt",
+      "c9_lss@2@ts",
+      "c9_lss@2_title",
+      "c9_m_r_uv",
+      "c9_m_s_cnt",
+      "c9_mrs@1@ts",
+      "c9_mrs@1@uv",
+      "c9_mrs@1_title",
+      "c9_mrs@2@ts",
+      "c9_mrs@2@uv",
+      "c9_mrs@2_title",
+      "c9_mss@1@cnt",
+      "c9_mss@1@ts",
+      "c9_mss@1_title",
+      "c9_mss@2@cnt",
+      "c9_mss@2@ts",
+      "c9_mss@2_title",
+      "c9_r_pv",
+      "c9_r_uv",
+      "c9_ros",
+      "c9_ros_minus",
+      "c9_ros_one",
+      "c9_s_cnt",
+      "c9_s_pv",
+      "d1_ros_cf_rank",
+      "d1_ros_cf_score",
+      "d1_rov_cf_rank",
+      "d1_rov_cf_score",
+      "d2_onlines",
+      "d2_rank",
+      "d2_score",
+      "d3_exp",
+      "d3_return_n",
+      "d3_rovn",
+      "h@bit_rate",
+      "h@total_time",
+      "h@ts",
+      "h@tt@1",
+      "hour",
+      "hr_sim_cate1_list",
+      "hr_sim_cate2",
+      "hr_sim_cate2_list",
+      "hr_sim_keywords",
+      "hr_sim_title",
+      "hr_sim_topic",
+      "r@bit_rate",
+      "r@total_time",
+      "r@ts",
+      "r@tt@1",
+      "h@cate1_list@1",
+      "h@cate1_list@10",
+      "h@cate1_list@11",
+      "h@cate1_list@13",
+      "h@cate1_list@14",
+      "h@cate1_list@15",
+      "h@cate1_list@2",
+      "h@cate1_list@3",
+      "h@cate1_list@4",
+      "h@cate1_list@5",
+      "h@cate1_list@6",
+      "h@cate1_list@7",
+      "h@cate1_list@9",
+      "h@cate2@1",
+      "h@cate2@12",
+      "h@cate2@13",
+      "h@cate2@15",
+      "h@cate2@17",
+      "h@cate2@18",
+      "h@cate2@19",
+      "h@cate2@2",
+      "h@cate2@21",
+      "h@cate2@22",
+      "h@cate2@23",
+      "h@cate2@3",
+      "h@cate2@5",
+      "h@cate2@7",
+      "h@cate2@8",
+      "h@festive_label1@1",
+      "h@festive_label1@2",
+      "h@festive_label1@3",
+      "h@festive_label1@4",
+      "r@cate1_list@1",
+      "r@cate1_list@10",
+      "r@cate1_list@13",
+      "r@cate1_list@14",
+      "r@cate1_list@15",
+      "r@cate1_list@2",
+      "r@cate1_list@3",
+      "r@cate1_list@5",
+      "r@cate1_list@6",
+      "r@cate1_list@7",
+      "r@cate1_list@9",
+      "r@cate2@1",
+      "r@cate2@10",
+      "r@cate2@12",
+      "r@cate2@13",
+      "r@cate2@14",
+      "r@cate2@15",
+      "r@cate2@17",
+      "r@cate2@19",
+      "r@cate2@2",
+      "r@cate2@22",
+      "r@cate2@23",
+      "r@cate2@26",
+      "r@cate2@3",
+      "r@cate2@7",
+      "r@cate2@9",
+      "r@festive_label1@1",
+      "r@festive_label1@3"
+      ]
+    }
+  }
+}