特征落盘

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -17,6 +17,8 @@ public class RankItem implements Comparable<RankItem> {
 
     // featureMap中保存所有的特征
     public Map<String, String> featureMap = new HashMap<>();
+    // 所有特征，包括视频、用户等等
+    public Map<String, String> allFeatureMap = new HashMap<>();
     public Map<String, Double> featureMapDouble = new HashMap<>();
     public String id;
     public Map<String, Double> scoresMap = new HashMap<>();
@@ -53,10 +55,10 @@ public class RankItem implements Comparable<RankItem> {
     // 排序侧信息
     private Map<String, Double> rankerScore = Maps.newHashMap();
     private Map<String, Integer> rankerIndex = Maps.newHashMap();
-    public RankItem(){
 
-    }
+    public RankItem() {
 
+    }
 
 
     public RankItem(Video video) {
@@ -148,7 +150,4 @@ public class RankItem implements Comparable<RankItem> {
     }
 
 
-
-
-
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/Video.java

@@ -39,5 +39,9 @@ public class Video {
     public Map<String, Double> scoresMap = new HashMap<>();
     public Map<String, List<String>> pushFromIndex = new HashMap<>();
     public Map<String, Integer> pushFromRank = new HashMap<>();
+    // 处理后，传给模型的特征
+    public Map<String, String> allFeatureMap = new HashMap<>();
+    // 原始特征
+    public Map<String, Map<String, String>> metaFeatureMap = new HashMap<>();
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -202,6 +202,8 @@ public class RecommendService {
                 map.put("scoreStr", String.valueOf(v.getScoreStr()));
                 map.put("score", String.valueOf(v.getScore()));
                 map.put("scoresMap", JSONUtils.toJson(v.getScoresMap()));
+                map.put("allFeatureMap", JSONUtils.toJson(v.getAllFeatureMap()));
+                map.put("metaFeatureMap", JSONUtils.toJson(v.getMetaFeatureMap()));
 
                 map.put("pushFromRank", JSONUtils.toJson(v.getPushFromRank()));
                 map.put("abExpCode", JSONUtils.toJson(param.getAbExpCodes()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV999.java

@@ -10,14 +10,15 @@ import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.*;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.*;
 
 @Service
 @Slf4j
@@ -92,8 +93,6 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
         Map<String, Map<String, Map<String, String>>> featureOriginVideo = feature.getVideoFeature();
 
 
-
-
         // TODO 2 特征处理
         Map<String, Double> userFeatureMapDouble = new HashMap<>();
         String mid = param.getMid();
@@ -107,13 +106,13 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
         Map<String, String> c8 = featureOriginUser.getOrDefault("alg_mid_feature_sharecf", new HashMap<>());
         Map<String, String> c9 = featureOriginUser.getOrDefault("alg_mid_feature_returncf", new HashMap<>());
 
-        if (!c1.isEmpty()){
+        if (!c1.isEmpty()) {
             userFeatureMapDouble.put("playcnt_6h", Double.parseDouble(c1.getOrDefault("playcnt_6h", "0")));
             userFeatureMapDouble.put("playcnt_1d", Double.parseDouble(c1.getOrDefault("playcnt_1d", "0")));
             userFeatureMapDouble.put("playcnt_3d", Double.parseDouble(c1.getOrDefault("playcnt_3d", "0")));
             userFeatureMapDouble.put("playcnt_7d", Double.parseDouble(c1.getOrDefault("playcnt_7d", "0")));
         }
-        if (!c2.isEmpty()){
+        if (!c2.isEmpty()) {
             userFeatureMapDouble.put("share_pv_12h", Double.parseDouble(c2.getOrDefault("share_pv_12h", "0")));
             userFeatureMapDouble.put("share_pv_1d", Double.parseDouble(c2.getOrDefault("share_pv_1d", "0")));
             userFeatureMapDouble.put("share_pv_3d", Double.parseDouble(c2.getOrDefault("share_pv_3d", "0")));
@@ -132,25 +131,25 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
                 new Tuple2(c6, "c6_feature"),
                 new Tuple2(c7, "c7_feature")
         );
-        for (Tuple2 tuple2: tmpList0){
-            for (String key_time: Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+        for (Tuple2 tuple2 : tmpList0) {
+            for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
                 String tags = tuple2.first.getOrDefault(key_time, "");
-                if (!tags.isEmpty()){
+                if (!tags.isEmpty()) {
                     c34567Map.put(tuple2.name + "_" + key_time, tags);
                 }
             }
         }
 
-        Map<String, Map<String, String []>> c89Map = new HashMap<>(4);
+        Map<String, Map<String, String[]>> c89Map = new HashMap<>(4);
         List<Tuple2> tmpList1 = Arrays.asList(
                 new Tuple2(c8, "c8_feature"),
                 new Tuple2(c9, "c9_feature")
         );
-        for (Tuple2 tuple2: tmpList1){
-            for (String key_action: Arrays.asList("share", "return")){
+        for (Tuple2 tuple2 : tmpList1) {
+            for (String key_action : Arrays.asList("share", "return")) {
                 String cfListStr = tuple2.first.getOrDefault(key_action, "");
-                if (!cfListStr.isEmpty()){
-                    Map<String, String []> cfMap = new HashMap<>();
+                if (!cfListStr.isEmpty()) {
+                    Map<String, String[]> cfMap = new HashMap<>();
                     String[] entries = cfListStr.split(",");
                     for (String entry : entries) {
                         String[] rList = entry.split(":");
@@ -159,7 +158,7 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
                             String value1 = rList[1];
                             String value2 = rList[2];
                             String value3 = rList[3];
-                            String [] strs = {value1, value2, value3};
+                            String[] strs = {value1, value2, value3};
                             cfMap.put(key, strs);
                         }
                     }
@@ -199,9 +198,9 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
 
             for (Tuple4 tuple4 : originData) {
                 for (String prefix2 : Arrays.asList("1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d")) {
-                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault( "exp_pv_" + prefix2, "0.0"));
-                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault( "share_pv_" + prefix2, "0.0"));
-                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault( "return_uv_" + prefix2, "0.0"));
+                    double exp = tuple4.first.isEmpty() ? 0 : Double.parseDouble(tuple4.first.getOrDefault("exp_pv_" + prefix2, "0.0"));
+                    double share = tuple4.second.isEmpty() ? 0 : Double.parseDouble(tuple4.second.getOrDefault("share_pv_" + prefix2, "0.0"));
+                    double returns = tuple4.third.isEmpty() ? 0 : Double.parseDouble(tuple4.third.getOrDefault("return_uv_" + prefix2, "0.0"));
 
                     double f1 = ExtractorUtils.calDiv(share, exp);
                     double f2 = ExtractorUtils.calLog(share);
@@ -228,11 +227,11 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
             featureMap.put("bit_rate", Double.parseDouble(videoInfo.getOrDefault("bit_rate", "0")));
 
             String title = videoInfo.getOrDefault("title", "");
-            if (!title.isEmpty()){
-                for (String name: Arrays.asList("c3_feature","c4_feature","c5_feature","c6_feature","c7_feature")){
-                    for (String key_time: Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
+            if (!title.isEmpty()) {
+                for (String name : Arrays.asList("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
+                    for (String key_time : Arrays.asList("tags_1d", "tags_3d", "tags_7d")) {
                         String tags = c34567Map.getOrDefault(name + "_" + key_time, "");
-                        if (!tags.isEmpty()){
+                        if (!tags.isEmpty()) {
                             Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
                             featureMap.put(name + "_" + key_time + "_matchnum", doubles[0]);
                             featureMap.put(name + "_" + key_time + "_maxscore", doubles[1]);
@@ -242,12 +241,12 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
                 }
             }
 
-            if (!vid.isEmpty()){
-                for (String key_feature: Arrays.asList("c8_feature", "c9_feature")){
-                    for (String key_action: Arrays.asList("share", "return")){
-                        Map<String, String []> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
-                        if (cfMap.containsKey(vid)){
-                            String [] scores = cfMap.get(vid);
+            if (!vid.isEmpty()) {
+                for (String key_feature : Arrays.asList("c8_feature", "c9_feature")) {
+                    for (String key_action : Arrays.asList("share", "return")) {
+                        Map<String, String[]> cfMap = c89Map.getOrDefault(key_feature + "_" + key_action, new HashMap<>());
+                        if (cfMap.containsKey(vid)) {
+                            String[] scores = cfMap.get(vid);
                             Double score1 = Double.parseDouble(scores[0]);
                             Double score2 = Double.parseDouble(scores[1]);
                             Double score3 = Double.parseDouble(scores[2]) <= 0 ? 0D : 1.0 / Double.parseDouble(scores[2]);
@@ -259,7 +258,7 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
                 }
             }
             Map<String, String> d1 = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_recsys_feature_cf_i2i_new", new HashMap<>());
-            if (!d1.isEmpty()){
+            if (!d1.isEmpty()) {
                 featureMap.put("d1_exp", Double.parseDouble(d1.getOrDefault("exp", "0")));
                 featureMap.put("d1_return_n", Double.parseDouble(d1.getOrDefault("return_n", "0")));
                 featureMap.put("d1_rovn", Double.parseDouble(d1.getOrDefault("rovn", "0")));
@@ -270,29 +269,29 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
         // 3 连续值特征分桶
         readBucketFile();
         Map<String, String> userFeatureMap = new HashMap<>(userFeatureMapDouble.size());
-        for (Map.Entry<String, Double> entry: userFeatureMapDouble.entrySet()){
+        for (Map.Entry<String, Double> entry : userFeatureMapDouble.entrySet()) {
             String name = entry.getKey();
             Double score = entry.getValue();
             // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)){
+            if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
                 Double bucketNum = this.bucketsLen.get(name);
-                double [] buckets = this.bucketsMap.get(name);
+                double[] buckets = this.bucketsMap.get(name);
                 Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
                 userFeatureMap.put(name, String.valueOf(scoreNew));
             }
         }
 
-        for (RankItem item : rankItems){
+        for (RankItem item : rankItems) {
             Map<String, String> featureMap = new HashMap<>();
             Map<String, Double> featureMapDouble = item.featureMapDouble;
 
-            for (Map.Entry<String, Double> entry: featureMapDouble.entrySet()){
+            for (Map.Entry<String, Double> entry : featureMapDouble.entrySet()) {
                 String name = entry.getKey();
                 Double score = entry.getValue();
                 // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)){
+                if (score > 1E-8 && this.bucketsLen.containsKey(name) && this.bucketsMap.containsKey(name)) {
                     Double bucketNum = this.bucketsLen.get(name);
-                    double [] buckets = this.bucketsMap.get(name);
+                    double[] buckets = this.bucketsMap.get(name);
                     Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
                     featureMap.put(name, String.valueOf(scoreNew));
                 }
@@ -302,6 +301,7 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
 
         // TODO 3 排序
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
+
         List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_20240609.conf")
                 .scoring(sceneFeatureMap, userFeatureMap, rankItems);
         List<Video> result = new ArrayList<>();
@@ -311,6 +311,16 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
             video.setScore(score);
             video.setSortScore(score);
             video.setScoresMap(item.getScoresMap());
+            video.setAllFeatureMap(item.getAllFeatureMap());
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getVideoFeature())
+                    && MapUtils.isNotEmpty(feature.getVideoFeature().get(item.getVideoId()))) {
+                video.getMetaFeatureMap().putAll(feature.getVideoFeature().get(item.getVideoId()));
+            }
+            if (feature != null
+                    && MapUtils.isNotEmpty(feature.getUserFeature())) {
+                video.getMetaFeatureMap().putAll(feature.getUserFeature());
+            }
             result.add(video);
         }
         result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
@@ -333,7 +343,7 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
         return null;
     }
 
-    private void readBucketFile(){
+    private void readBucketFile() {
         InputStream resourceStream = RankStrategy4RegionMergeModelV999.class.getClassLoader().getResourceAsStream("20240609_bucket_274.txt");
         if (resourceStream != null) {
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
@@ -374,7 +384,7 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
 
         public String name;
 
-        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name){
+        public Tuple4(Map<String, String> first, Map<String, String> second, Map<String, String> third, String name) {
             this.first = first;
             this.second = second;
             this.third = third;
@@ -388,7 +398,7 @@ public class RankStrategy4RegionMergeModelV999 extends RankStrategy4RegionMergeM
 
         public String name;
 
-        public Tuple2(Map<String, String> first, String name){
+        public Tuple2(Map<String, String> first, String name) {
             this.first = first;
             this.name = name;
         }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/VlogRovFMScorer.java

@@ -148,12 +148,14 @@ public class VlogRovFMScorer extends BaseFMModelScorer {
         if (MapUtils.isNotEmpty(featureMap)) {
             try {
                 pro = model.score(featureMap);
-                LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+                // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
             } catch (Exception e) {
                 LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
         }
         item.setScoreRov(pro);
+        item.getScoresMap().put("RovFMScore", pro);
+        item.setAllFeatureMap(featureMap);
         return pro;
     }
 }