Merge branch 'feature_zhaohaipeng_20260410_dnn_v2_param' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV563.java

@@ -15,6 +15,7 @@ import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.*;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -23,6 +24,8 @@ import org.springframework.stereotype.Service;
 import java.util.*;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @Service
 @Slf4j
@@ -91,8 +94,22 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         // 1. 批量获取特征  省份参数要对齐  headvid  要传递过来！
         // k1:视频、k2:表、k3:特征、v:特征值
         Map<String, String> headVideoInfo = param.getHeadInfo();
+
+        // 用户的序列特征
+        Map<String, Map<String, String>> unionIdFeature = featureService.getUnionIdFeature(param.getUnionId());
+        Map<String, String> userNetworkSeqFeature = unionIdFeature.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
+        List<String> actVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_v_s");
+        List<String> netVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "n_v_s");
+
         List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
-        Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", vids);
+
+        List<String> allVids = Stream.of(actVidSeq, netVidSeq, vids)
+                .flatMap(Collection::stream)
+                .distinct()
+                .filter(StringUtils::isNotBlank)
+                .collect(Collectors.toList());
+
+        Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", allVids);
         Map<String, Map<String, Map<String, String>>> videoBCData = featureService.getVideoStatistics(vids);
 
         FeatureService.Feature feature = featureService.getFeatureV4(param, headVideoInfo, videoBaseInfoMap, vids);
@@ -106,15 +123,20 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         Map<String, Map<String, String>> userBehaviorVideoMap = param.getBehaviorVideos();
         Map<String, String> creativeInfo = param.getCreativeInfoFeature();
 
+        Map<String, String> featureMapToString = new HashMap<>();
+        FeatureV6.parseStringFeatureMap(featureMapToString, param);
+        FeatureV6.putVideoStringFeatures("h", headVideoInfo, featureMapToString);
+
         // 3. 特征处理
         List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
         Map<String, Float> userFeatureMap = getUserFeature(currentMs, param, creativeInfo, headVideoInfo, userProfile, featureOriginUser);
         batchGetVideoFeature(currentMs, userProfile, creativeInfo, headVideoInfo, videoBaseInfoMap,
-                newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, rankItems);
+                newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, featureMapToString, userFeatureMap, rankItems);
+
 
         // 4. 排序模型计算
         Map<String, Float> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250729.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_xgb_ros_20250311.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
 
         // 5. 排序公式特征
         double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
@@ -136,9 +158,15 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         double b0Ror1hW = mergeWeight.getOrDefault("b0_ror_1h_w", 0d);
         double b0Ror24hW = mergeWeight.getOrDefault("b0_ror_24h_w", 0d);
 
+        double cnRovn1hW = mergeWeight.getOrDefault("cn_rovn_1h_w", 0d);
+        double cnRovn24hW = mergeWeight.getOrDefault("cn_rovn_24h_w", 0d);
+
+        double dnRovn1hW = mergeWeight.getOrDefault("dn_rovn_1h_w", 0d);
+        double dnRovn24hW = mergeWeight.getOrDefault("dn_rovn_24h_w", 0d);
+
         Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
 
-        // 获取权重
+
         Map<String, String> contextInfo = getContextInfo(param);
 
         List<Video> result = new ArrayList<>();
@@ -152,8 +180,9 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
             double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
 
-            double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
-            double newNorXGBScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
+            double norDNNScore = item.getScoresMap().getOrDefault("NorDNNScore", 0d);
+            double newNorDNNScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norDNNScore);
+            item.getScoresMap().put("newNorDNNScore", newNorDNNScore);
             item.getScoresMap().put("rosAdd", rosAdd);
             item.getScoresMap().put("rosW", rosW);
 
@@ -163,6 +192,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
             item.getScoresMap().put("vorW", vorW);
 
             Map<String, String> bcData = videoBCData.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault("alg_vid_feature_b_c_data", new HashMap<>());
+            Map<String, String> cdNData = videoBCData.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault("alg_vid_feature_cn_dn_data", new HashMap<>());
 
             double c1Rovn1h = Double.parseDouble(bcData.getOrDefault("c1_rovn_1h", "0"));
             double c1Rovn24h = Double.parseDouble(bcData.getOrDefault("c1_rovn_24h", "0"));
@@ -192,7 +222,25 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
             item.getScoresMap().put("b0Ror24hW", b0Ror24hW);
             item.getScoresMap().put("b0Ror24h", b0Ror24h);
 
-            score = fmRov * (rosAdd + rosW * newNorXGBScore) * (vorAdd + vorW * vor) + c1RovnScore + b0StrScore + b0RorScore;
+            double cnRovn1h = Double.parseDouble(cdNData.getOrDefault("cn_rovn_1h", "0"));
+            double cnRovn24h = Double.parseDouble(cdNData.getOrDefault("cn_rovn_24h", "0"));
+            double cnRovnScore = cnRovn1hW * cnRovn1h + cnRovn24hW * cnRovn24h;
+            item.getScoresMap().put("cnRovnScore", cnRovnScore);
+            item.getScoresMap().put("cnRovn1hW", cnRovn1hW);
+            item.getScoresMap().put("cnRovn1h", cnRovn1h);
+            item.getScoresMap().put("cnRovn24hW", cnRovn24hW);
+            item.getScoresMap().put("cnRovn24h", cnRovn24h);
+
+            double dnRovn1h = Double.parseDouble(cdNData.getOrDefault("dn_rovn_1h", "0"));
+            double dnRovn24h = Double.parseDouble(cdNData.getOrDefault("dn_rovn_24h", "0"));
+            double dnRovnScore = dnRovn1hW * dnRovn1h + dnRovn24hW * dnRovn24h;
+            item.getScoresMap().put("dnRovnScore", dnRovnScore);
+            item.getScoresMap().put("dnRovn1hW", dnRovn1hW);
+            item.getScoresMap().put("dnRovn1h", dnRovn1h);
+            item.getScoresMap().put("dnRovn24hW", dnRovn24hW);
+            item.getScoresMap().put("dnRovn24h", dnRovn24h);
+
+            score = fmRov * (rosAdd + rosW * newNorDNNScore) * (vorAdd + vorW * vor) + c1RovnScore + b0StrScore + b0RorScore + cnRovnScore + dnRovnScore;
 
             Video video = item.getVideo();
             video.setScore(score);
@@ -226,7 +274,7 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
             if (MapUtils.isNotEmpty(contextInfo)) {
                 video.getMetaFeatureMap().put("context", contextInfo);
             }
-            if (Objects.nonNull(video.getRankVideoInfoMap()) && video.getRankVideoInfoMap().containsKey(video.getVideoId())){
+            if (Objects.nonNull(video.getRankVideoInfoMap()) && video.getRankVideoInfoMap().containsKey(video.getVideoId())) {
                 video.getRankVideoInfoMap().get(video.getVideoId()).setScore(score);
                 video.getRankVideoInfoMap().get(video.getVideoId()).setScoresMap(video.getScoresMap());
             }
@@ -313,15 +361,28 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
                                       Map<String, Map<String, String>> userOriginInfo,
                                       Map<String, Map<String, String>> historyVideoMap,
                                       Map<String, Map<String, Map<String, String>>> videoOriginInfo,
+                                      Map<String, String> featureMapToString,
+                                      Map<String, Float> userFeatureMap,
                                       List<RankItem> rankItems) {
-        if (null != rankItems && !rankItems.isEmpty()) {
+        if (CollectionUtils.isNotEmpty(rankItems)) {
             List<Future<Integer>> futures = new ArrayList<>();
             for (RankItem item : rankItems) {
                 String vid = item.getVideoId() + "";
                 Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
                 Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                    item.featureMap = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
-                    item.norFeatureMap = item.featureMap;
+                    Map<String, Float> featureMap = new HashMap<>(userFeatureMap);
+                    Map<String, Float> videoFeature = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
+                    featureMap.putAll(videoFeature);
+                    item.featureMap = featureMap;
+
+                    Map<String, String> userNetworkSeqFeature = userOriginInfo.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
+
+                    Map<String, String> featureMapString = new HashMap<>(featureMapToString);
+                    FeatureV6.putVideoStringFeatures("r", rankInfo, featureMapString);
+                    featureMapString.put("r@vid", "r_vid_" + vid);
+                    FeatureV6.putProfileVideoCrossStringFeature(currentMs, userProfile, historyVideoMap, featureMapString);
+                    FeatureV6.putUserNetworkSeqFeature(featureMapString, userNetworkSeqFeature, videoBaseInfoMap);
+                    item.featureMapString = featureMapString;
                     return 1;
                 });
                 futures.add(future);
@@ -393,48 +454,4 @@ public class RankStrategy4RegionMergeModelV563 extends RankStrategy4RegionMergeM
         }
         return newScore;
     }
-
-    private Map<String, Double> findSimCateScore(String headCate2, int length) {
-        if (StringUtils.isBlank(headCate2)) {
-            return new HashMap<>();
-        }
-
-        String redisKey = String.format("alg_recsys_good_cate_pair_list:%s", headCate2);
-        String cate2Value = redisTemplate.opsForValue().get(redisKey);
-        if (StringUtils.isEmpty(cate2Value)) {
-            return new HashMap<>();
-        }
-
-        return this.parsePair(cate2Value, length);
-    }
-
-    private Map<String, Double> parsePair(String value, int length) {
-        if (StringUtils.isBlank(value)) {
-            return new HashMap<>();
-        }
-
-        String[] split = value.split("\t");
-        if (split.length != 2) {
-            return new HashMap<>();
-        }
-
-        String[] valueList = split[0].trim().split(",");
-        String[] scoreList = split[1].trim().split(",");
-        if (valueList.length != scoreList.length) {
-            return new HashMap<>();
-        }
-
-        int minLength = Math.min(length, valueList.length);
-        Map<String, Double> resultMap = new HashMap<>();
-        for (int i = 0; i < minLength; i++) {
-            resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
-        }
-
-        return resultMap;
-    }
-
-    private String findVideoMergeCate2(Map<String, Map<String, Map<String, String>>> featureOriginVideo, String vid) {
-        Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
-        return videoInfo.get("merge_second_level_cate");
-    }
 }