Merge branch 'master' into feature_20250526_zhaohaipeng_douhot

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV568.java

@@ -16,7 +16,6 @@ import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
-import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
@@ -107,14 +106,9 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         headCate2Rov = headCate2Rov.subList(0, Math.min(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), headCate2Rov.size()));
         rovRecallRank.addAll(headCate2Rov);
         setVideo.addAll(headCate2Rov.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //------------------- head cate2 and channel of rovn------------------
-        List<Video> headCate2AndChannelRov = extractAndSort(param, HeadCate2AndChannelRovRecallStrategy.PUSH_FROM);
-        // 视频去重
-        removeDuplicate(headCate2AndChannelRov);
-        headCate2AndChannelRov = headCate2AndChannelRov.stream().filter(o -> !setVideo.contains(o.getVideoId())).collect(Collectors.toList());
-        headCate2AndChannelRov = headCate2AndChannelRov.subList(0, Math.min(mergeWeight.getOrDefault("headCate2AndChannelRov", 5.0).intValue(), headCate2AndChannelRov.size()));
-        rovRecallRank.addAll(headCate2AndChannelRov);
-        setVideo.addAll(headCate2AndChannelRov.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        // -------------------premium rov------------------
+        int preRovN = mergeWeight.getOrDefault("preRovN", 10.0).intValue();
+        addRecall(param, preRovN, PremiumROVRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -145,7 +139,7 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
 
         // 4. 排序模型计算
         Map<String, String> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250423.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_fm_xgb_20250317.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
 
         // 5. 排序公式特征
         double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
@@ -160,11 +154,14 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
             String headVidStr = String.valueOf(param.getHeadVid());
             String mergeCate2 = this.findVideoMergeCate2(videoBaseInfoMap, headVidStr);
             Double length = mergeWeight.getOrDefault("cate2CoefficientLength", 10000d);
-            Map<String, Double> simCateScore = this.findSimCateScore(mergeCate2, param.getChannelName(), length.intValue());
+            Map<String, Double> simCateScore = this.findSimCateScore(mergeCate2, length.intValue());
             cate2Coefficient.putAll(simCateScore);
         }
         Double cate2CoefficientDenominator = mergeWeight.getOrDefault("cate2CoefficientDenominator", 1d);
 
+        Map<String, Double> preVideosWeightMap = getPremiumVideosWeight();
+        double premiumVideosCoefficient = mergeWeight.getOrDefault("premiumVideosCoefficient", 0.3);
+
         List<Video> result = new ArrayList<>();
         for (RankItem item : items) {
             double score;
@@ -185,6 +182,11 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
             item.getScoresMap().put("cate2CoefficientDenominator", cate2CoefficientDenominator);
 
             score = fmRov * (0.1 + newNorXGBScore) * (0.1 + vor) * (1 + scoreCoefficient / cate2CoefficientDenominator);
+            if (preVideosWeightMap.containsKey(item.getVideoId() + "")) {
+                double preWeight = 1.0 + premiumVideosCoefficient * preVideosWeightMap.get(item.getVideoId() + "");
+                score = score * preWeight;
+                item.getScoresMap().put("preWeight", preWeight);
+            }
 
             Video video = item.getVideo();
             video.setScore(score);
@@ -378,13 +380,6 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
                 baseInfo.put("system", system);
             }
         }
-        String userChannel = param.getChannelName();
-        if (null != userChannel && !userChannel.isEmpty()) {
-            baseInfo.put("user_channel", userChannel);
-        }
-        if (FeatureUtils.firstLevel(param.getUserShareDepth())) {
-            baseInfo.put("user_level", "1st");
-        }
         return baseInfo;
     }
 
@@ -408,12 +403,12 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         }
     }
 
-    private Map<String, Double> findSimCateScore(String headCate2, String channelName, int length) {
+    private Map<String, Double> findSimCateScore(String headCate2, int length) {
         if (StringUtils.isBlank(headCate2)) {
             return new HashMap<>();
         }
 
-        String redisKey = String.format("alg_recsys_channel_good_cate_pair_list:%s:%s", channelName, headCate2);
+        String redisKey = String.format("alg_recsys_good_cate_pair_list:%s", headCate2);
         String cate2Value = redisTemplate.opsForValue().get(redisKey);
         if (StringUtils.isEmpty(cate2Value)) {
             return new HashMap<>();
@@ -443,6 +438,7 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         for (int i = 0; i < minLength; i++) {
             resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
         }
+
         return resultMap;
     }
 
@@ -450,4 +446,9 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
         return videoInfo.get("merge_second_level_cate");
     }
+
+    private Map<String, Double> getPremiumVideosWeight() {
+        String redisValue = redisTemplate.opsForValue().get(PremiumROVRecallStrategy.redisKey);
+        return this.parsePair(redisValue, 100);
+    }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -118,7 +118,7 @@ public class RecallService implements ApplicationContextAware {
             strategies.add(strategyMap.get(HeadCate2AndChannelRovRecallStrategy.class.getSimpleName()));
         }
         if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568")) {
-            strategies.add(strategyMap.get(HeadCate2AndChannelRovRecallStrategy.class.getSimpleName()));
+            strategies.add(strategyMap.get(PremiumROVRecallStrategy.class.getSimpleName()));
         }
 
         // 命中用户黑名单不走流量池

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/PremiumROVRecallStrategy.java

@@ -0,0 +1,100 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class PremiumROVRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String PUSH_FROM = "premium_rov";
+    public static final String redisKey = "high_rovn_low_exp_recall:all";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FROM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            String redisValue = redisTemplate.opsForValue().get(redisKey);
+            if (null == redisValue || redisValue.isEmpty()) {
+                return videosResult;
+            }
+            Pair<List<Long>, Map<Long, Double>> pair = parsePair(redisValue, param.getVideoId(), 100);
+            fillVideoResult(param, pair, videosResult);
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private Pair<List<Long>, Map<Long, Double>> parsePair(String data, long headVid, int size) {
+        List<Long> idsList = new ArrayList<>();
+        Map<Long, Double> scoresMap = new HashMap<>();
+        if (!StringUtils.isBlank(data)) {
+            String[] cells = data.split("\t");
+            if (2 == cells.length) {
+                List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty() && ids.size() == scores.size()) {
+                    int minSize = Math.min(size, ids.size());
+                    for (int i = 0; i < minSize; ++i) {
+                        long id = ids.get(i);
+                        double score = scores.get(i);
+                        if (headVid != id && !scoresMap.containsKey(id)) {
+                            idsList.add(id);
+                            scoresMap.put(id, score);
+                        }
+                    }
+                }
+            }
+        }
+        return new MutablePair<>(idsList, scoresMap);
+    }
+
+    private void fillVideoResult(RecallParam param, Pair<List<Long>, Map<Long, Double>> pair, List<Video> videosResult) {
+        if (null != pair) {
+            List<Long> ids = pair.getLeft();
+            Map<Long, Double> scoresMap = pair.getRight();
+            if (null != ids && null != scoresMap && !ids.isEmpty()) {
+                FilterParam filterParam = FilterParamFactory.create(param, ids);
+                FilterResult filterResult = filterService.filter(filterParam);
+                if (null != filterResult && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                    filterResult.getVideoIds().forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setRovScore(scoresMap.getOrDefault(vid, 0D));
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
+                }
+            }
+        }
+    }
+}