feat:添加头部视频同品类召回

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/FeatureService.java

@@ -79,6 +79,8 @@ public class FeatureService {
             //protos.add(genWithVidAndHeadVid("alg_recsys_feature_cf_i2i_new_v2", vid, headVid));
         }
 
+        // 头部视频的基础信息
+        protos.add(genWithVid("alg_vid_feature_basic_info", headVid));
 
         // user
         protos.add(genWithMid("alg_mid_feature_play", mid));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV567.java

@@ -12,6 +12,7 @@ import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.math3.util.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -78,6 +79,13 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         rovRecallRank.addAll(sceneCFRosn);
         setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
 
+        //-------------------head cate2 of rovn------------------
+        List<Video> headCate2Rov = extractAndSort(param, HeadCate2RovRecallStrategy.PUSH_FROM);
+        headCate2Rov = headCate2Rov.stream().filter(o -> !setVideo.contains(o.getVideoId())).collect(Collectors.toList());
+        headCate2Rov = headCate2Rov.subList(0, Math.min(mergeWeight.getOrDefault("headCate2Rov", 5.0).intValue(), headCate2Rov.size()));
+        rovRecallRank.addAll(headCate2Rov);
+        setVideo.addAll(headCate2Rov.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+
         //-------------------排-------------------
         //-------------------序-------------------
         //-------------------逻-------------------
@@ -318,62 +326,36 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         // 5 排序公式特征
         Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
 
-        // Ros增强传播因子
-        Map<String, Map<String, String>> rosSpreadDivMap = this.getVideoRedisFeature(vids, "vid_for_spread:");
+        // 获取权重
+        Map<String, Double> cate2Coefficient = new HashMap<>();
+        double cate2CoefficientFunc = mergeWeight.getOrDefault("cate2CoefficientFunc", 0d);
+        if (cate2CoefficientFunc == 1d) {
+            String headVidStr = String.valueOf(param.getHeadVid());
+            String mergeCate2 = this.findVideoMergeCate2(featureOriginVideo, headVidStr);
+            Double length = mergeWeight.getOrDefault("cate2CoefficientLength", 10000d);
+            Map<String, Double> simCateScore = this.findSimCateScore(mergeCate2, length.intValue());
+            cate2Coefficient.putAll(simCateScore);
+        }
 
         List<Video> result = new ArrayList<>();
-
-        double calcVorMode = mergeWeight.getOrDefault("calcVorMode", 3d);
-        double calcRosMode = mergeWeight.getOrDefault("calcRosMode", 0d);
-        double calcStrMode = mergeWeight.getOrDefault("calcStrMode", 3d);
-
-        double rosAdd = mergeWeight.getOrDefault("ros_add", 0.1d);
-        double ros2Multi = mergeWeight.getOrDefault("ros2_multi", 1d);
-        double vorAdd = mergeWeight.getOrDefault("vor_add", 0d);
-
-        double rosSpreadDivisorIndex = mergeWeight.getOrDefault("rosSpreadDivisorIndex", 2d);
-        String spreadDivisorKey = this.indexCoverKey(rosSpreadDivisorIndex);
-        log.info("567 spreadDivisorKey is: {}", spreadDivisorKey);
-
         for (RankItem item : items) {
             double score;
             double fmRovOrigin = item.getScoreRov();
             item.getScoresMap().put("fmRovOrigin", fmRovOrigin);
-            double str = restoreScore(fmRovOrigin);
-            item.getScoresMap().put("originStr", str);
-            str = this.handleStr(str, calcStrMode, item, mergeWeight);
-            item.getScoresMap().put("xgbRovNegRate", 0.9d);
-            item.getScoresMap().put("fmRov", str);
-            item.getScoresMap().put("str", str);
-            item.getScoresMap().put("calcStrMode", calcStrMode);
-
-            double originRos = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
-            double ros = this.handleRos(originRos, calcRosMode, item, mergeWeight);
-            item.getScoresMap().put("hasReturnRovScore", ros);
-            item.getScoresMap().put("ros", ros);
-            item.getScoresMap().put("originRos", originRos);
-            item.getScoresMap().put("calcRosMode", calcRosMode);
-
-            String spreadDivStr = rosSpreadDivMap.getOrDefault(String.valueOf(item.getVideoId()), new HashMap<>()).getOrDefault(spreadDivisorKey, "0");
-            double rosSpreadDiv = Double.parseDouble(spreadDivStr);
-            item.getScoresMap().put("rosSpreadDiv", rosSpreadDiv);
-
-            double originVor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
-            double vor = this.handleVor(originVor, calcVorMode, item, mergeWeight);
-            item.getScoresMap().put("originVor", originVor);
+            double fmRov = restoreScore(fmRovOrigin);
+            item.getScoresMap().put("fmRov", fmRov);
+            double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
+            item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
+            double vor = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("vor", "0"));
             item.getScoresMap().put("vor", vor);
-            item.getScoresMap().put("calcVorMode", calcVorMode);
 
+            String vidMergeCate2 = this.findVideoMergeCate2(featureOriginVideo, String.valueOf(item.getVideoId()));
+            Double scoreCoefficient = cate2Coefficient.getOrDefault(vidMergeCate2, 1d);
+            item.getScoresMap().put("scoreCoefficient", scoreCoefficient);
 
-            item.getScoresMap().put("rosAdd", rosAdd);
-            item.getScoresMap().put("vorAdd", vorAdd);
-            item.getScoresMap().put("ros2Multi", ros2Multi);
-            item.getScoresMap().put("rosSpreadDivisorIndex", rosSpreadDivisorIndex);
-            score = str * (rosAdd + ros + ros2Multi * rosSpreadDiv) * (vorAdd + vor);
+            score = fmRov * (0.1 + hasReturnRovScore) * (0.1 + vor) * scoreCoefficient;
 
             Video video = item.getVideo();
-            video.setScoreStr(str);
-            video.setScoreRos(rosAdd + ros + ros2Multi * rosSpreadDiv);
             video.setScore(score);
             video.setSortScore(score);
             video.setScoresMap(item.getScoresMap());
@@ -390,17 +372,47 @@ public class RankStrategy4RegionMergeModelV567 extends RankStrategy4RegionMergeM
         return result;
     }
 
-    private String indexCoverKey(double index) {
-        switch (String.valueOf(index)) {
-            case "1":
-                return "head_video_rov1";
-            case "3":
-                return "head_video_recommend_rovn";
-            case "4":
-                return "head_video_recommend_fission_rate";
-            default:
-                return "recommend_123_depth_fission_rate";
+    private Map<String, Double> findSimCateScore(String headCate2, int length) {
+        if (StringUtils.isBlank(headCate2)) {
+            return new HashMap<>();
+        }
+
+        String redisKey = String.format("alg_recsys_good_cate_pair_list:%s", headCate2);
+        String cate2Value = redisTemplate.opsForValue().get(redisKey);
+        if (StringUtils.isEmpty(cate2Value)) {
+            return new HashMap<>();
+        }
+
+        return this.parsePair(cate2Value, length);
+    }
+
+    private Map<String, Double> parsePair(String value, int length) {
+        if (StringUtils.isBlank(value)) {
+            return new HashMap<>();
+        }
+
+        String[] split = value.split("\t");
+        if (split.length != 2) {
+            return new HashMap<>();
+        }
+
+        String[] valueList = split[0].trim().split(",");
+        String[] scoreList = split[1].trim().split(",");
+        if (valueList.length != scoreList.length) {
+            return new HashMap<>();
+        }
+
+        int minLength = Math.min(length, valueList.length);
+        Map<String, Double> resultMap = new HashMap<>();
+        for (int i = 0; i < minLength; i++) {
+            resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
         }
+
+        return resultMap;
     }
 
-}
+    private String findVideoMergeCate2(Map<String, Map<String, Map<String, String>>> featureOriginVideo, String vid) {
+        Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+        return videoInfo.get("merge_second_level_cate");
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -122,6 +122,10 @@ public class RecallService implements ApplicationContextAware {
             strategies.add(strategyMap.get(HeadProvinceCate2RecallStrategy.class.getSimpleName()));
         }
 
+        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("567")) {
+            strategies.add(strategyMap.get(HeadCate2RovRecallStrategy.class.getSimpleName()));
+        }
+
         // 命中用户黑名单不走流量池
         if (!param.isRiskUser()) {
             strategies.add(strategyMap.get(QuickFlowPoolWithLevelRecallStrategy.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/HeadCate2RovRecallStrategy.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service.recall.strategy;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.FeatureService;
 import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
@@ -36,7 +37,10 @@ public class HeadCate2RovRecallStrategy implements RecallStrategy {
     @Autowired
     private FeatureService featureService;
 
-    private static final String PUSH_FROM = "recall_strategy_head_cate2_rov";
+    @ApolloJsonValue("${head.cate2.rov.recall.param:{}}")
+    private Map<String, String> paramJson;
+
+    public static final String PUSH_FROM = "recall_strategy_head_cate2_rov";
 
     private static final String SIM_MERGE_CATE2_KEY_FORMAT = "alg_recsys_good_cate_pair_list:%s";
     private static final String MERGE_CATE2_VIDEO_LIST_KEY_FORMAT = "alg_recsys_recall_good_cate_pair_rovn:%s";
@@ -69,7 +73,8 @@ public class HeadCate2RovRecallStrategy implements RecallStrategy {
 
         List<Video> videoResult = new ArrayList<>();
 
-        Map<String, Double> mergeCate2Pair = this.parsePair(simCate2List);
+        int simCateLength = Integer.parseInt(paramJson.getOrDefault("sim_cate_length", "10000"));
+        Map<String, Double> mergeCate2Pair = this.parsePair(simCate2List, simCateLength);
 
         Map<String, Map<String, Double>> recallVideoMap = this.cate2Recall(new ArrayList<>(mergeCate2Pair.keySet()));
 
@@ -83,6 +88,7 @@ public class HeadCate2RovRecallStrategy implements RecallStrategy {
         FilterParam filterParam = FilterParamFactory.create(param, allVid);
         FilterResult filterResult = filterService.filter(filterParam);
         Set<Long> filterVids = new HashSet<>(filterResult.getVideoIds());
+        filterVids.add(param.getVideoId());
 
         for (Map.Entry<String, Double> entry : mergeCate2Pair.entrySet()) {
             String cate = entry.getKey();
@@ -113,7 +119,7 @@ public class HeadCate2RovRecallStrategy implements RecallStrategy {
         return videoResult;
     }
 
-    private Map<String, Double> parsePair(String value) {
+    private Map<String, Double> parsePair(String value, int length) {
         if (StringUtils.isBlank(value)) {
             return new HashMap<>();
         }
@@ -129,8 +135,9 @@ public class HeadCate2RovRecallStrategy implements RecallStrategy {
             return new HashMap<>();
         }
 
+        int minLength = Math.min(length, valueList.length);
         Map<String, Double> resultMap = new HashMap<>();
-        for (int i = 0; i < valueList.length; i++) {
+        for (int i = 0; i < minLength; i++) {
             resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
         }
 
@@ -148,6 +155,7 @@ public class HeadCate2RovRecallStrategy implements RecallStrategy {
 
         Map<String, Map<String, Double>> resultMap = new HashMap<>();
 
+        int recallVidLength = Integer.parseInt(paramJson.getOrDefault("recall_vid_length", "10000"));
         for (int i = 0; i < mergeCate2List.size(); i++) {
             String mergeCate2 = mergeCate2List.get(i);
 
@@ -156,7 +164,7 @@ public class HeadCate2RovRecallStrategy implements RecallStrategy {
                 continue;
             }
 
-            Map<String, Double> recallVideoMap = this.parsePair(value);
+            Map<String, Double> recallVideoMap = this.parsePair(value, recallVidLength);
             if (MapUtils.isEmpty(recallVideoMap)) {
                 continue;
             }