Merge branch 'feature_20260511' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -38,6 +38,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
@@ -52,6 +53,7 @@ import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * @author dyp
@@ -534,6 +536,19 @@ public class RecommendService {
             userSocialRecallInfo = featureService.getUserSocialRecallInfo(param.getMid());
         }
 
+        Map<String, Map<String, String>> unionIdFeature = featureService.getUnionIdFeature(param.getUnionId());
+        Map<String, String> userNetworkSeqFeature = unionIdFeature.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
+        List<String> actVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_v_s");
+        List<String> netVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "n_v_s");
+
+        List<String> allVids = Stream.of(actVidSeq, netVidSeq)
+                .flatMap(Collection::stream)
+                .distinct()
+                .filter(StringUtils::isNotBlank)
+                .collect(Collectors.toList());
+
+        Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", allVids);
+
         Map<String, String> headVideoInfo;
         if (null != behaviorVideos && behaviorVideos.containsKey(vid)) {
             headVideoInfo = behaviorVideos.get(vid);
@@ -544,12 +559,21 @@ public class RecommendService {
         // timerLogMapTL.get().put("infoTime", infoTime);
         stopwatch.reset().start();
 
+        Map<Long, Map<String, String>> userNetworkSeqVideoInfoMap = new HashMap<>(allVids.size());
+        for (String vidItem : allVids) {
+            long videoId = NumberUtils.toLong(vidItem, 0L);
+            Map<String, String> videoBaseInfo = videoBaseInfoMap.getOrDefault(vidItem, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            userNetworkSeqVideoInfoMap.put(videoId, videoBaseInfo);
+        }
+
         RecallParam recallParam = convertToRecallParam(param);
         recallParam.setUserProfile(userProfile);
         recallParam.setHeadInfo(headVideoInfo);
         recallParam.setUserRTShareList(param.getUserRTShareList());
         recallParam.setBehaviorVideos(behaviorVideos);
         recallParam.setUserSocialRecallInfo(userSocialRecallInfo);
+        recallParam.setUserNetworkSeqFeature(userNetworkSeqFeature);
+        recallParam.setUserNetworkSeqVideoInfoMap(userNetworkSeqVideoInfoMap);
         RecallResult recallResult = recallService.recall(recallParam);
 
         long recallTime = stopwatch.elapsed(TimeUnit.MILLISECONDS);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV568.java

@@ -15,6 +15,7 @@ import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.*;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -23,6 +24,8 @@ import org.springframework.stereotype.Service;
 import java.util.*;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @Service
 @Slf4j
@@ -78,9 +81,10 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         RecallUtils.extractRecall(mergeWeight.getOrDefault("return1Cate2Ros", 5.0).intValue(), param, Return1Cate2RosRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
         //-------------------return1 cate2 str------------------
         RecallUtils.extractRecall(mergeWeight.getOrDefault("return1Cate2Str", 5.0).intValue(), param, Return1Cate2StrRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
-        //--------------deconstruction keywords ros-------------
-        RecallUtils.extractRecall(mergeWeight.getOrDefault("deconstructionKeywordsRos", 5.0).intValue(), param, UserDeconstructionKeywordsRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
 
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("yearShareCate1", 5.0).intValue(), param, YearShareCate1RecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("yearShareCate2", 5.0).intValue(), param, YearShareCate2RecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("yearReturnCate2", 5.0).intValue(), param, YearReturnCate2RecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
 
         // 记录召回源中的视频
         this.rankBeforePostProcessor(rovRecallRank);
@@ -94,8 +98,22 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         // 1. 批量获取特征  省份参数要对齐  headvid  要传递过来！
         // k1:视频、k2:表、k3:特征、v:特征值
         Map<String, String> headVideoInfo = param.getHeadInfo();
+
+        // 用户的序列特征
+        Map<String, Map<String, String>> unionIdFeature = featureService.getUnionIdFeature(param.getUnionId());
+        Map<String, String> userNetworkSeqFeature = unionIdFeature.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
+        List<String> actVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_v_s");
+        List<String> netVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "n_v_s");
+
         List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
-        Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", vids);
+
+        List<String> allVids = Stream.of(actVidSeq, netVidSeq, vids)
+                .flatMap(Collection::stream)
+                .distinct()
+                .filter(StringUtils::isNotBlank)
+                .collect(Collectors.toList());
+
+        Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo("", allVids);
         Map<String, Map<String, Map<String, String>>> videoBCData = featureService.getVideoStatistics(vids);
 
         FeatureService.Feature feature = featureService.getFeatureV4(param, headVideoInfo, videoBaseInfoMap, vids);
@@ -109,15 +127,20 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         Map<String, Map<String, String>> userBehaviorVideoMap = param.getBehaviorVideos();
         Map<String, String> creativeInfo = param.getCreativeInfoFeature();
 
+        Map<String, String> featureMapToString = new HashMap<>();
+        FeatureV6.parseStringFeatureMap(featureMapToString, param);
+        FeatureV6.putVideoStringFeatures("h", headVideoInfo, featureMapToString);
+
         // 3. 特征处理
         List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
         Map<String, Float> userFeatureMap = getUserFeature(currentMs, param, creativeInfo, headVideoInfo, userProfile, featureOriginUser);
         batchGetVideoFeature(currentMs, userProfile, creativeInfo, headVideoInfo, videoBaseInfoMap,
-                newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, rankItems);
+                newC7Map, newC8Map, featureOriginUser, userBehaviorVideoMap, featureOriginVideo, featureMapToString, userFeatureMap, rankItems);
+
 
         // 4. 排序模型计算
         Map<String, Float> sceneFeatureMap = new HashMap<>(0);
-        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_str_and_ros_20260319.conf").scoring(sceneFeatureMap, userFeatureMap, userFeatureMap, rankItems);
+        List<RankItem> items = ScorerUtils.getScorerPipeline("feeds_score_config_dnn_20260407.conf").scoring(sceneFeatureMap, userFeatureMap, rankItems);
 
         // 5. 排序公式特征
         double xgbRovNegRate = mergeWeight.getOrDefault("xgbRovNegRate", 0.059);
@@ -147,6 +170,7 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
 
         Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "redis:vid_hasreturn_vor:");
 
+
         Map<String, String> contextInfo = getContextInfo(param);
 
         List<Video> result = new ArrayList<>();
@@ -157,12 +181,12 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
             double fmRov = restoreScore(fmRovOrigin, xgbRovNegRate);
             item.getScoresMap().put("fmRov", fmRov);
 
-
             double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
 
-            double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);
-            double newNorXGBScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norXGBScore);
+            double norDNNScore = item.getScoresMap().getOrDefault("NorDNNScore", 0d);
+            double newNorDNNScore = norPowerCalibration(xgbNorPowerWeight, xgbNorPowerExp, norDNNScore);
+            item.getScoresMap().put("newNorDNNScore", newNorDNNScore);
             item.getScoresMap().put("rosAdd", rosAdd);
             item.getScoresMap().put("rosW", rosW);
 
@@ -220,14 +244,13 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
             item.getScoresMap().put("dnRovn24hW", dnRovn24hW);
             item.getScoresMap().put("dnRovn24h", dnRovn24h);
 
-            score = fmRov * (rosAdd + rosW * newNorXGBScore) * (vorAdd + vorW * vor) + c1RovnScore + b0StrScore + b0RorScore + cnRovnScore + dnRovnScore;
+            score = fmRov * (rosAdd + rosW * newNorDNNScore) * (vorAdd + vorW * vor) + c1RovnScore + b0StrScore + b0RorScore + cnRovnScore + dnRovnScore;
 
             Video video = item.getVideo();
             video.setScore(score);
             video.setSortScore(score);
             video.setScoresMap(item.getScoresMap());
             video.setAllFeatureMap(item.getAllFeatureMap());
-            video.setAllFeatureMap(item.getAllFeatureMap());
 
             String mergeCate2 = ExtractVideoMergeCate.parseMergeCate2(String.valueOf(item.getVideoId()), videoBaseInfoMap);
             if (StringUtils.isNotBlank(mergeCate2)) {
@@ -342,15 +365,28 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
                                       Map<String, Map<String, String>> userOriginInfo,
                                       Map<String, Map<String, String>> historyVideoMap,
                                       Map<String, Map<String, Map<String, String>>> videoOriginInfo,
+                                      Map<String, String> featureMapToString,
+                                      Map<String, Float> userFeatureMap,
                                       List<RankItem> rankItems) {
-        if (null != rankItems && !rankItems.isEmpty()) {
+        if (CollectionUtils.isNotEmpty(rankItems)) {
             List<Future<Integer>> futures = new ArrayList<>();
             for (RankItem item : rankItems) {
                 String vid = item.getVideoId() + "";
                 Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
                 Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
-                    item.featureMap = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
-                    item.norFeatureMap = item.featureMap;
+                    Map<String, Float> featureMap = new HashMap<>(userFeatureMap);
+                    Map<String, Float> videoFeature = getVideoFeature(currentMs, vid, userProfile, creativeInfo, headInfo, rankInfo, c7Map, c8Map, userOriginInfo, historyVideoMap, videoOriginInfo);
+                    featureMap.putAll(videoFeature);
+                    item.featureMap = featureMap;
+
+                    Map<String, String> userNetworkSeqFeature = userOriginInfo.getOrDefault("alg_user_network_seq_feature", new HashMap<>());
+
+                    Map<String, String> featureMapString = new HashMap<>(featureMapToString);
+                    FeatureV6.putVideoStringFeatures("r", rankInfo, featureMapString);
+                    featureMapString.put("r@vid", "r_vid_" + vid);
+                    FeatureV6.putProfileVideoCrossStringFeature(currentMs, userProfile, historyVideoMap, featureMapString);
+                    FeatureV6.putUserNetworkSeqFeature(featureMapString, userNetworkSeqFeature, videoBaseInfoMap);
+                    item.featureMapString = featureMapString;
                     return 1;
                 });
                 futures.add(future);
@@ -422,48 +458,4 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         }
         return newScore;
     }
-
-    private Map<String, Double> findSimCateScore(String headCate2, int length) {
-        if (StringUtils.isBlank(headCate2)) {
-            return new HashMap<>();
-        }
-
-        String redisKey = String.format("alg_recsys_good_cate_pair_list:%s", headCate2);
-        String cate2Value = redisTemplate.opsForValue().get(redisKey);
-        if (StringUtils.isEmpty(cate2Value)) {
-            return new HashMap<>();
-        }
-
-        return this.parsePair(cate2Value, length);
-    }
-
-    private Map<String, Double> parsePair(String value, int length) {
-        if (StringUtils.isBlank(value)) {
-            return new HashMap<>();
-        }
-
-        String[] split = value.split("\t");
-        if (split.length != 2) {
-            return new HashMap<>();
-        }
-
-        String[] valueList = split[0].trim().split(",");
-        String[] scoreList = split[1].trim().split(",");
-        if (valueList.length != scoreList.length) {
-            return new HashMap<>();
-        }
-
-        int minLength = Math.min(length, valueList.length);
-        Map<String, Double> resultMap = new HashMap<>();
-        for (int i = 0; i < minLength; i++) {
-            resultMap.put(valueList[i].trim(), Double.parseDouble(scoreList[i].trim()));
-        }
-
-        return resultMap;
-    }
-
-    private String findVideoMergeCate2(Map<String, Map<String, Map<String, String>>> featureOriginVideo, String vid) {
-        Map<String, String> videoInfo = featureOriginVideo.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
-        return videoInfo.get("merge_second_level_cate");
-    }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -53,4 +53,8 @@ public class RecallParam {
     private String unionId;
 
     private Map<String, String> userSocialRecallInfo;
+
+    private Map<String, String> userNetworkSeqFeature;
+
+    private Map<Long, Map<String, String>> userNetworkSeqVideoInfoMap;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -146,6 +146,13 @@ public class RecallService implements ApplicationContextAware {
             strategies.add(strategyMap.get(UserDeconstructionKeywordsRecallStrategy.class.getSimpleName()));
         }
 
+        boolean isHit568Exp = experimentService.judgeHitExp(param.getAppType(), param.getRootSessionId(), abExpCodes, "568");
+        if (isHit568Exp) {
+            strategies.add(strategyMap.get(YearShareCate1RecallStrategy.class.getSimpleName()));
+            strategies.add(strategyMap.get(YearShareCate2RecallStrategy.class.getSimpleName()));
+            strategies.add(strategyMap.get(YearReturnCate2RecallStrategy.class.getSimpleName()));
+        }
+
         // 命中用户黑名单不走流量池
         // 命中安全测试风险地域不走流量池
         if (!param.isRiskUser() && !param.isTestingRiskRegion()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/YearReturnCate2RecallStrategy.java

@@ -0,0 +1,185 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Slf4j
+@Component
+public class YearReturnCate2RecallStrategy implements RecallStrategy {
+
+    @Autowired
+    @Qualifier("redisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private FilterService filterService;
+
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+
+    public static final String PUSH_FROM = "recall_user_year_return_cate2";
+    public static final String redisKeyPrefix = "return_1_cate2_recall";
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+
+        List<Video> videosResult = new ArrayList<>();
+        try {
+
+            if (MapUtils.isEmpty(param.getUserNetworkSeqVideoInfoMap())) {
+                return videosResult;
+            }
+
+            List<Pair<Long, String>> userNetworkVideoCate2 = this.parseUserActionVideoAndCate2(param.getUserNetworkSeqFeature(), param.getUserNetworkSeqVideoInfoMap());
+            if (CollectionUtils.isEmpty(userNetworkVideoCate2)) {
+                return videosResult;
+            }
+            int limit = Math.min(userNetworkVideoCate2.size(), 3);
+            List<String> lastTopNCate = userNetworkVideoCate2.subList(0, limit).stream()
+                    .map(Pair::getValue)
+                    .distinct()
+                    .collect(Collectors.toList());
+
+            List<String> freqTopNCate = userNetworkVideoCate2.stream()
+                    .map(Pair::getValue)
+                    .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())).entrySet()
+                    .stream()
+                    .sorted(Map.Entry.<String, Long>comparingByValue().reversed())
+                    .limit(limit)
+                    .map(Map.Entry::getKey)
+                    .collect(Collectors.toList());
+
+
+            List<String> allCate2 = Stream.of(lastTopNCate, freqTopNCate)
+                    .flatMap(Collection::stream)
+                    .distinct()
+                    .filter(StringUtils::isNotBlank)
+                    .collect(Collectors.toList());
+
+            List<String> keys = this.getRedisKey(allCate2);
+            List<String> values = redisTemplate.opsForValue().multiGet(keys);
+            Pair<List<Long>, Map<Long, Double>> pair = parseVidAndScore(param.getVideoId(), values);
+            fillVideoResult(param, pair, videosResult);
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+
+        return videosResult;
+    }
+
+    private List<Pair<Long, String>> parseUserActionVideoAndCate2(Map<String, String> userNetworkSeqFeature, Map<Long, Map<String, String>> userNetworkSeqVideoInfoMap) {
+        List<Pair<Long, String>> result = new ArrayList<>();
+        List<String> actVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_v_s");
+        List<String> actTypeSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_t_s");
+        if (actVidSeq.size() != actTypeSeq.size()) {
+            return new ArrayList<>();
+        }
+
+        for (int i = 0; i < actVidSeq.size(); i++) {
+            long videoIdL = Long.parseLong(actVidSeq.get(i));
+            String type = actTypeSeq.get(i);
+            if (!"click".equals(type)) {
+                continue;
+            }
+
+            Map<String, String> videoBaseInfo = userNetworkSeqVideoInfoMap.getOrDefault(videoIdL, new HashMap<>());
+            String mergeCate2 = videoBaseInfo.get("merge_second_level_cate");
+            if (StringUtils.isBlank(mergeCate2)) {
+                continue;
+            }
+            result.add(Pair.of(videoIdL, mergeCate2));
+        }
+        return result;
+    }
+
+    private List<String> getRedisKey(List<String> cate2List) {
+        List<String> keys = new ArrayList<>();
+        for (String cate2 : cate2List) {
+            keys.add(String.format("%s:%s:%s", redisKeyPrefix, cate2, "ros"));
+        }
+        return keys;
+    }
+
+    private Pair<List<Long>, Map<Long, Double>> parseVidAndScore(Long headVid, List<String> values) {
+        Map<Long, Double> scoresMap = new HashMap<>();
+        if (null != values && !values.isEmpty()) {
+            for (String value : values) {
+                if (null != value && !value.isEmpty()) {
+                    String[] cells = value.split("\t");
+                    if (2 == cells.length) {
+                        List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                        List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                        if (!ids.isEmpty() && ids.size() == scores.size()) {
+                            for (int i = 0; i < ids.size(); ++i) {
+                                long vid = ids.get(i);
+                                double score = scores.get(i);
+                                if (vid == headVid) {
+                                    continue;
+                                }
+                                double oldScore = scoresMap.getOrDefault(vid, 0.0);
+                                scoresMap.put(vid, Math.max(oldScore, score));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        List<Long> idsList = new ArrayList<>();
+        if (!scoresMap.isEmpty()) {
+            List<Pair<Long, Double>> list = new ArrayList<>();
+            for (Map.Entry<Long, Double> entry : scoresMap.entrySet()) {
+                list.add(MutablePair.of(entry.getKey(), entry.getValue()));
+            }
+            list.sort(Comparator.comparingDouble(o -> -o.getRight()));
+            for (Pair<Long, Double> pair : list) {
+                idsList.add(pair.getLeft());
+            }
+        }
+        return new MutablePair<>(idsList, scoresMap);
+    }
+
+    private void fillVideoResult(RecallParam param, Pair<List<Long>, Map<Long, Double>> pair, List<Video> videosResult) {
+        if (null != pair) {
+            List<Long> ids = pair.getLeft();
+            Map<Long, Double> scoresMap = pair.getRight();
+            if (null != ids && null != scoresMap && !ids.isEmpty()) {
+                FilterParam filterParam = FilterParamFactory.create(param, ids);
+                FilterResult filterResult = filterService.filter(filterParam);
+                if (null != filterResult && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                    filterResult.getVideoIds().forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setRovScore(scoresMap.getOrDefault(vid, 0D));
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
+                }
+            }
+        }
+    }
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FROM;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/YearShareCate1RecallStrategy.java

@@ -0,0 +1,176 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Slf4j
+@Component
+public class YearShareCate1RecallStrategy implements RecallStrategy {
+
+    @Autowired
+    @Qualifier("redisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private FilterService filterService;
+
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+
+    public static final String PUSH_FROM = "recall_user_year_share_cate1";
+    public static final String redisKeyPrefix = "merge_cate_recall:cate1";
+
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+
+        List<Video> videosResult = new ArrayList<>();
+        try {
+
+            if (MapUtils.isEmpty(param.getUserNetworkSeqVideoInfoMap())) {
+                return videosResult;
+            }
+
+            List<Pair<Long, String>> userNetworkVideoCate1 = this.parseUserActionVideoAndCate1(param.getUserNetworkSeqFeature(), param.getUserNetworkSeqVideoInfoMap());
+            if (CollectionUtils.isEmpty(userNetworkVideoCate1)) {
+                return videosResult;
+            }
+            int limit = Math.min(userNetworkVideoCate1.size(), 3);
+            List<String> lastTopNCate = userNetworkVideoCate1.subList(0, limit).stream()
+                    .map(Pair::getValue)
+                    .distinct()
+                    .collect(Collectors.toList());
+
+            List<String> freqTopNCate = userNetworkVideoCate1.stream()
+                    .map(Pair::getValue)
+                    .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())).entrySet()
+                    .stream()
+                    .sorted(Map.Entry.<String, Long>comparingByValue().reversed())
+                    .limit(limit)
+                    .map(Map.Entry::getKey)
+                    .collect(Collectors.toList());
+
+
+            List<String> allCate1 = Stream.of(lastTopNCate, freqTopNCate)
+                    .flatMap(Collection::stream)
+                    .distinct()
+                    .filter(StringUtils::isNotBlank)
+                    .collect(Collectors.toList());
+
+            List<String> keys = this.getRedisKey(allCate1);
+            List<String> values = redisTemplate.opsForValue().multiGet(keys);
+            List<Long> ids = recall(param.getVideoId(), values);
+
+            FilterParam filterParam = FilterParamFactory.create(param, ids);
+            FilterResult filterResult = filterService.filter(filterParam);
+            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                List<Long> filterIds = filterResult.getVideoIds();
+                int n = filterIds.size();
+                for (int i = 0; i < n; i++) {
+                    Video video = new Video();
+                    video.setVideoId(filterIds.get(i));
+                    video.setRovScore(n - i);
+                    video.setPushFrom(pushFrom());
+                    videosResult.add(video);
+                }
+            }
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+
+        return videosResult;
+    }
+
+    private List<Pair<Long, String>> parseUserActionVideoAndCate1(Map<String, String> userNetworkSeqFeature, Map<Long, Map<String, String>> userNetworkSeqVideoInfoMap) {
+        List<Pair<Long, String>> result = new ArrayList<>();
+        List<String> actVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_v_s");
+        List<String> actTypeSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_t_s");
+        if (actVidSeq.size() != actTypeSeq.size()) {
+            return new ArrayList<>();
+        }
+
+        for (int i = 0; i < actVidSeq.size(); i++) {
+            long videoIdL = Long.parseLong(actVidSeq.get(i));
+            String type = actTypeSeq.get(i);
+            if (!"share".equals(type)) {
+                continue;
+            }
+
+            Map<String, String> videoBaseInfo = userNetworkSeqVideoInfoMap.getOrDefault(videoIdL, new HashMap<>());
+            String mergeCate1 = videoBaseInfo.get("merge_first_level_cate");
+            if (StringUtils.isBlank(mergeCate1)) {
+                continue;
+            }
+            result.add(Pair.of(videoIdL, mergeCate1));
+        }
+        return result;
+    }
+
+    private List<String> getRedisKey(List<String> cate1List) {
+        List<String> keys = new ArrayList<>();
+        for (String cate1 : cate1List) {
+            keys.add(String.format("%s:%s", redisKeyPrefix, cate1));
+        }
+        return keys;
+    }
+
+    private List<Long> recall(Long headVid, List<String> values) {
+        List<Long> vidList = new ArrayList<>();
+        if (null != values && !values.isEmpty()) {
+            Set<Long> hits = new HashSet<>();
+            hits.add(headVid);
+            List<org.apache.commons.math3.util.Pair<Long, Double>> list = new ArrayList<>();
+            for (String value : values) {
+                if (null != value && !value.isEmpty()) {
+                    String[] cells = value.split("\t");
+                    if (2 == cells.length) {
+                        List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                        List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                        if (!ids.isEmpty() && ids.size() == scores.size()) {
+                            for (int i = 0; i < ids.size(); ++i) {
+                                long id = ids.get(i);
+                                double score = scores.get(i);
+                                if (hits.contains(id)) {
+                                    continue;
+                                }
+                                hits.add(id);
+                                list.add(org.apache.commons.math3.util.Pair.create(id, score));
+                            }
+                        }
+                    }
+                }
+            }
+            if (!list.isEmpty()) {
+                list.sort(Comparator.comparingDouble(o -> -o.getSecond()));
+                for (org.apache.commons.math3.util.Pair<Long, Double> pair : list) {
+                    vidList.add(pair.getFirst());
+                }
+            }
+        }
+        return vidList;
+    }
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FROM;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/YearShareCate2RecallStrategy.java

@@ -0,0 +1,175 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Slf4j
+@Component
+public class YearShareCate2RecallStrategy implements RecallStrategy {
+
+    @Autowired
+    @Qualifier("redisTemplate")
+    private RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private FilterService filterService;
+
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+
+    public static final String PUSH_FROM = "recall_user_year_share_cate2";
+    public static final String redisKeyPrefix = "merge_cate_recall:cate2";
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+
+        List<Video> videosResult = new ArrayList<>();
+        try {
+
+            if (MapUtils.isEmpty(param.getUserNetworkSeqVideoInfoMap())) {
+                return videosResult;
+            }
+
+            List<Pair<Long, String>> userNetworkVideoCate2 = this.parseUserActionVideoAndCate2(param.getUserNetworkSeqFeature(), param.getUserNetworkSeqVideoInfoMap());
+            if (CollectionUtils.isEmpty(userNetworkVideoCate2)) {
+                return videosResult;
+            }
+            int limit = Math.min(userNetworkVideoCate2.size(), 3);
+            List<String> lastTopNCate = userNetworkVideoCate2.subList(0, limit).stream()
+                    .map(Pair::getValue)
+                    .distinct()
+                    .collect(Collectors.toList());
+
+            List<String> freqTopNCate = userNetworkVideoCate2.stream()
+                    .map(Pair::getValue)
+                    .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())).entrySet()
+                    .stream()
+                    .sorted(Map.Entry.<String, Long>comparingByValue().reversed())
+                    .limit(limit)
+                    .map(Map.Entry::getKey)
+                    .collect(Collectors.toList());
+
+
+            List<String> allCate2 = Stream.of(lastTopNCate, freqTopNCate)
+                    .flatMap(Collection::stream)
+                    .distinct()
+                    .filter(StringUtils::isNotBlank)
+                    .collect(Collectors.toList());
+
+            List<String> keys = this.getRedisKey(allCate2);
+            List<String> values = redisTemplate.opsForValue().multiGet(keys);
+            List<Long> ids = recall(param.getVideoId(), values);
+
+            FilterParam filterParam = FilterParamFactory.create(param, ids);
+            FilterResult filterResult = filterService.filter(filterParam);
+            if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                List<Long> filterIds = filterResult.getVideoIds();
+                int n = filterIds.size();
+                for (int i = 0; i < n; i++) {
+                    Video video = new Video();
+                    video.setVideoId(filterIds.get(i));
+                    video.setRovScore(n - i);
+                    video.setPushFrom(pushFrom());
+                    videosResult.add(video);
+                }
+            }
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+
+        return videosResult;
+    }
+
+    private List<Pair<Long, String>> parseUserActionVideoAndCate2(Map<String, String> userNetworkSeqFeature, Map<Long, Map<String, String>> userNetworkSeqVideoInfoMap) {
+        List<Pair<Long, String>> result = new ArrayList<>();
+        List<String> actVidSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_v_s");
+        List<String> actTypeSeq = FeatureUtils.extractVidsFromUserNetworkSeqFeature(userNetworkSeqFeature, "a_t_s");
+        if (actVidSeq.size() != actTypeSeq.size()) {
+            return new ArrayList<>();
+        }
+
+        for (int i = 0; i < actVidSeq.size(); i++) {
+            long videoIdL = Long.parseLong(actVidSeq.get(i));
+            String type = actTypeSeq.get(i);
+            if (!"share".equals(type)) {
+                continue;
+            }
+
+            Map<String, String> videoBaseInfo = userNetworkSeqVideoInfoMap.getOrDefault(videoIdL, new HashMap<>());
+            String mergeCate2 = videoBaseInfo.get("merge_second_level_cate");
+            if (StringUtils.isBlank(mergeCate2)) {
+                continue;
+            }
+            result.add(Pair.of(videoIdL, mergeCate2));
+        }
+        return result;
+    }
+
+    private List<String> getRedisKey(List<String> cate2List) {
+        List<String> keys = new ArrayList<>();
+        for (String cate2 : cate2List) {
+            keys.add(String.format("%s:%s", redisKeyPrefix, cate2));
+        }
+        return keys;
+    }
+
+    private List<Long> recall(Long headVid, List<String> values) {
+        List<Long> vidList = new ArrayList<>();
+        if (null != values && !values.isEmpty()) {
+            Set<Long> hits = new HashSet<>();
+            hits.add(headVid);
+            List<org.apache.commons.math3.util.Pair<Long, Double>> list = new ArrayList<>();
+            for (String value : values) {
+                if (null != value && !value.isEmpty()) {
+                    String[] cells = value.split("\t");
+                    if (2 == cells.length) {
+                        List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                        List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                        if (!ids.isEmpty() && ids.size() == scores.size()) {
+                            for (int i = 0; i < ids.size(); ++i) {
+                                long id = ids.get(i);
+                                double score = scores.get(i);
+                                if (hits.contains(id)) {
+                                    continue;
+                                }
+                                hits.add(id);
+                                list.add(org.apache.commons.math3.util.Pair.create(id, score));
+                            }
+                        }
+                    }
+                }
+            }
+            if (!list.isEmpty()) {
+                list.sort(Comparator.comparingDouble(o -> -o.getSecond()));
+                for (org.apache.commons.math3.util.Pair<Long, Double> pair : list) {
+                    vidList.add(pair.getFirst());
+                }
+            }
+        }
+        return vidList;
+    }
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FROM;
+    }
+}