相关推荐

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/common/base/RankItem.java

@@ -38,6 +38,9 @@ public class RankItem implements Comparable<RankItem> {
     // 记录召回信息
     private String queue;
 
+    // title
+    private String title;
+
     // 多样性过滤因子
     // 排序因子
     private Map<String, Double> rankItemCategories = Maps.newHashMap();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/model/RecommendParam.java

@@ -54,5 +54,8 @@ public class RecommendParam {
 
     private String rootSessionId;
     private List<UserSRBO> userRTShareList;
+    private int recommendType;
+    private Long requestVideoId;
+    private String currentPageSource;
 }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -239,6 +239,12 @@ public class RecommendService {
                             .count();
                 }
                 map.put("featureTableSize", String.valueOf(featureTableSize));
+                if (2 == param.getRecommendType()) {
+                    Map<String, String> extend = new HashMap<>();
+                    extend.put("reqVideo", String.valueOf(request.getRequestVideoId()));
+                    extend.put("currentPage", request.getCurrentPageSource());
+                    map.put("extend", JSONUtils.toJson(extend));
+                }
 
                 return map;
 
@@ -378,6 +384,9 @@ public class RecommendService {
         param.setPageNum(request.getPageNum());
         param.setRootSessionId(request.getRootSessionId());
         param.setUserRTShareList(getUserRTShareList(request.getShareListList()));
+        param.setRecommendType(recommendType);
+        param.setRequestVideoId(request.getRequestVideoId());
+        param.setCurrentPageSource(request.getCurrentPageSource());
         return param;
     }
 
@@ -460,6 +469,9 @@ public class RecommendService {
         recallParam.setUserShareDepth(param.getUserShareDepth());
         recallParam.setChannelName(param.getChannelName());
         recallParam.setRootSessionId(param.getRootSessionId());
+        recallParam.setRecommendType(param.getRecommendType());
+        recallParam.setRequestVideoId(param.getRequestVideoId());
+        recallParam.setCurrentPageSource(param.getCurrentPageSource());
         return recallParam;
     }
 
@@ -488,6 +500,9 @@ public class RecommendService {
         rankParam.setPageNum(param.getPageNum());
         rankParam.setChannelName(param.getChannelName());
         rankParam.setRootSessionId(param.getRootSessionId());
+        rankParam.setRecommendType(param.getRecommendType());
+        rankParam.setRequestVideoId(param.getRequestVideoId());
+        rankParam.setCurrentPageSource(param.getCurrentPageSource());
         return rankParam;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankParam.java

@@ -43,4 +43,7 @@ public class RankParam {
 
     private String rootSessionId;
     private List<UserSRBO> userRTShareList;
+    private int recommendType;
+    private Long requestVideoId;
+    private String currentPageSource;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -18,6 +18,7 @@ public class RankRouter {
 
     private LinkedHashMap<String, RankService> strategyMap;
 
+    private static final String relevantRank = "relevant";
     private static final Map<String, Class<? extends RankService>> STRATEGY_CLASSES = new HashMap<>();
 
     static {
@@ -30,6 +31,7 @@ public class RankRouter {
         STRATEGY_CLASSES.put("567", RankStrategy4RegionMergeModelV567.class);
         STRATEGY_CLASSES.put("569", RankStrategy4RegionMergeModelV569.class);
         STRATEGY_CLASSES.put("568", RankStrategy4RegionMergeModelV568.class);
+        STRATEGY_CLASSES.put(relevantRank, RankStrategy4RelevantModelV1.class);
     }
 
     @Autowired
@@ -45,6 +47,10 @@ public class RankRouter {
     }
 
     public RankResult rank(RankParam param) {
+        if (2 == param.getRecommendType() && strategyMap.containsKey(relevantRank)) {
+            return strategyMap.get(relevantRank).rank(param);
+        }
+
         Set<String> abExpCodes = param.getAbExpCodes();
         if (CollectionUtils.isNotEmpty(abExpCodes)) {
             for (Map.Entry<String, RankService> entry : strategyMap.entrySet()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankService.java

@@ -12,7 +12,6 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.math.NumberUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
 
@@ -38,6 +37,12 @@ public abstract class RankService {
             return null;
         }
 
+        if (2 == param.getRecommendType()) {
+            tagDuplicateVideos(param);
+            List<Video> rovRecallRank = mergeAndRankRovRecall(param);
+            return new RankResult(rovRecallRank);
+        }
+
         if (param.isSpecialRecommend()) {
             Optional<RecallResult.RecallData> data = param.getRecallResult().getData().stream()
                     .filter(d -> d.getPushFrom().equals(SpecialRecallStrategy.PUSH_FROM))
@@ -245,7 +250,7 @@ public abstract class RankService {
         }
     }
 
-    public abstract RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank,List<Video> douHotFlowPoolRank);
+    public abstract RankResult mergeAndSort(RankParam param, List<Video> rovRecallRank, List<Video> flowPoolRank, List<Video> douHotFlowPoolRank);
 
     private boolean matchSpecialApp(int appId) {
         Set<Integer> notSpecialApp = new HashSet<>(Arrays.asList(0, 4, 5));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RelevantModelV1.java

@@ -0,0 +1,174 @@
+package com.tzld.piaoquan.recommend.server.service.rank.strategy;
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
+import com.tzld.piaoquan.recommend.server.common.base.RankItem;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
+import com.tzld.piaoquan.recommend.server.service.rank.RankParam;
+import com.tzld.piaoquan.recommend.server.service.rank.tansform.FeatureV6;
+import com.tzld.piaoquan.recommend.server.service.recall.strategy.HotReturnUvRecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.FeatureBucketUtils;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
+import com.tzld.piaoquan.recommend.server.util.RecallUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+@Service
+@Slf4j
+public class RankStrategy4RelevantModelV1 extends RankStrategy4RegionMergeModelBasic {
+    @ApolloJsonValue("${relevant.params.v1:}")
+    private Map<String, Double> apolloParams;
+
+    @Autowired
+    private FeatureService featureService;
+
+    @Override
+    public List<Video> mergeAndRankRovRecall(RankParam param) {
+        Map<String, Double> apolloParams = this.apolloParams != null ? this.apolloParams : new HashMap<>(0);
+
+        // ------------------- 召回 -------------------
+        Set<Long> setVideo = new HashSet<>();
+        List<Video> rovRecallRank = new ArrayList<>();
+        // return uv
+        RecallUtils.extractRecall(apolloParams.getOrDefault("returnUv", 100d).intValue(), param, HotReturnUvRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+
+        // ------------------- 排序 -------------------
+        Map<String, String> rtFeatureDumpsMap = dumpsRtFeature(param.getUserRTShareList());
+
+        // 1. 批量获取特征
+        List<String> vids = CommonCollectionUtils.toListDistinct(rovRecallRank, v -> String.valueOf(v.getVideoId()));
+        String requestId = String.valueOf(param.getRequestVideoId());
+        Map<String, Map<String, Map<String, String>>> videoBaseInfoMap = featureService.getVideoBaseInfo(requestId, vids);
+        Map<String, String> requestVideoInfo = videoBaseInfoMap.getOrDefault(requestId, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+        String requestTitle = FeatureUtils.extractContent(requestVideoInfo.get("title"));
+        if (requestTitle.isEmpty()) {
+            return new ArrayList<>();
+        }
+
+        // 2. 特征处理
+        int batchSize = apolloParams.getOrDefault("batchSize", 10d).intValue();
+        long timeout = apolloParams.getOrDefault("timeout", 1000d).longValue();
+        List<RankItem> rankItems = CommonCollectionUtils.toList(rovRecallRank, RankItem::new);
+        parallelGetVideoFeature(requestVideoInfo, videoBaseInfoMap, rankItems, batchSize, timeout);
+        Map<String, Map<String, String>> vid2MapFeature = this.getVideoRedisFeature(vids, "feature_video_relevant:");
+
+        // 3. 排序公式特征
+        double titleThreshold = apolloParams.getOrDefault("titleThreshold", 0.9);
+        double titleWeight = apolloParams.getOrDefault("titleWeight", 0.5);
+        double kwWeight = apolloParams.getOrDefault("kwWeight", 0.2);
+        double cate1Weight = apolloParams.getOrDefault("cate1Weight", 0.1);
+        double cate2Weight = apolloParams.getOrDefault("cate2Weight", 0.1);
+        double rovnWeight = apolloParams.getOrDefault("rovnWeight", 0.1);
+        List<Video> result = new ArrayList<>();
+        for (RankItem item : rankItems) {
+            String title = item.getTitle();
+            if (null != title && title.equals(requestTitle)) {
+                continue;
+            }
+
+            double score;
+            Map<String, Float> featureMap = item.getFeatureMap() != null ? item.getFeatureMap() : new HashMap<>(0);
+            double titleSim = featureMap.getOrDefault("sim@title", 0f);
+            double kwSim = featureMap.getOrDefault("sim@keywords", 0f);
+            double cate1Sim = featureMap.getOrDefault("sim@merge_first_level_cate", 0f);
+            double cate2Sim = featureMap.getOrDefault("sim@merge_second_level_cate", 0f);
+            double rovn24h = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rovn", "0"));
+            if (titleSim > titleThreshold) {
+                continue;
+            }
+            item.getScoresMap().put("titleSim", titleSim);
+            item.getScoresMap().put("kwSim", kwSim);
+            item.getScoresMap().put("cate1Sim", cate1Sim);
+            item.getScoresMap().put("cate2Sim", cate2Sim);
+            item.getScoresMap().put("rovn", rovn24h);
+
+            score = titleWeight * titleSim
+                    + kwWeight * (kwSim + 0.05)
+                    + cate1Weight * (cate1Sim + 0.05)
+                    + cate2Weight * (cate2Sim + 0.05)
+                    + rovnWeight * rovn24h * 5;
+
+            Video video = item.getVideo();
+            video.setScore(score);
+            video.setSortScore(score);
+            video.setScoresMap(item.getScoresMap());
+            if (MapUtils.isNotEmpty(videoBaseInfoMap) && MapUtils.isNotEmpty(videoBaseInfoMap.get(item.getVideoId() + ""))) {
+                video.getMetaFeatureMap().putAll(videoBaseInfoMap.get(item.getVideoId() + ""));
+            }
+            if (MapUtils.isNotEmpty(requestVideoInfo)) {
+                video.getMetaFeatureMap().put("req_video", requestVideoInfo);
+            }
+            if (null != rtFeatureDumpsMap && !rtFeatureDumpsMap.isEmpty()) {
+                video.getMetaFeatureMap().put("rt", rtFeatureDumpsMap);
+            }
+            result.add(video);
+        }
+        result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
+        return result;
+    }
+
+    private Map<String, Float> getVideoFeature(Map<String, String> requestInfo, Map<String, String> rankInfo) {
+        Map<String, Double> featMap = new HashMap<>();
+        FeatureV6.getRequestRankVideoCrossFeature(requestInfo, rankInfo, featMap);
+        return FeatureBucketUtils.noBucketFeature(featMap);
+    }
+
+    private void batchGetVideoFeature(Map<String, String> requestInfo,
+                                      Map<String, Map<String, Map<String, String>>> videoBaseInfoMap,
+                                      List<RankItem> rankItems) {
+        if (null == rankItems || rankItems.isEmpty()) {
+            return;
+        }
+        for (RankItem item : rankItems) {
+            String vid = item.getVideoId() + "";
+            Map<String, String> rankInfo = videoBaseInfoMap.getOrDefault(vid, new HashMap<>()).getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+            item.featureMap = getVideoFeature(requestInfo, rankInfo);
+            item.setTitle(FeatureUtils.extractContent(rankInfo.get("title")));
+        }
+    }
+
+    private void parallelGetVideoFeature(Map<String, String> requestInfo,
+                                         Map<String, Map<String, Map<String, String>>> videoBaseInfoMap,
+                                         List<RankItem> rankItems,
+                                         int batchSize,
+                                         long timeout) {
+        if (null == rankItems || rankItems.isEmpty()) {
+            return;
+        }
+        List<Future<Integer>> futures = new ArrayList<>();
+        LinkedList<List<RankItem>> batchList = CommonCollectionUtils.splitListToBatch(rankItems, batchSize);
+        for (List<RankItem> batch : batchList) {
+            Future<Integer> future = ThreadPoolFactory.defaultPool().submit(() -> {
+                batchGetVideoFeature(requestInfo, videoBaseInfoMap, batch);
+                return 1;
+            });
+            futures.add(future);
+        }
+
+        try {
+            for (Future<Integer> future : futures) {
+                future.get(timeout, TimeUnit.MILLISECONDS);
+            }
+        } catch (Exception e) {
+            log.error("get feature error", e);
+        }
+        // 超时后取消
+        for (Future<Integer> future : futures) {
+            try {
+                if (!future.isDone()) {
+                    future.cancel(true);
+                }
+            } catch (Exception e) {
+                log.error("cancel feature error", e);
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/tansform/FeatureV6.java

@@ -32,6 +32,7 @@ public class FeatureV6 {
     private static final List<String> videoCateAttrs = Arrays.asList(FeatureUtils.cate1Attr, FeatureUtils.cate2Attr, FeatureUtils.festive1Attr,
             FeatureUtils.channelAttr, FeatureUtils.sourceAttr, FeatureUtils.uidAttr, FeatureUtils.mergeCate1Attr, FeatureUtils.mergeCate2Attr);
     private static final List<String> videoSimAttrs = Arrays.asList("title", "cate2", "cate2_list", "keywords");
+    private static final List<String> reqVideoSimAttrs = Arrays.asList("title", "keywords", "merge_first_level_cate", "merge_second_level_cate");
     private static final List<String> hVideoSimAttrs = Arrays.asList("title");
     private static final List<String> cfList = Arrays.asList("share", "return");
     private static final List<String> userAttrList = Arrays.asList("province", "city", "model", "brand", "system", "user_channel", "user_level");
@@ -207,6 +208,10 @@ public class FeatureV6 {
         getTwoVideoCrossFeature("hr_sim", videoSimAttrs, headInfo, rankInfo, featMap);
     }
 
+    public static void getRequestRankVideoCrossFeature(Map<String, String> headInfo, Map<String, String> rankInfo, Map<String, Double> featMap) {
+        getTwoVideoCrossFeature("sim", reqVideoSimAttrs, headInfo, rankInfo, featMap);
+    }
+
     public static void getProfileVideoCrossFeature(long currentMs, UserShareReturnProfile profile, Map<String, String> rankVideo, Map<String, Map<String, String>> hVideoMap, Map<String, Double> featMap) {
         if (null == profile) {
             return;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -43,4 +43,7 @@ public class RecallParam {
 
     private String rootSessionId;
     private List<UserSRBO> userRTShareList;
+    private int recommendType;
+    private Long requestVideoId;
+    private String currentPageSource;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -84,6 +84,10 @@ public class RecallService implements ApplicationContextAware {
 
     private List<RecallStrategy> getRecallStrategy(RecallParam param) {
         List<RecallStrategy> strategies = new ArrayList<>();
+        if (2 == param.getRecommendType()) {
+            strategies.add(strategyMap.get(HotReturnUvRecallStrategy.class.getSimpleName()));
+            return strategies;
+        }
         if (param.isSpecialRecommend()) {
             strategies.add(strategyMap.get(SpecialRecallStrategy.class.getSimpleName()));
             return strategies;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/HotReturnUvRecallStrategy.java

@@ -0,0 +1,103 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class HotReturnUvRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String PUSH_FROM = "hot_return_uv";
+    public static final String redisKey = "hot_return_uv_recall:all";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FROM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            String redisValue = redisTemplate.opsForValue().get(redisKey);
+            if (null == redisValue || redisValue.isEmpty()) {
+                return videosResult;
+            }
+            Set<Long> invalidIds = new HashSet<>();
+            invalidIds.add(param.getVideoId());
+            invalidIds.add(param.getRequestVideoId());
+            Pair<List<Long>, Map<Long, Double>> pair = parsePair(redisValue, invalidIds, 200);
+            fillVideoResult(param, pair, videosResult);
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private Pair<List<Long>, Map<Long, Double>> parsePair(String data, Set<Long> invalidIds, int size) {
+        List<Long> idsList = new ArrayList<>();
+        Map<Long, Double> scoresMap = new HashMap<>();
+        if (!StringUtils.isBlank(data)) {
+            String[] cells = data.split("\t");
+            if (2 == cells.length) {
+                List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty() && ids.size() == scores.size()) {
+                    int minSize = Math.min(size, ids.size());
+                    for (int i = 0; i < minSize; ++i) {
+                        long id = ids.get(i);
+                        double score = scores.get(i);
+                        if (!invalidIds.contains(id) && !scoresMap.containsKey(id)) {
+                            idsList.add(id);
+                            scoresMap.put(id, score);
+                        }
+                    }
+                }
+            }
+        }
+        return new MutablePair<>(idsList, scoresMap);
+    }
+
+    private void fillVideoResult(RecallParam param, Pair<List<Long>, Map<Long, Double>> pair, List<Video> videosResult) {
+        if (null != pair) {
+            List<Long> ids = pair.getLeft();
+            Map<Long, Double> scoresMap = pair.getRight();
+            if (null != ids && null != scoresMap && !ids.isEmpty()) {
+                FilterParam filterParam = FilterParamFactory.create(param, ids);
+                FilterResult filterResult = filterService.filter(filterParam);
+                if (null != filterResult && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                    filterResult.getVideoIds().forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setRovScore(scoresMap.getOrDefault(vid, 0D));
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
+                }
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/CommonCollectionUtils.java

@@ -56,4 +56,22 @@ public class CommonCollectionUtils {
         }
         return col.contains(ele);
     }
+
+    public static <T> LinkedList<List<T>> splitListToBatch(List<T> TList, int batchSize) {
+        int index = 0;
+        List<T> subList = new ArrayList<>(batchSize);
+        LinkedList<List<T>> batchList = new LinkedList<>();
+        for (T t : TList) {
+            if (0 == index) {
+                batchList.add(subList);
+            }
+            subList.add(t);
+            index++;
+            if (index == batchSize) {
+                subList = new ArrayList<>(batchSize);
+                index = 0;
+            }
+        }
+        return batchList;
+    }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/FeatureUtils.java

@@ -145,4 +145,28 @@ public class FeatureUtils {
         }
         return "";
     }
+
+    public static String extractContent(String data) {
+        StringBuilder result = new StringBuilder();
+        if (null != data) {
+            for (char c : data.toCharArray()) {
+                if (isChinese(c) || isDigit(c) || isLetter(c)) {
+                    result.append(c);
+                }
+            }
+        }
+        return result.toString();
+    }
+
+    private static boolean isChinese(char c) {
+        return c >= 0x4E00 && c <= 0x9FA5;
+    }
+
+    private static boolean isDigit(char c) {
+        return c >= '0' && c <= '9';
+    }
+
+    private static boolean isLetter(char c) {
+        return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
+    }
 }