提升外部流量裂变率

recommend-server-service/pom.xml

@@ -252,7 +252,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j-spark_2.12</artifactId>
-            <version>2.0.1</version>
+            <version>1.7.6</version>
             <exclusions>
                 <exclusion>
                     <artifactId>scala-library</artifactId>

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -416,6 +416,7 @@ public class RecommendService {
 
         recallParam.setHotSceneType(param.getHotSceneType());
         recallParam.setClientIp(param.getClientIp());
+        recallParam.setRootSourceId(param.getRootSourceId());
 
         return recallParam;
     }
@@ -440,6 +441,7 @@ public class RecommendService {
         rankParam.setHeadVid(param.getVideoId());
         rankParam.setHotSceneType(param.getHotSceneType());
         rankParam.setVersionCode(param.getVersionCode());
+        rankParam.setRootSourceId(param.getRootSourceId());
         return rankParam;
     }
 

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankParam.java

@@ -30,5 +30,6 @@ public class RankParam {
 
     private Integer categoryId;
     private Integer versionCode;
+    private String rootSourceId;
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV568.java

@@ -10,6 +10,7 @@ import com.tzld.piaoquan.recommend.server.service.rank.extractor.ExtractorUtils;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
 import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
 import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.math3.util.Pair;
@@ -77,18 +78,17 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
         rovRecallRank.addAll(sceneCFRosn);
         setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------省份ros指标实时召回------------------
-        List<Video> hourROSRecall = extractAndSort(param, RegionRealtimeRecallStrategyROS.PUSH_FORM);
-        hourROSRecall = hourROSRecall.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        hourROSRecall = hourROSRecall.subList(0, Math.min(mergeWeight.getOrDefault("hourROSRecall", 5.0).intValue(), hourROSRecall.size()));
-        rovRecallRank.addAll(hourROSRecall);
-        setVideo.addAll(hourROSRecall.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------长周期ros------------------
-        List<Video> LongTermROSRecall = extractAndSort(param, RegionRealtimeRecallStrategyV7LongTermV1.PUSH_FORM);
-        LongTermROSRecall = LongTermROSRecall.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        LongTermROSRecall = LongTermROSRecall.subList(0, Math.min(mergeWeight.getOrDefault("LongTermROSRecall", 1.0).intValue(), LongTermROSRecall.size()));
-        rovRecallRank.addAll(LongTermROSRecall);
-        setVideo.addAll(LongTermROSRecall.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        if (FeatureUtils.lowFissionChannel(param.getRootSourceId())) {
+            // head cate1 str
+            int headCate1StrN = mergeWeight.getOrDefault("headCate1StrN", 5.0).intValue();
+            addRecall(param, headCate1StrN, HeadCate1STRRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+            // head cate2 str
+            int headCate2StrN = mergeWeight.getOrDefault("headCate2StrN", 5.0).intValue();
+            addRecall(param, headCate2StrN, HeadCate2STRRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+            // province str
+            int provinceStrN = mergeWeight.getOrDefault("provinceStrN", 5.0).intValue();
+            addRecall(param, provinceStrN, ProvinceSTRRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+        }
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -357,4 +357,14 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         result.sort(Comparator.comparingDouble(o -> -o.getSortScore()));
         return result;
     }
+
+    private void addRecall(RankParam param, int recallNum, String recallName, Set<Long> setVideo, List<Video> rovRecallRank) {
+        if (recallNum > 0) {
+            List<Video> list = extractAndSort(param, recallName);
+            list = list.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+            list = list.subList(0, Math.min(recallNum, list.size()));
+            rovRecallRank.addAll(list);
+            setVideo.addAll(list.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+        }
+    }
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallParam.java

@@ -31,5 +31,6 @@ public class RecallParam {
     private String clientIp;
 
     private boolean riskUser;
+    private String rootSourceId;
 
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -5,6 +5,7 @@ import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -120,6 +121,11 @@ public class RecallService implements ApplicationContextAware {
             strategies.add(strategyMap.get(HeadProvinceCate1RecallStrategy.class.getSimpleName()));
             strategies.add(strategyMap.get(HeadProvinceCate2RecallStrategy.class.getSimpleName()));
         }
+        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568") && FeatureUtils.lowFissionChannel(param.getRootSourceId())) {
+            strategies.add(strategyMap.get(HeadCate1STRRecallStrategy.class.getSimpleName()));
+            strategies.add(strategyMap.get(HeadCate2STRRecallStrategy.class.getSimpleName()));
+            strategies.add(strategyMap.get(ProvinceSTRRecallStrategy.class.getSimpleName()));
+        }
 
         // 命中用户黑名单不走流量池
         if (!param.isRiskUser()) {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/HeadCate1STRRecallStrategy.java

@@ -0,0 +1,105 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class HeadCate1STRRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private FeatureService featureService;
+
+    public static final String PUSH_FORM = "recall_strategy_head_cate1_str";
+    public static final String redisKeyPrefix = "merge_cate_str_recall:cate1";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            if (null == param.getVideoId() || 0 == param.getVideoId()) {
+                return videosResult;
+            }
+            Long headVid = param.getVideoId();
+            Map<String, String> headVideoInfo = featureService.getHeadVideoInfo(String.valueOf(headVid));
+            if (null != headVideoInfo) {
+                String cate = headVideoInfo.getOrDefault("merge_first_level_cate", "").trim();
+                if (!cate.isEmpty() && !"unknown".equals(cate)) {
+                    String key = String.format("%s:%s", redisKeyPrefix, cate);
+                    String value = redisTemplate.opsForValue().get(key);
+                    if (null == value || value.isEmpty()) {
+                        return videosResult;
+                    }
+                    List<Long> vidList = parseVidList(headVid, value);
+                    fillVideoResult(param, vidList, videosResult);
+                }
+            }
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private List<Long> parseVidList(Long headVid, String data) {
+        List<Long> vidList = new ArrayList<>();
+        if (null != data && !data.isEmpty()) {
+            String[] pair = data.split("\t");
+            if (2 == pair.length) {
+                Set<Long> hit = new HashSet<>();
+                hit.add(headVid);
+                List<Long> ids = Arrays.stream(pair[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty()) {
+                    for (Long id : ids) {
+                        if (!hit.contains(id)) {
+                            hit.add(id);
+                            vidList.add(id);
+                        }
+                    }
+                }
+            }
+        }
+        return vidList;
+    }
+
+    private void fillVideoResult(RecallParam param, List<Long> vidList, List<Video> videosResult) {
+        FilterParam filterParam = FilterParamFactory.create(param, vidList);
+        FilterResult filterResult = filterService.filter(filterParam);
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            List<Long> filterIds = filterResult.getVideoIds();
+            int n = filterIds.size();
+            for (int i = 0; i < n; i++) {
+                Video video = new Video();
+                video.setVideoId(filterIds.get(i));
+                video.setRovScore(n - i);
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/HeadCate2STRRecallStrategy.java

@@ -0,0 +1,105 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.FeatureService;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class HeadCate2STRRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    @Autowired
+    private FeatureService featureService;
+
+    public static final String PUSH_FORM = "recall_strategy_head_cate2_str";
+    public static final String redisKeyPrefix = "merge_cate_str_recall:cate2";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            if (null == param.getVideoId() || 0 == param.getVideoId()) {
+                return videosResult;
+            }
+            Long headVid = param.getVideoId();
+            Map<String, String> headVideoInfo = featureService.getHeadVideoInfo(String.valueOf(headVid));
+            if (null != headVideoInfo) {
+                String cate = headVideoInfo.getOrDefault("merge_second_level_cate", "").trim();
+                if (!cate.isEmpty() && !"unknown".equals(cate)) {
+                    String key = String.format("%s:%s", redisKeyPrefix, cate);
+                    String value = redisTemplate.opsForValue().get(key);
+                    if (null == value || value.isEmpty()) {
+                        return videosResult;
+                    }
+                    List<Long> vidList = parseVidList(headVid, value);
+                    fillVideoResult(param, vidList, videosResult);
+                }
+            }
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private List<Long> parseVidList(Long headVid, String data) {
+        List<Long> vidList = new ArrayList<>();
+        if (null != data && !data.isEmpty()) {
+            String[] pair = data.split("\t");
+            if (2 == pair.length) {
+                Set<Long> hit = new HashSet<>();
+                hit.add(headVid);
+                List<Long> ids = Arrays.stream(pair[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty()) {
+                    for (Long id : ids) {
+                        if (!hit.contains(id)) {
+                            hit.add(id);
+                            vidList.add(id);
+                        }
+                    }
+                }
+            }
+        }
+        return vidList;
+    }
+
+    private void fillVideoResult(RecallParam param, List<Long> vidList, List<Video> videosResult) {
+        FilterParam filterParam = FilterParamFactory.create(param, vidList);
+        FilterResult filterResult = filterService.filter(filterParam);
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            List<Long> filterIds = filterResult.getVideoIds();
+            int n = filterIds.size();
+            for (int i = 0; i < n; i++) {
+                Video video = new Video();
+                video.setVideoId(filterIds.get(i));
+                video.setRovScore(n - i);
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/ProvinceSTRRecallStrategy.java

@@ -0,0 +1,95 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class ProvinceSTRRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String PUSH_FORM = "recall_strategy_province_str";
+    public static final String redisKeyPrefix = "province_str_recall";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            String province = param.getProvince().replaceAll("省$", "");
+            if (province.isEmpty()) {
+                return videosResult;
+            }
+            String key = String.format("%s:%s", redisKeyPrefix, province);
+            String value = redisTemplate.opsForValue().get(key);
+            if (null == value || value.isEmpty()) {
+                return videosResult;
+            }
+            List<Long> vidList = parseVidList(param.getVideoId(), value);
+            fillVideoResult(param, vidList, videosResult);
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private List<Long> parseVidList(Long headVid, String data) {
+        List<Long> vidList = new ArrayList<>();
+        if (null != data && !data.isEmpty()) {
+            String[] pair = data.split("\t");
+            if (2 == pair.length) {
+                Set<Long> hit = new HashSet<>();
+                hit.add(headVid);
+                List<Long> ids = Arrays.stream(pair[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty()) {
+                    for (Long id : ids) {
+                        if (!hit.contains(id)) {
+                            hit.add(id);
+                            vidList.add(id);
+                        }
+                    }
+                }
+            }
+        }
+        return vidList;
+    }
+
+    private void fillVideoResult(RecallParam param, List<Long> vidList, List<Video> videosResult) {
+        FilterParam filterParam = FilterParamFactory.create(param, vidList);
+        FilterResult filterResult = filterService.filter(filterParam);
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            List<Long> filterIds = filterResult.getVideoIds();
+            int n = filterIds.size();
+            for (int i = 0; i < n; i++) {
+                Video video = new Video();
+                video.setVideoId(filterIds.get(i));
+                video.setRovScore(n - i);
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/FeatureUtils.java

@@ -17,9 +17,11 @@ public class FeatureUtils {
     private static final String goodMorningRegex = "(早安|早上好|早晨好|上午好)";
     private static final String goodAfternoonRegex = "(午安|中午好|下午好)";
     private static final String goodEveningRegex = "(晚安|晚上好)";
+    private static final String lowFissionChannelRegex = "(longArticles_|dyyjs_|daitou_tencentgzh|DaiTou_gh)";
     private static final Pattern goodMorningPattern = Pattern.compile(goodMorningRegex);
     private static final Pattern goodAfternoonPattern = Pattern.compile(goodAfternoonRegex);
     private static final Pattern goodEveningPattern = Pattern.compile(goodEveningRegex);
+    private static final Pattern lowFissionChannelPattern = Pattern.compile(lowFissionChannelRegex);
 
     static {
         cate1MAP.put("情感", 1);
@@ -146,4 +148,12 @@ public class FeatureUtils {
     public static boolean greaterThanZero(double value) {
         return value > 1E-8;
     }
+
+    public static boolean lowFissionChannel(String s) {
+        if (null != s && !s.isEmpty()) {
+            Matcher morning = lowFissionChannelPattern.matcher(s);
+            return morning.find();
+        }
+        return false;
+    }
 }