Merge branch 'feature/external_increase_fission_v3' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV568.java

@@ -20,6 +20,7 @@ import org.springframework.stereotype.Service;
 import java.util.*;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
 import java.util.stream.Collectors;
 
 @Service
@@ -60,34 +61,30 @@ public class RankStrategy4RegionMergeModelV568 extends RankStrategy4RegionMergeM
         v6 = v6.subList(0, Math.min(mergeWeight.getOrDefault("v6", 5.0).intValue(), v6.size()));
         rovRecallRank.addAll(v6);
         setVideo.addAll(v6.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------新地域召回------------------
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
-        v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
-        rovRecallRank.addAll(v1);
-        setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------scene cf rovn------------------
-        List<Video> sceneCFRovn = extractAndSort(param, SceneCFRovnRecallStrategy.PUSH_FORM);
-        sceneCFRovn = sceneCFRovn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        sceneCFRovn = sceneCFRovn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), sceneCFRovn.size()));
-        rovRecallRank.addAll(sceneCFRovn);
-        setVideo.addAll(sceneCFRovn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        //-------------------scene cf rosn------------------
-        List<Video> sceneCFRosn = extractAndSort(param, SceneCFRosnRecallStrategy.PUSH_FORM);
-        sceneCFRosn = sceneCFRosn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
-        sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
-        rovRecallRank.addAll(sceneCFRosn);
-        setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
-        if (FeatureUtils.lowFissionChannel(param.getRootSourceId()) && FeatureUtils.firstLevel(param.getUserShareDepth())) {
-            // head cate1 str
-            int headCate1StrN = mergeWeight.getOrDefault("headCate1StrN", 5.0).intValue();
-            addRecall(param, headCate1StrN, HeadCate1STRRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
-            // head cate2 str
-            int headCate2StrN = mergeWeight.getOrDefault("headCate2StrN", 5.0).intValue();
-            addRecall(param, headCate2StrN, HeadCate2STRRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
-            // province str
-            int provinceStrN = mergeWeight.getOrDefault("provinceStrN", 5.0).intValue();
-            addRecall(param, provinceStrN, ProvinceSTRRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+
+        Matcher matcher = FeatureUtils.getChannelMatcher(param.getRootSourceId());
+        if (null != matcher && matcher.find() && FeatureUtils.firstLevel(param.getUserShareDepth())) {
+            int channelROVN = mergeWeight.getOrDefault("channelROVN", 5.0).intValue();
+            addRecall(param, channelROVN, ChannelROVRecallStrategy.PUSH_FORM, setVideo, rovRecallRank);
+        } else {
+            //-------------------新地域召回------------------
+            List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+            v1 = v1.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+            v1 = v1.subList(0, Math.min(mergeWeight.getOrDefault("v1", 5.0).intValue(), v1.size()));
+            rovRecallRank.addAll(v1);
+            setVideo.addAll(v1.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+            //-------------------scene cf rovn------------------
+            List<Video> sceneCFRovn = extractAndSort(param, SceneCFRovnRecallStrategy.PUSH_FORM);
+            sceneCFRovn = sceneCFRovn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+            sceneCFRovn = sceneCFRovn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRovn", 5.0).intValue(), sceneCFRovn.size()));
+            rovRecallRank.addAll(sceneCFRovn);
+            setVideo.addAll(sceneCFRovn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
+            //-------------------scene cf rosn------------------
+            List<Video> sceneCFRosn = extractAndSort(param, SceneCFRosnRecallStrategy.PUSH_FORM);
+            sceneCFRosn = sceneCFRosn.stream().filter(r -> !setVideo.contains(r.getVideoId())).collect(Collectors.toList());
+            sceneCFRosn = sceneCFRosn.subList(0, Math.min(mergeWeight.getOrDefault("sceneCFRosn", 5.0).intValue(), sceneCFRosn.size()));
+            rovRecallRank.addAll(sceneCFRosn);
+            setVideo.addAll(sceneCFRosn.stream().map(Video::getVideoId).collect(Collectors.toSet()));
         }
 
         //-------------------排-------------------

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -5,7 +5,6 @@ import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
 import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
-import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -110,23 +109,14 @@ public class RecallService implements ApplicationContextAware {
         strategies.add(strategyMap.get(SceneCFRovnRecallStrategy.class.getSimpleName()));
         strategies.add(strategyMap.get(SceneCFRosnRecallStrategy.class.getSimpleName()));
         Set<String> abExpCodes = param.getAbExpCodes();
-        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568")) {
-            // strategies.remove(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
-            strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV7LongTermV1.class.getSimpleName()));
-            strategies.add(strategyMap.get(RegionRealtimeRecallStrategyROS.class.getSimpleName()));
-        }
         if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("567")) {
             strategies.add(strategyMap.get(UserCate1RecallStrategy.class.getSimpleName()));
             strategies.add(strategyMap.get(UserCate2RecallStrategy.class.getSimpleName()));
             strategies.add(strategyMap.get(HeadProvinceCate1RecallStrategy.class.getSimpleName()));
             strategies.add(strategyMap.get(HeadProvinceCate2RecallStrategy.class.getSimpleName()));
         }
-        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568")
-                && FeatureUtils.lowFissionChannel(param.getRootSourceId())
-                && FeatureUtils.firstLevel(param.getUserShareDepth())) {
-            strategies.add(strategyMap.get(HeadCate1STRRecallStrategy.class.getSimpleName()));
-            strategies.add(strategyMap.get(HeadCate2STRRecallStrategy.class.getSimpleName()));
-            strategies.add(strategyMap.get(ProvinceSTRRecallStrategy.class.getSimpleName()));
+        if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568")) {
+            strategies.add(strategyMap.get(ChannelROVRecallStrategy.class.getSimpleName()));
         }
 
         // 命中用户黑名单不走流量池

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/ChannelROVRecallStrategy.java

@@ -0,0 +1,134 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.util.FeatureUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class ChannelROVRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String channelGroupPrefix = "alg_recsys_user_channel_group";
+
+    public static final String PUSH_FORM = "recall_strategy_channel_rovn";
+    public static final String redisKeyPrefix = "channel_rovn_recall";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            if (!FeatureUtils.firstLevel(param.getUserShareDepth())) {
+                return videosResult;
+            }
+
+            String channelName = getChannelName(param.getRootSourceId());
+            if (channelName.isEmpty()) {
+                return videosResult;
+            }
+
+            String key = String.format("%s:%s", redisKeyPrefix, channelName);
+            String value = redisTemplate.opsForValue().get(key);
+            if (null == value || value.isEmpty()) {
+                return videosResult;
+            }
+            List<Long> vidList = parseVidList(param.getVideoId(), value);
+            fillVideoResult(param, vidList, videosResult);
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private List<Long> parseVidList(Long headVid, String data) {
+        List<Long> vidList = new ArrayList<>();
+        if (null != data && !data.isEmpty()) {
+            String[] pair = data.split("\t");
+            if (2 == pair.length) {
+                Set<Long> hit = new HashSet<>();
+                hit.add(headVid);
+                List<Long> ids = Arrays.stream(pair[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty()) {
+                    for (Long id : ids) {
+                        if (!hit.contains(id)) {
+                            hit.add(id);
+                            vidList.add(id);
+                        }
+                    }
+                }
+            }
+        }
+        return vidList;
+    }
+
+    private void fillVideoResult(RecallParam param, List<Long> vidList, List<Video> videosResult) {
+        FilterParam filterParam = FilterParamFactory.create(param, vidList);
+        FilterResult filterResult = filterService.filter(filterParam);
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            List<Long> filterIds = filterResult.getVideoIds();
+            int n = filterIds.size();
+            for (int i = 0; i < n; i++) {
+                Video video = new Video();
+                video.setVideoId(filterIds.get(i));
+                video.setRovScore(n - i);
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            }
+        }
+    }
+
+    private String getChannelName(String s) {
+        Matcher matcher = FeatureUtils.getChannelMatcher(s);
+        if (null != matcher && matcher.find()) {
+            switch (matcher.group(0)) {
+                case "longArticles_":
+                    return getChannelType(s);
+                case "dyyjs_":
+                    return "公众号代运营-即转";
+                case "touliu_tencent_":
+                    return "小程序投流";
+                case "touliu_tencentgzh_":
+                case "touliu_tencentGzhArticle_":
+                case "GzhTouLiu_Articles_gh":
+                    return "公众号投流";
+                default:
+                    return "";
+            }
+        }
+        return "";
+    }
+
+    private String getChannelType(String s) {
+        String key = String.format("%s:%s", channelGroupPrefix, s);
+        String value = redisTemplate.opsForValue().get(key);
+        if (null != value && value.equals("公众号买号")) {
+            return value;
+        }
+        return "公众号代运营-Daily";
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/FeatureUtils.java

@@ -17,11 +17,11 @@ public class FeatureUtils {
     private static final String goodMorningRegex = "(早安|早上好|早晨好|上午好)";
     private static final String goodAfternoonRegex = "(午安|中午好|下午好)";
     private static final String goodEveningRegex = "(晚安|晚上好)";
-    private static final String lowFissionChannelRegex = "(longArticles_|dyyjs_|daitou_tencentgzh|DaiTou_gh)";
+    private static final String channelRegex = "(longArticles_|dyyjs_|touliu_tencent_|touliu_tencentgzh_|touliu_tencentGzhArticle_|GzhTouLiu_Articles_gh)";
     private static final Pattern goodMorningPattern = Pattern.compile(goodMorningRegex);
     private static final Pattern goodAfternoonPattern = Pattern.compile(goodAfternoonRegex);
     private static final Pattern goodEveningPattern = Pattern.compile(goodEveningRegex);
-    private static final Pattern lowFissionChannelPattern = Pattern.compile(lowFissionChannelRegex);
+    private static final Pattern channelPattern = Pattern.compile(channelRegex);
 
     static {
         cate1MAP.put("情感", 1);
@@ -149,15 +149,14 @@ public class FeatureUtils {
         return value > 1E-8;
     }
 
-    public static boolean lowFissionChannel(String s) {
-        if (null != s && !s.isEmpty()) {
-            Matcher morning = lowFissionChannelPattern.matcher(s);
-            return morning.find();
-        }
-        return false;
-    }
-
     public static boolean firstLevel(Integer userShareDepth) {
         return null != userShareDepth && 0 == userShareDepth;
     }
+
+    public static Matcher getChannelMatcher(String s) {
+        if (null != s && !s.isEmpty()) {
+            return channelPattern.matcher(s);
+        }
+        return null;
+    }
 }