Merge branch 'feature/add_priori_province_recall' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV564.java

@@ -70,7 +70,11 @@ public class RankStrategy4RegionMergeModelV564 extends RankStrategy4RegionMergeM
         //-------------------city rovn------------------
         RecallUtils.extractRecall(mergeWeight.getOrDefault("cityRov", 5.0).intValue(), param, CityRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
         //-------------------priori province rovn------------------
-        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 5.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 3.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        //-------------------priori province str------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceStr", 1.0).intValue(), param, PrioriProvinceStrRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        //-------------------priori province ros------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRos", 1.0).intValue(), param, PrioriProvinceRosRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
 
         //-------------------排-------------------
         //-------------------序-------------------

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV565.java

@@ -24,7 +24,6 @@ import org.springframework.stereotype.Service;
 import java.util.*;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 @Service
 @Slf4j
@@ -70,7 +69,11 @@ public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeM
         //-------------------city rovn------------------
         RecallUtils.extractRecall(mergeWeight.getOrDefault("cityRov", 5.0).intValue(), param, CityRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
         //-------------------priori province rovn------------------
-        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 5.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 3.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        //-------------------priori province str------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceStr", 1.0).intValue(), param, PrioriProvinceStrRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        //-------------------priori province ros------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRos", 1.0).intValue(), param, PrioriProvinceRosRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
 
         //-------------------排-------------------
         //-------------------序-------------------
@@ -137,7 +140,7 @@ public class RankStrategy4RegionMergeModelV565 extends RankStrategy4RegionMergeM
 
             double fmRov = fmRovCoefficientBefore * strPlusCalibrationCoefficient;
             item.getScoresMap().put("fmRov", fmRov);
-            item.getScoresMap().put("strPlusCalibrationCoefficient", fmRov);
+            item.getScoresMap().put("strPlusCalibrationCoefficient", strPlusCalibrationCoefficient);
             double hasReturnRovScore = Double.parseDouble(vid2MapFeature.getOrDefault(item.getVideoId() + "", new HashMap<>()).getOrDefault("rov", "0"));
             item.getScoresMap().put("hasReturnRovScore", hasReturnRovScore);
             double norXGBScore = item.getScoresMap().getOrDefault("NorXGBScore", 0d);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV566.java

@@ -70,7 +70,11 @@ public class RankStrategy4RegionMergeModelV566 extends RankStrategy4RegionMergeM
         //-------------------city rovn------------------
         RecallUtils.extractRecall(mergeWeight.getOrDefault("cityRov", 5.0).intValue(), param, CityRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
         //-------------------priori province rovn------------------
-        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 5.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRov", 3.0).intValue(), param, PrioriProvinceRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        //-------------------priori province str------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceStr", 1.0).intValue(), param, PrioriProvinceStrRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
+        //-------------------priori province ros------------------
+        RecallUtils.extractRecall(mergeWeight.getOrDefault("prioriProvinceRos", 1.0).intValue(), param, PrioriProvinceRosRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
         //-------------------channel layer rovn------------------
         RecallUtils.extractRecall(mergeWeight.getOrDefault("channelLayerRov", 5.0).intValue(), param, ChannelLayerRovnRecallStrategy.PUSH_FROM, setVideo, rovRecallRank);
         //-------------------channel layer head rovn------------------

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -119,6 +119,8 @@ public class RecallService implements ApplicationContextAware {
         strategies.add(strategyMap.get(HeadCate2RovRecallStrategy.class.getSimpleName()));
         strategies.add(strategyMap.get(CityRovnRecallStrategy.class.getSimpleName()));
         strategies.add(strategyMap.get(PrioriProvinceRovnRecallStrategy.class.getSimpleName()));
+        strategies.add(strategyMap.get(PrioriProvinceStrRecallStrategy.class.getSimpleName()));
+        strategies.add(strategyMap.get(PrioriProvinceRosRecallStrategy.class.getSimpleName()));
         Set<String> abExpCodes = param.getAbExpCodes();
         if (CollectionUtils.isNotEmpty(abExpCodes) && abExpCodes.contains("568")) {
             strategies.add(strategyMap.get(PremiumROVRecallStrategy.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/PrioriProvinceRosRecallStrategy.java

@@ -0,0 +1,102 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class PrioriProvinceRosRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String PUSH_FROM = "priori_province_ros";
+    public static final String redisKeyPrefix = "priori_province_recall";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FROM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            String province = param.getProvince().replaceAll("省$", "");
+            String redisKey = String.format("%s:%s:%s", redisKeyPrefix, province, "ros");
+            String redisValue = redisTemplate.opsForValue().get(redisKey);
+            if (null == redisValue || redisValue.isEmpty()) {
+                return videosResult;
+            }
+            Pair<List<Long>, Map<Long, Double>> pair = parsePair(redisValue, param.getVideoId(), 50);
+            fillVideoResult(param, pair, videosResult);
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private Pair<List<Long>, Map<Long, Double>> parsePair(String data, long headVid, int size) {
+        List<Long> idsList = new ArrayList<>();
+        Map<Long, Double> scoresMap = new HashMap<>();
+        if (!StringUtils.isBlank(data)) {
+            String[] cells = data.split("\t");
+            if (2 == cells.length) {
+                List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty() && ids.size() == scores.size()) {
+                    int minSize = Math.min(size, ids.size());
+                    for (int i = 0; i < minSize; ++i) {
+                        long id = ids.get(i);
+                        double score = scores.get(i);
+                        if (headVid != id && !scoresMap.containsKey(id)) {
+                            idsList.add(id);
+                            scoresMap.put(id, score);
+                        }
+                    }
+                }
+            }
+        }
+        return new MutablePair<>(idsList, scoresMap);
+    }
+
+    private void fillVideoResult(RecallParam param, Pair<List<Long>, Map<Long, Double>> pair, List<Video> videosResult) {
+        if (null != pair) {
+            List<Long> ids = pair.getLeft();
+            Map<Long, Double> scoresMap = pair.getRight();
+            if (null != ids && null != scoresMap && !ids.isEmpty()) {
+                FilterParam filterParam = FilterParamFactory.create(param, ids);
+                FilterResult filterResult = filterService.filter(filterParam);
+                if (null != filterResult && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                    filterResult.getVideoIds().forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setRovScore(scoresMap.getOrDefault(vid, 0D));
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
+                }
+            }
+        }
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/PrioriProvinceRovnRecallStrategy.java

@@ -31,7 +31,7 @@ public class PrioriProvinceRovnRecallStrategy implements RecallStrategy {
     public RedisTemplate<String, String> redisTemplate;
 
     public static final String PUSH_FROM = "priori_province_rovn";
-    public static final String redisKeyPrefix = "priori_province_rovn_recall";
+    public static final String redisKeyPrefix = "priori_province_recall";
 
     @Override
     public String pushFrom() {
@@ -43,7 +43,7 @@ public class PrioriProvinceRovnRecallStrategy implements RecallStrategy {
         List<Video> videosResult = new ArrayList<>();
         try {
             String province = param.getProvince().replaceAll("省$", "");
-            String redisKey = String.format("%s:%s", redisKeyPrefix, province);
+            String redisKey = String.format("%s:%s:%s", redisKeyPrefix, province, "rovn");
             String redisValue = redisTemplate.opsForValue().get(redisKey);
             if (null == redisValue || redisValue.isEmpty()) {
                 return videosResult;

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/PrioriProvinceStrRecallStrategy.java

@@ -0,0 +1,102 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class PrioriProvinceStrRecallStrategy implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private FilterService filterService;
+    @Autowired
+    @Qualifier("redisTemplate")
+    public RedisTemplate<String, String> redisTemplate;
+
+    public static final String PUSH_FROM = "priori_province_str";
+    public static final String redisKeyPrefix = "priori_province_recall";
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FROM;
+    }
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        List<Video> videosResult = new ArrayList<>();
+        try {
+            String province = param.getProvince().replaceAll("省$", "");
+            String redisKey = String.format("%s:%s:%s", redisKeyPrefix, province, "str");
+            String redisValue = redisTemplate.opsForValue().get(redisKey);
+            if (null == redisValue || redisValue.isEmpty()) {
+                return videosResult;
+            }
+            Pair<List<Long>, Map<Long, Double>> pair = parsePair(redisValue, param.getVideoId(), 50);
+            fillVideoResult(param, pair, videosResult);
+        } catch (Exception e) {
+            log.error("recall is wrong in {}, error={}", CLASS_NAME, e);
+        }
+        return videosResult;
+    }
+
+    private Pair<List<Long>, Map<Long, Double>> parsePair(String data, long headVid, int size) {
+        List<Long> idsList = new ArrayList<>();
+        Map<Long, Double> scoresMap = new HashMap<>();
+        if (!StringUtils.isBlank(data)) {
+            String[] cells = data.split("\t");
+            if (2 == cells.length) {
+                List<Long> ids = Arrays.stream(cells[0].split(",")).map(Long::valueOf).collect(Collectors.toList());
+                List<Double> scores = Arrays.stream(cells[1].split(",")).map(Double::valueOf).collect(Collectors.toList());
+                if (!ids.isEmpty() && ids.size() == scores.size()) {
+                    int minSize = Math.min(size, ids.size());
+                    for (int i = 0; i < minSize; ++i) {
+                        long id = ids.get(i);
+                        double score = scores.get(i);
+                        if (headVid != id && !scoresMap.containsKey(id)) {
+                            idsList.add(id);
+                            scoresMap.put(id, score);
+                        }
+                    }
+                }
+            }
+        }
+        return new MutablePair<>(idsList, scoresMap);
+    }
+
+    private void fillVideoResult(RecallParam param, Pair<List<Long>, Map<Long, Double>> pair, List<Video> videosResult) {
+        if (null != pair) {
+            List<Long> ids = pair.getLeft();
+            Map<Long, Double> scoresMap = pair.getRight();
+            if (null != ids && null != scoresMap && !ids.isEmpty()) {
+                FilterParam filterParam = FilterParamFactory.create(param, ids);
+                FilterResult filterResult = filterService.filter(filterParam);
+                if (null != filterResult && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+                    filterResult.getVideoIds().forEach(vid -> {
+                        Video video = new Video();
+                        video.setVideoId(vid);
+                        video.setRovScore(scoresMap.getOrDefault(vid, 0D));
+                        video.setPushFrom(pushFrom());
+                        videosResult.add(video);
+                    });
+                }
+            }
+        }
+    }
+}