ADD: Share Deep and width recall

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -145,6 +145,12 @@ public class RecallService implements ApplicationContextAware {
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
                     break;
+                case "60150":
+                    strategies.add(strategyMap.get(SimHotVideoRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(ReturnVideoRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(ShareWidthRecallStrategy.class.getSimpleName()));
+                    strategies.add(strategyMap.get(ShareDeepRecallStrategy.class.getSimpleName()));
+                    break;
                 case "60117": // 567
                 case "60118": // 568
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/ShareDeepRecallStrategy.java

@@ -0,0 +1,72 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+/**
+ * @author sunxy
+ */
+@Component
+public class ShareDeepRecallStrategy implements RecallStrategy {
+
+    @Value("${ShareDeepRecallStrategyRecallNumber:500}")
+    private String shareDeepRecallStrategyRecallNumber;
+
+    public static final String PUSH_FORM = "share_deep_recall_strategy";
+
+    @Autowired
+    private RegionFilterService filterService;
+
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        Map<String, String> param4Model = new HashMap<>(1);
+        param4Model.put("limit", shareDeepRecallStrategyRecallNumber);
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_score_config_share_deep.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
+        Map<Long, Double> videoMap = new LinkedHashMap<>();
+        List<Pair<Long, Double>> result = results.get(0);
+        for (int i = 1; i < results.size(); ++i) {
+            result.addAll(results.get(i));
+        }
+        for (Pair<Long, Double> v : result) {
+            videoMap.put(v.getLeft(), v.getRight());
+        }
+        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        List<Video> videosResult = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().forEach(vid -> {
+                Video video = new Video();
+                video.setVideoId(vid);
+                video.setAbCode(param.getAbCode());
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            });
+        }
+        return videosResult;
+    }
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/ShareWidthRecallStrategy.java

@@ -0,0 +1,72 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+/**
+ * @author sunxy
+ */
+@Component
+public class ShareWidthRecallStrategy implements RecallStrategy {
+
+    @Value("${ShareDeepRecallStrategyRecallNumber:500}")
+    private String shareDeepRecallStrategyRecallNumber;
+
+    public static final String PUSH_FORM = "share_width_recall_strategy";
+
+    @Autowired
+    private RegionFilterService filterService;
+
+
+    @Override
+    public List<Video> recall(RecallParam param) {
+        Map<String, String> param4Model = new HashMap<>(1);
+        param4Model.put("limit", shareDeepRecallStrategyRecallNumber);
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_score_config_share_width.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
+        Map<Long, Double> videoMap = new LinkedHashMap<>();
+        List<Pair<Long, Double>> result = results.get(0);
+        for (int i = 1; i < results.size(); ++i) {
+            result.addAll(results.get(i));
+        }
+        for (Pair<Long, Double> v : result) {
+            videoMap.put(v.getLeft(), v.getRight());
+        }
+        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        List<Video> videosResult = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().forEach(vid -> {
+                Video video = new Video();
+                video.setVideoId(vid);
+                video.setAbCode(param.getAbCode());
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            });
+        }
+        return videosResult;
+    }
+
+    @Override
+    public String pushFrom() {
+        return PUSH_FORM;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score/ScorerUtils.java

@@ -44,6 +44,8 @@ public final class ScorerUtils {
         ScorerUtils.init4Recall("feeds_score_config_bless.conf");
         ScorerUtils.init4Recall("feeds_recall_config_tomson.conf");
         ScorerUtils.init4Recall("feeds_recall_config_tomson_v2.conf");
+        ScorerUtils.init4Recall("feeds_score_config_share_width.conf");
+        ScorerUtils.init4Recall("feeds_score_config_share_deep.conf");
     }
 
     private ScorerUtils() {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/model4recall/Model4Recall2List.java

@@ -0,0 +1,52 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.model4recall;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class Model4Recall2List extends AbstractModel {
+    private static final Logger LOGGER = LoggerFactory.getLogger(Model4Recall2List.class);
+    public List<Pair<Long, Double>> recallList;
+
+    public Model4Recall2List() {
+        //配置不同环境的hdfs conf
+        this.recallList = new ArrayList<>();
+    }
+
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            for (String item : items) {
+                String[] itemArr = item.split(",");
+                if (itemArr.length != 2) {
+                    continue;
+                }
+                String id = itemArr[0].trim();
+                String score = itemArr[1].trim();
+                try {
+                    Pair<Long, Double> pair = Pair.of(Long.valueOf(id), Double.valueOf(score));
+                    this.recallList.add(pair);
+                } catch (Exception e) {
+                    LOGGER.error(String.format("something is wrong with parse pair %s %s: ", id, score), e);
+                }
+            }
+        }
+        input.close();
+        in.close();
+        return true;
+    }
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/ShareDeepRecallScore.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.score.ScorerConfigInfo;
+import com.tzld.piaoquan.recommend.server.service.score4recall.AbstractScorer4Recall;
+import com.tzld.piaoquan.recommend.server.service.score4recall.model4recall.Model4Recall2List;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+
+public class ShareDeepRecallScore extends AbstractScorer4Recall {
+
+    public ShareDeepRecallScore(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(Model4Recall2List.class);
+    }
+
+    @Override
+    public List<Pair<Long, Double>> recall(Map<String, String> params) {
+        if (params == null || params.isEmpty()) {
+            return new ArrayList<>();
+        }
+        Model4Recall2List model = (Model4Recall2List) this.getModel();
+        if (model == null || model.recallList == null) {
+            return new ArrayList<>();
+        }
+        int limit = Integer.parseInt(params.getOrDefault("limit", "500"));
+        return model.recallList.stream().limit(limit).collect(Collectors.toList());
+    }
+
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/score4recall/strategy/ShareWidthRecallScore.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.recommend.server.service.score4recall.strategy;
+
+import com.tzld.piaoquan.recommend.server.service.score.ScorerConfigInfo;
+import com.tzld.piaoquan.recommend.server.service.score4recall.AbstractScorer4Recall;
+import com.tzld.piaoquan.recommend.server.service.score4recall.model4recall.Model4Recall2List;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+
+public class ShareWidthRecallScore extends AbstractScorer4Recall {
+
+    public ShareWidthRecallScore(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(Model4Recall2List.class);
+    }
+
+    @Override
+    public List<Pair<Long, Double>> recall(Map<String, String> params) {
+        if (params == null || params.isEmpty()) {
+            return new ArrayList<>();
+        }
+        Model4Recall2List model = (Model4Recall2List) this.getModel();
+        if (model == null || model.recallList == null) {
+            return new ArrayList<>();
+        }
+        int limit = Integer.parseInt(params.getOrDefault("limit", "500"));
+        return model.recallList.stream().limit(limit).collect(Collectors.toList());
+    }
+
+
+}

recommend-server-service/src/main/resources/feeds_score_config_share_deep.conf

@@ -0,0 +1,7 @@
+scorer-config = {
+  festival-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score4recall.strategy.ShareDeepRecallScore"
+    scorer-priority = 100
+    model-path = "alg_recall_file/videos_share_deep.txt"
+  }
+}

recommend-server-service/src/main/resources/feeds_score_config_share_width.conf

@@ -0,0 +1,7 @@
+scorer-config = {
+  festival-score-config = {
+    scorer-name = "com.tzld.piaoquan.recommend.server.service.score4recall.strategy.ShareWidthRecallScore"
+    scorer-priority = 100
+    model-path = "alg_recall_file/videos_share_width.txt"
+  }
+}