|
|
@@ -0,0 +1,131 @@
|
|
|
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
|
|
|
+
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.google.common.collect.Lists;
|
|
|
+import com.tzld.piaoquan.recommend.server.model.Video;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
|
|
|
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.collections4.CollectionUtils;
|
|
|
+import org.apache.commons.lang3.tuple.Pair;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.stereotype.Component;
|
|
|
+
|
|
|
+import java.util.*;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+/**
|
|
|
+ * @author zhangbo
|
|
|
+ */
|
|
|
+@Slf4j
|
|
|
+@Component
|
|
|
+public class RegionRealtimeRecallStrategyV1_default implements RecallStrategy {
|
|
|
+ private final String CLASS_NAME = this.getClass().getSimpleName();
|
|
|
+ @Autowired
|
|
|
+ private RegionFilterService filterService;
|
|
|
+ @Override
|
|
|
+ public List<Video> recall(RecallParam param) {
|
|
|
+ long t0 = System.currentTimeMillis();
|
|
|
+ // 1 获取省份key 放入参数map中
|
|
|
+ String provinceCn = param.getProvince();
|
|
|
+ if (provinceCn == null){
|
|
|
+ // provinceCn = "中国";
|
|
|
+ return new ArrayList<>();
|
|
|
+ }else{
|
|
|
+ provinceCn = provinceCn.replaceAll("省$", "");
|
|
|
+ }
|
|
|
+ final Set<String> provinceCns = new HashSet<>(Arrays.asList(
|
|
|
+ "北京", "天津", "河北", "山西", "内蒙古", "辽宁", "吉林",
|
|
|
+ "黑龙江", "上海", "江苏", "浙江", "安徽", "福建", "江西",
|
|
|
+ "山东", "河南", "湖北", "湖南", "广东", "广西", "海南",
|
|
|
+ "重庆", "四川", "贵州", "云南", "西藏", "陕西", "甘肃",
|
|
|
+ "青海", "宁夏", "新疆", "台湾", "香港", "澳门", "中国"
|
|
|
+ ));
|
|
|
+ if (!provinceCns.contains(provinceCn)){
|
|
|
+ return new ArrayList<>();
|
|
|
+ }
|
|
|
+ Map<String, String> param4Model = new HashMap<>(1);
|
|
|
+ param4Model.put("region_province", provinceCn);
|
|
|
+ // 2 通过model拿到召回list
|
|
|
+ ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_region_v1.conf");
|
|
|
+ List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
|
|
|
+ List<Pair<Long, Double>> result = results.get(0);
|
|
|
+ for (int i=1; i<results.size(); ++i){
|
|
|
+ result.addAll(results.get(i));
|
|
|
+ }
|
|
|
+ Map<Long, Double> videoMap = new LinkedHashMap<>();
|
|
|
+ for (Pair<Long, Double> v: result){
|
|
|
+ videoMap.put(v.getLeft(), v.getRight());
|
|
|
+ }
|
|
|
+ long t1 = System.currentTimeMillis();
|
|
|
+ // 3 召回内部过滤
|
|
|
+ FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
|
|
|
+ filterParam.setForceTruncation(10000);
|
|
|
+ filterParam.setConcurrent(true);
|
|
|
+ filterParam.setNotUsePreView(false);
|
|
|
+ FilterResult filterResult = filterService.filter(filterParam);
|
|
|
+ long t2 = System.currentTimeMillis();
|
|
|
+ JSONObject obj = new JSONObject();
|
|
|
+ obj.put("name", this.CLASS_NAME);
|
|
|
+ obj.put("filter_time", t2-t1);
|
|
|
+ obj.put("recall_time", t1-t0);
|
|
|
+ obj.put("provinceCn", provinceCn);
|
|
|
+ obj.put("sizeOld", videoMap.size());
|
|
|
+ obj.put("mid", param.getMid());
|
|
|
+ obj.put("regionCode", param.getRegionCode());
|
|
|
+ obj.put("video_input", result.stream()
|
|
|
+ .map(pair -> String.valueOf(pair.getLeft()))
|
|
|
+ .collect(Collectors.joining(",")));
|
|
|
+ List<Video> videosResult = new ArrayList<>();
|
|
|
+ List<Long> videosResultId = new ArrayList<>();
|
|
|
+ if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
|
|
|
+ filterResult.getVideoIds().forEach(vid -> {
|
|
|
+ Video video = new Video();
|
|
|
+ video.setVideoId(vid);
|
|
|
+ video.setAbCode(param.getAbCode());
|
|
|
+ video.setRovScore(videoMap.get(vid));
|
|
|
+ video.setPushFrom(pushFrom());
|
|
|
+ videosResult.add(video);
|
|
|
+ videosResultId.add(vid);
|
|
|
+ });
|
|
|
+ }
|
|
|
+ videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
|
|
|
+// obj.put("video_output", videosResult.stream()
|
|
|
+// .map(v -> String.valueOf(v.getVideoId()))
|
|
|
+// .collect(Collectors.joining(",")));
|
|
|
+ obj.put("video_filter", result.stream()
|
|
|
+ .filter(v -> !videosResultId.contains(v.getLeft()))
|
|
|
+ .map(v -> String.valueOf(v.getLeft()))
|
|
|
+ .collect(Collectors.joining(",")));
|
|
|
+ obj.put("sizeNew", videosResult.size());
|
|
|
+ log.info(obj.toString());
|
|
|
+ return videosResult;
|
|
|
+ }
|
|
|
+ public static final String PUSH_FORM = "recall_strategy_region_1h";
|
|
|
+ @Override
|
|
|
+ public String pushFrom(){
|
|
|
+ return PUSH_FORM;
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ public static List<List<Long>> groupKeys(Map<Long, Double> videoMap, int groupSize) {
|
|
|
+ List<List<Long>> result = new ArrayList<>();
|
|
|
+ List<Long> keys = new ArrayList<>(videoMap.keySet());
|
|
|
+
|
|
|
+ int size = keys.size();
|
|
|
+ for (int i = 0; i < size; i += groupSize) {
|
|
|
+ int endIndex = Math.min(i + groupSize, size);
|
|
|
+ List<Long> group = keys.subList(i, endIndex);
|
|
|
+ result.add(group);
|
|
|
+ }
|
|
|
+
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
