536 实验迭代 省份为空时不走地域召回

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/strategy/RankStrategy4RegionMergeModelV1.java

@@ -131,9 +131,9 @@ public class RankStrategy4RegionMergeModelV1 extends RankService {
 
         //-------------------地域相关召回 融合+去重-------------------
         List<Video> rovRecallRank = new ArrayList<>();
-        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1.PUSH_FORM);
+        List<Video> v1 = extractAndSort(param, RegionRealtimeRecallStrategyV1_default.PUSH_FORM);
         List<Video> v2 = extractAndSort(param, RegionRealtimeRecallStrategyV2.PUSH_FORM);
-        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3.PUSH_FORM);
+        List<Video> v3 = extractAndSort(param, RegionRealtimeRecallStrategyV3_default.PUSH_FORM);
         List<Video> v4 = extractAndSort(param, RegionRealtimeRecallStrategyV4.PUSH_FORM);
         this.duplicate(setVideo, v1);
         this.duplicate(setVideo, v2);

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -96,6 +96,12 @@ public class RecallService implements ApplicationContextAware {
         } else {
             switch (abCode) {
                 case "60121": // 536
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1_default.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3_default.class.getSimpleName()));
+                    strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV4.class.getSimpleName()));
+                    strategies.addAll(getRegionRecallStrategy(param));
+                    break;
                 case "60122": // 537
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV1.class.getSimpleName()));
                     strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
@@ -117,7 +123,7 @@ public class RecallService implements ApplicationContextAware {
                 default:
                     strategies.addAll(getRegionRecallStrategy(param));
             }
-            //2:通过“流量池标记”控制“流量池召回子策略”
+            //2:通过“流量池标记”控制“流量池召回子策略” 其中有9组会走EXPERIMENTAL_FLOW_SET_LEVEL 有1组会走EXPERIMENTAL_FLOW_SET_LEVEL_SCORE
             if (param.getFlowPoolAbtestGroup().equals(FlowPoolConstants.EXPERIMENTAL_FLOW_SET_LEVEL)) {
                 strategies.add(strategyMap.get(QuickFlowPoolWithLevelRecallStrategy.class.getSimpleName()));
                 strategies.add(strategyMap.get(FlowPoolWithLevelRecallStrategy.class.getSimpleName()));

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV1_default.java

@@ -0,0 +1,131 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author zhangbo
+ */
+@Slf4j
+@Component
+public class RegionRealtimeRecallStrategyV1_default implements RecallStrategy {
+    private final String CLASS_NAME = this.getClass().getSimpleName();
+    @Autowired
+    private RegionFilterService filterService;
+    @Override
+    public List<Video> recall(RecallParam param) {
+        long t0 = System.currentTimeMillis();
+        // 1 获取省份key 放入参数map中
+        String provinceCn = param.getProvince();
+        if (provinceCn == null){
+            provinceCn = "中国";
+            return new ArrayList<>();
+        }else{
+            provinceCn = provinceCn.replaceAll("省$", "");
+        }
+        final Set<String> provinceCns = new HashSet<>(Arrays.asList(
+                "北京", "天津", "河北", "山西", "内蒙古", "辽宁", "吉林",
+                "黑龙江", "上海", "江苏", "浙江", "安徽", "福建", "江西",
+                "山东", "河南", "湖北", "湖南", "广东", "广西", "海南",
+                "重庆", "四川", "贵州", "云南", "西藏", "陕西", "甘肃",
+                "青海", "宁夏", "新疆", "台湾", "香港", "澳门", "中国"
+        ));
+        if (!provinceCns.contains(provinceCn)){
+            return new ArrayList<>();
+        }
+        Map<String, String> param4Model = new HashMap<>(1);
+        param4Model.put("region_province", provinceCn);
+        // 2 通过model拿到召回list
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_region_v1.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
+        List<Pair<Long, Double>> result = results.get(0);
+        for (int i=1; i<results.size(); ++i){
+            result.addAll(results.get(i));
+        }
+        Map<Long, Double> videoMap = new LinkedHashMap<>();
+        for (Pair<Long, Double> v: result){
+            videoMap.put(v.getLeft(), v.getRight());
+        }
+        long t1 = System.currentTimeMillis();
+        // 3 召回内部过滤
+        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        long t2 = System.currentTimeMillis();
+        JSONObject obj = new JSONObject();
+        obj.put("name", this.CLASS_NAME);
+        obj.put("filter_time", t2-t1);
+        obj.put("recall_time", t1-t0);
+        obj.put("provinceCn", provinceCn);
+        obj.put("sizeOld", videoMap.size());
+        obj.put("mid", param.getMid());
+        obj.put("regionCode", param.getRegionCode());
+        obj.put("video_input", result.stream()
+                .map(pair -> String.valueOf(pair.getLeft()))
+                .collect(Collectors.joining(",")));
+        List<Video> videosResult = new ArrayList<>();
+        List<Long> videosResultId = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().forEach(vid -> {
+                Video video = new Video();
+                video.setVideoId(vid);
+                video.setAbCode(param.getAbCode());
+                video.setRovScore(videoMap.get(vid));
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+                videosResultId.add(vid);
+            });
+        }
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+//        obj.put("video_output", videosResult.stream()
+//                .map(v -> String.valueOf(v.getVideoId()))
+//                .collect(Collectors.joining(",")));
+        obj.put("video_filter", result.stream()
+                .filter(v -> !videosResultId.contains(v.getLeft()))
+                .map(v -> String.valueOf(v.getLeft()))
+                .collect(Collectors.joining(",")));
+        obj.put("sizeNew", videosResult.size());
+        log.info(obj.toString());
+        return videosResult;
+    }
+    public static final String PUSH_FORM = "recall_strategy_region_1h";
+    @Override
+    public String pushFrom(){
+        return PUSH_FORM;
+    }
+
+
+    public static List<List<Long>> groupKeys(Map<Long, Double> videoMap, int groupSize) {
+        List<List<Long>> result = new ArrayList<>();
+        List<Long> keys = new ArrayList<>(videoMap.keySet());
+
+        int size = keys.size();
+        for (int i = 0; i < size; i += groupSize) {
+            int endIndex = Math.min(i + groupSize, size);
+            List<Long> group = keys.subList(i, endIndex);
+            result.add(group);
+        }
+
+        return result;
+    }
+
+}

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/strategy/RegionRealtimeRecallStrategyV3_default.java

@@ -0,0 +1,87 @@
+package com.tzld.piaoquan.recommend.server.service.recall.strategy;
+
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.recommend.server.model.Video;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterParam;
+import com.tzld.piaoquan.recommend.server.service.filter.FilterResult;
+import com.tzld.piaoquan.recommend.server.service.filter.RegionFilterService;
+import com.tzld.piaoquan.recommend.server.service.recall.FilterParamFactory;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallParam;
+import com.tzld.piaoquan.recommend.server.service.recall.RecallStrategy;
+import com.tzld.piaoquan.recommend.server.service.score.ScorerUtils;
+import com.tzld.piaoquan.recommend.server.service.score4recall.ScorerPipeline4Recall;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+/**
+ * @author zhangbo
+ */
+@Slf4j
+@Component
+public class RegionRealtimeRecallStrategyV3_default implements RecallStrategy {
+    @Autowired
+    protected RegionFilterService filterService;
+    @Override
+    public List<Video> recall(RecallParam param) {
+        // 1 获取省份key 放入参数map中
+        String provinceCn = param.getProvince();
+        if (provinceCn == null){
+            provinceCn = "中国";
+            return new ArrayList<>();
+        }else{
+            provinceCn = provinceCn.replaceAll("省$", "");
+        }
+        final Set<String> provinceCns = new HashSet<>(Arrays.asList(
+                "北京", "天津", "河北", "山西", "内蒙古", "辽宁", "吉林",
+                "黑龙江", "上海", "江苏", "浙江", "安徽", "福建", "江西",
+                "山东", "河南", "湖北", "湖南", "广东", "广西", "海南",
+                "重庆", "四川", "贵州", "云南", "西藏", "陕西", "甘肃",
+                "青海", "宁夏", "新疆", "台湾", "香港", "澳门", "中国"
+        ));
+        if (!provinceCns.contains(provinceCn)){
+            return new ArrayList<>();
+        }
+        Map<String, String> param4Model = new HashMap<>(1);
+        param4Model.put("region_province", provinceCn);
+        // 2 通过model拿到召回list
+        ScorerPipeline4Recall pipeline = ScorerUtils.getScorerPipeline4Recall("feeds_recall_config_region_v1.conf");
+        List<List<Pair<Long, Double>>> results = pipeline.recall(param4Model);
+        List<Pair<Long, Double>> result = results.get(0);
+        for (int i=1; i<results.size(); ++i){
+            result.addAll(results.get(i));
+        }
+        Map<Long, Double> videoMap = new LinkedHashMap<>();
+        for (Pair<Long, Double> v: result){
+            videoMap.put(v.getLeft(), v.getRight());
+        }
+        FilterParam filterParam = FilterParamFactory.create(param, Lists.newArrayList(videoMap.keySet()));
+        filterParam.setForceTruncation(10000);
+        filterParam.setConcurrent(true);
+        filterParam.setNotUsePreView(false);
+        FilterResult filterResult = filterService.filter(filterParam);
+        List<Video> videosResult = new ArrayList<>();
+        if (filterResult != null && CollectionUtils.isNotEmpty(filterResult.getVideoIds())) {
+            filterResult.getVideoIds().forEach(vid -> {
+                Video video = new Video();
+                video.setVideoId(vid);
+                video.setAbCode(param.getAbCode());
+                video.setRovScore(videoMap.get(vid));
+                video.setPushFrom(pushFrom());
+                videosResult.add(video);
+            });
+        }
+        videosResult.sort(Comparator.comparingDouble(o -> -o.getRovScore()));
+        return videosResult;
+    }
+    public static final String PUSH_FORM = "recall_strategy_region_24h";
+    @Override
+    public String pushFrom(){
+        return PUSH_FORM;
+    }
+
+}