Merge branch 'feature_supplyExp' of algorithm/recommend-server into master

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -680,6 +680,7 @@ public class RecommendService {
         private String layerId;
         private int bucketNum;
         private List<Exp> exps;
+        private Map<String, String> groupRule;
 
         public String getLayerId() {
             return layerId;
@@ -704,6 +705,14 @@ public class RecommendService {
         public void setExps(List<Exp> exps) {
             this.exps = exps;
         }
+
+        public Map<String, String> getGroupRule() {
+            return groupRule;
+        }
+
+        public void setGroupRule(Map<String, String> groupRule) {
+            this.groupRule = groupRule;
+        }
     }
 
     public static class Exp {

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/AbstractFilterService.java

@@ -1,6 +1,6 @@
 package com.tzld.piaoquan.recommend.server.service.filter;
 
-import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.Lists;
 import com.tzld.piaoquan.recommend.server.common.ThreadPoolFactory;
@@ -12,6 +12,8 @@ import com.tzld.piaoquan.recommend.server.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -41,6 +43,15 @@ public abstract class AbstractFilterService {
     @Value("${newFilterAbExpCode:}")
     private String newFilterAbExpCode;
 
+    @ApolloJsonValue("${supply.exp.list:[6]}")
+    private Set<Integer> supplyExps;
+
+    @Value("${supply.exp.id:666}")
+    private int supplyExpId;
+
+    @ApolloJsonValue("${supply.exp.video.list:[]}")
+    private Set<Long> supplyExpVideos;
+
 
     private List<Long> viewFilterOld(FilterParam param) {
         // log.info("filterParam={}", JSONUtils.toJson(param));
@@ -48,30 +59,48 @@ public abstract class AbstractFilterService {
         List<Long> videoIds = filterWithRiskVideo(param.getRiskFilterFlag(),
                 param.getAppType(), param.getRegionCode(), param.getAppRegionFiltered(), param.getVideosWithRisk(),
                 param.getVideoIds(), param.getForceTruncation());
-        // log.info("filterByRiskVideos videoIds={}", JSONUtils.toJson(videoIds));
+
-//        JSONObject obj = new JSONObject();
+        // TODO 供给实验
-//        obj.put("zhangbotest", "");
+        Map<String, String> obj = new HashMap<>();
-//        obj.put("concurrent", param.concurrent);
+        obj.put("before", JSONUtils.toJson(videoIds));
-//        obj.put("notUsePreView", param.notUsePreView);
+
-//        obj.put("size1", videoIds.size());
+        videoIds = filterBySupplyExp(param.getAppType(), param.getExpIdMap(), videoIds);
+
+        obj.put("after", JSONUtils.toJson(videoIds));
+        log.info("supply filter {}", JSONUtils.toJson(obj));
+
         if (param.isNotUsePreView()) {
-            ;
+
         } else {
             videoIds = filterByPreViewed(param.getAppType(), param.getMid(), videoIds);
         }
-//        obj.put("size2", videoIds.size());
-        // log.info("filterByPreViewed videoIds={}", JSONUtils.toJson(videoIds));
         if (param.isConcurrent()) {
             videoIds = filterByViewedConcurrent(param.getAppType(), param.getMid(), param.getUid(), videoIds);
         } else {
             videoIds = filterByViewed(param.getAppType(), param.getMid(), param.getUid(), videoIds);
         }
-//        obj.put("size3", videoIds.size());
-//        log.info(obj.toString());
-        // log.info("filterByViewed videoIds={}", JSONUtils.toJson(videoIds));
         return videoIds;
     }
 
+    private List<Long> filterBySupplyExp(int appType, Map<String, String> expIdMap, List<Long> videoIds) {
+        if (!supplyExps.contains(appType)) {
+            return videoIds;
+        }
+        if (MapUtils.isEmpty(expIdMap)) {
+            return videoIds;
+        }
+        // A VS A+B 模式：供给实验的视频只在实验组出
+        if (supplyExpId == NumberUtils.toInt(expIdMap.get("supply"), -1)) {
+            // 命中实验，都可以出
+            return videoIds;
+        }
+        // 对照组，需要过滤掉视频
+        return videoIds.stream()
+                .filter(l -> !supplyExpVideos.contains(l))
+                .collect(Collectors.toList());
+
+    }
+
     private List<Long> filterByViewedConcurrent(int appType, String mid, String uid, List<Long> videoIds) {
         // TODO uid为空时，还需要过滤么？
         if (StringUtils.isBlank(mid)
@@ -79,11 +108,6 @@ public abstract class AbstractFilterService {
             return videoIds;
         }
 
-//        int chunkSize = 20;
-//        Collection<List<Long>> chunks = videoIds.stream()
-//                .collect(Collectors.groupingBy(it -> it / chunkSize))
-//                .values();
-
         int chunkSize = 20;
         List<List<Long>> chunks = new ArrayList<>();
         int size = videoIds.size();

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/filter/FilterParam.java

@@ -27,4 +27,7 @@ public class FilterParam {
 
     public boolean concurrent; // hardcode 临时解决过滤慢的问题
     public boolean notUsePreView;
+
+    // 层 - 实验
+    private Map<String, String> expIdMap;
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/FilterParamFactory.java

@@ -17,6 +17,7 @@ public class FilterParamFactory {
         filterParam.setMid(param.getMid());
         filterParam.setUid(param.getUid());
         filterParam.setFlowPoolMap(videoFlowPoolMap);
+        filterParam.setExpIdMap(param.getExpIdMap());
 
         // 风险过滤
         filterParam.setRiskFilterFlag(param.getRiskFilterFlag());
@@ -24,6 +25,7 @@ public class FilterParamFactory {
         filterParam.setAppRegionFiltered(param.getAppRegionFiltered());
         filterParam.setRegionCode(param.getRegionCode());
         filterParam.setAbExpCodes(param.getAbExpCodes());
+
         return filterParam;
     }
 
@@ -33,6 +35,8 @@ public class FilterParamFactory {
         filterParam.setAppType(param.getAppType());
         filterParam.setMid(param.getMid());
         filterParam.setUid(param.getUid());
+        filterParam.setExpIdMap(param.getExpIdMap());
+        
         // 风险过滤
         filterParam.setRiskFilterFlag(param.getRiskFilterFlag());
         filterParam.setVideosWithRisk(param.getVideosWithRisk());