Merge branch 'master' into vlog_merge_refactor_smz

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/RecommendService.java

@@ -28,6 +28,7 @@ import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import com.tzld.piaoquan.recommend.server.util.ParserUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -66,6 +67,12 @@ public class RecommendService {
     @ApolloJsonValue("${city_code:[]}")
     private Set<String> cityCodes;
 
+    @ApolloJsonValue("${new.exp.list:[6]}")
+    private Set<Integer> newExpList;
+
+    @ApolloJsonValue("${new.exp,config.v2:[]}")
+    private List<ExpConfig> expConfigs;
+
     @Autowired
     private FlowPoolConfigService flowPoolConfigService;
     @Autowired
@@ -345,9 +352,35 @@ public class RecommendService {
             param.setMachineInfo(machineInfo);
         }
 
-        param.setExpIdMap(JSONUtils.fromJson(request.getNewExpGroup(), new TypeToken<Map<String, String>>() {
-        }, Collections.emptyMap()));
 
+        // TODO：hard code 为了快速做AB验证，应该由AB系统支持
+        if (newExpList.contains(param.getAppType())) {
+            Map<String, List<Layer>> layerMap = CommonCollectionUtils.toMap(expConfigs, c -> c.getAppType(),
+                    c -> c.getLayers());
+            if (MapUtils.isNotEmpty(layerMap)) {
+                Map<String, Integer> bucketMap = JSONUtils.fromJson(request.getNewExpGroup(),
+                        new TypeToken<Map<String, Integer>>() {
+                        }, Collections.emptyMap());
+                if (MapUtils.isNotEmpty(bucketMap)) {
+                    List<Layer> layers = layerMap.get(param.getAppType() + "");
+
+                    if (CollectionUtils.isNotEmpty(layers)) {
+                        Map<String, String> expIdMap = new HashMap<>();
+                        for (Layer layer : layers) {
+                            for (Exp exp : layer.getExps()) {
+                                if (bucketMap.containsKey(layer.getLayerId())
+                                        && exp.getRange().length == 2
+                                        && exp.getRange()[0] <= bucketMap.get(layer.getLayerId())
+                                        && exp.getRange()[1] >= bucketMap.get(layer.getLayerId())) {
+                                    expIdMap.put(layer.getLayerId(), exp.getExpId());
+                                }
+                            }
+                        }
+                        param.setExpIdMap(expIdMap);
+                    }
+                }
+            }
+        }
 
         return param;
     }
@@ -388,7 +421,27 @@ public class RecommendService {
     private RecallParam convertToRecallParam(RecommendParam param) {
         RecallParam recallParam = new RecallParam();
         recallParam.setAppType(param.getAppType());
+        // hard code 算法实验配置化之前，复用abcode做AB验证
+        // note 避免非实验产品被覆盖
         recallParam.setAbCode(param.getAbCode());
+        recallParam.setRuleKey(param.getRuleKey());
+        recallParam.setDataKey(param.getDataKey());
+        recallParam.setHRuleKey(param.getHRuleKey());
+        recallParam.setHDataKey(param.getHDataKey());
+        if (newExpList.contains(param.getAppType())) {
+            recallParam.setAbCode("");
+            if (MapUtils.isNotEmpty(param.getExpIdMap()) && param.getExpIdMap().containsKey("recall")) {
+                String expId = param.getExpIdMap().get("recall");
+                if (abExpCodeMap.containsKey(expId)) {
+                    recallParam.setAbCode(abExpCodeMap.get(expId).get("ab_code"));
+                    recallParam.setRuleKey(abExpCodeMap.get(expId).get("rule_key"));
+                    recallParam.setDataKey(abExpCodeMap.get(expId).get("data_key"));
+                    recallParam.setHDataKey(abExpCodeMap.get(expId).get("h_data_key"));
+                    recallParam.setHRuleKey(abExpCodeMap.get(expId).get("h_rule_key"));
+                }
+            }
+        }
+
         recallParam.setVideoId(param.getVideoId());
         recallParam.setFlowPoolAbtestGroup(param.getFlowPoolAbtestGroup());
 
@@ -399,13 +452,6 @@ public class RecommendService {
                 ? param.getCityCode()
                 : "";
         recallParam.setRegionCode(cityCodes.contains(cityCode) ? cityCode : provinceCode);
-
-        recallParam.setRuleKey(param.getRuleKey());
-        recallParam.setDataKey(param.getDataKey());
-        recallParam.setHRuleKey(param.getHRuleKey());
-        recallParam.setHDataKey(param.getHDataKey());
-
-
         recallParam.setMid(param.getMid());
         recallParam.setSize(param.getSize());
         recallParam.setUid(param.getUid());
@@ -427,11 +473,31 @@ public class RecommendService {
     private RankParam convertToRankParam(RecommendParam param, RecallResult recallResult) {
         RankParam rankParam = new RankParam();
         rankParam.setRecallResult(recallResult);
+        // hard code 算法实验配置化之前，复用abcode做AB验证
+        // note 避免非实验产品被覆盖
         rankParam.setAbCode(param.getAbCode());
+        rankParam.setRankKeyPrefix(param.getRankKeyPrefix());
+        if (newExpList.contains(param.getAppType())) {
+            rankParam.setAbCode("");
+//            if (MapUtils.isNotEmpty(param.getExpIdMap())
+//                    && param.getExpIdMap().containsKey("rank")) {
+//                String expId = param.getExpIdMap().get("rank");
+            // TODO hard code  MVP版本 为了快速验证，对rank强制染色，不做实验编排
+            if (MapUtils.isNotEmpty(param.getExpIdMap())
+                    && param.getExpIdMap().containsKey("recall")) {
+                String expId = param.getExpIdMap().get("recall");
+                if (abExpCodeMap.containsKey(expId)) {
+                    rankParam.setAbCode(abExpCodeMap.get(expId).get("ab_code"));
+                    rankParam.setRankKeyPrefix(StringUtils.isNotBlank(abExpCodeMap.get(expId).get("rank_key_prefix"))
+                            ? abExpCodeMap.get(expId).get("rank_key_prefix")
+                            : "rank:score1:");
+                }
+            }
+        }
+
         rankParam.setSize(param.getSize());
         rankParam.setFlowPoolP(param.getFlowPoolP());
         rankParam.setTopK(param.getTopK());
-        rankParam.setRankKeyPrefix(param.getRankKeyPrefix());
         rankParam.setAppType(param.getAppType());
         rankParam.setMid(param.getMid());
         rankParam.setProvince(param.getProvince());
@@ -535,4 +601,76 @@ public class RecommendService {
         consumer.accept(RegionHWithoutDupRecallStrategy.PUSH_FORM);
     }
 
+    public static class ExpConfig {
+        private String appType;
+        private List<Layer> layers;
+
+        public String getAppType() {
+            return appType;
+        }
+
+        public void setAppType(String appType) {
+            this.appType = appType;
+        }
+
+        public List<Layer> getLayers() {
+            return layers;
+        }
+
+        public void setLayers(List<Layer> layers) {
+            this.layers = layers;
+        }
+    }
+
+    public static class Layer {
+        private String layerId;
+        private int bucketNum;
+        private List<Exp> exps;
+
+        public String getLayerId() {
+            return layerId;
+        }
+
+        public void setLayerId(String layerId) {
+            this.layerId = layerId;
+        }
+
+        public int getBucketNum() {
+            return bucketNum;
+        }
+
+        public void setBucketNum(int bucketNum) {
+            this.bucketNum = bucketNum;
+        }
+
+        public List<Exp> getExps() {
+            return exps;
+        }
+
+        public void setExps(List<Exp> exps) {
+            this.exps = exps;
+        }
+    }
+
+    public static class Exp {
+        private String expId;
+        private int[] range;
+
+        public String getExpId() {
+            return expId;
+        }
+
+        public void setExpId(String expId) {
+            this.expId = expId;
+        }
+
+        public int[] getRange() {
+            return range;
+        }
+
+        public void setRange(int[] range) {
+            this.range = range;
+        }
+    }
+
 }

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/rank/RankRouter.java

@@ -1,6 +1,8 @@
 package com.tzld.piaoquan.recommend.server.service.rank;
 
+import com.aliyun.odps.utils.StringUtils;
 import com.tzld.piaoquan.recommend.server.service.rank.strategy.*;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -38,11 +40,12 @@ public class RankRouter {
 
 
     public RankResult rank(RankParam param) {
+        log.info("RankParam {}", JSONUtils.toJson(param));
         String abCode = param.getAbCode();
-        if (abCode == null) {
+        if (StringUtils.isBlank(abCode)) {
             return rankService.rank(param);
         }
-        switch (abCode){
+        switch (abCode) {
             case "60106":
                 return rankStrategy4Rankv2Model.rank(param);
             case "60101":

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/service/recall/RecallService.java

@@ -5,6 +5,7 @@ import com.tzld.piaoquan.recommend.server.common.enums.AppTypeEnum;
 import com.tzld.piaoquan.recommend.server.model.Video;
 import com.tzld.piaoquan.recommend.server.service.flowpool.FlowPoolConstants;
 import com.tzld.piaoquan.recommend.server.service.recall.strategy.*;
+import com.tzld.piaoquan.recommend.server.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeansException;
@@ -43,6 +44,7 @@ public class RecallService implements ApplicationContextAware {
     }
 
     public RecallResult recall(RecallParam param) {
+        log.info("RecallParam {}", JSONUtils.toJson(param));
         List<RecallStrategy> strategies = getRecallStrategy(param);
         // log.info("strategies {}", JSONUtils.toJson(CommonCollectionUtils.toList(strategies, o -> o.pushFrom())));
         CountDownLatch cdl = new CountDownLatch(strategies.size());
@@ -86,12 +88,12 @@ public class RecallService implements ApplicationContextAware {
         if (param.getAppType() == AppTypeEnum.LAO_HAO_KAN_VIDEO.getCode()
                 || param.getAppType() == AppTypeEnum.ZUI_JING_QI.getCode()) {
             strategies.addAll(getRegionRecallStrategy(param));
-        } else if (param.getAppType() == AppTypeEnum.PIAO_QUAN_MEIHAO_ZHUFU.getCode()){
+        } else if (param.getAppType() == AppTypeEnum.PIAO_QUAN_MEIHAO_ZHUFU.getCode()) {
             strategies.add(strategyMap.get(BlessRecallStrategy.class.getSimpleName()));
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV2.class.getSimpleName()));
             strategies.add(strategyMap.get(RegionRealtimeRecallStrategyV3.class.getSimpleName()));
             return strategies;
-        }else {
+        } else {
             switch (abCode) {
                 case "60120": // 576
                 case "60121": // 536

recommend-server-service/src/main/java/com/tzld/piaoquan/recommend/server/util/ProbabilityCalculator.java

@@ -1,5 +1,7 @@
 package com.tzld.piaoquan.recommend.server.util;
 
+import org.apache.commons.lang3.StringUtils;
+
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -31,11 +33,17 @@ public class ProbabilityCalculator {
     }
 
     public static double calculateCurrentProbability(String filterRate, String filterDate) {
+        if (StringUtils.isBlank(filterRate) || StringUtils.isBlank(filterDate)) {
+            return -1;
+        }
         // 解析日期区间
         String[] dateRanges = filterDate.split(",");
         List<DateRange> parsedDateRanges = new ArrayList<>();
         for (String range : dateRanges) {
             String[] dates = range.split("-");
+            if (dates.length != 2) {
+                continue;
+            }
             LocalDateTime start = LocalDateTime.parse(dates[0], DateTimeFormatter.ofPattern("yyyyMMddHH"));
             LocalDateTime end = LocalDateTime.parse(dates[1], DateTimeFormatter.ofPattern("yyyyMMddHH"));
             parsedDateRanges.add(new DateRange(start, end));
@@ -45,6 +53,9 @@ public class ProbabilityCalculator {
         String[] rateRanges = filterRate.split(",");
         List<RateRange> parsedRateRanges = new ArrayList<>();
         for (String range : rateRanges) {
+            if (range.split("-").length != 2) {
+                continue;
+            }
             String[] rates = range.split("-");
             double start = Double.parseDouble(rates[0]);
             double end = Double.parseDouble(rates[1]);

recommend-server-service/src/main/resources/application.yml

@@ -48,7 +48,7 @@ app:
 apollo:
   bootstrap:
     enabled: true
-    namespaces: application
+    namespaces: application,RD.Experiment
   cacheDir: /datalog/apollo-cache-dir
 
 feign: