Merge branch 'master' into pre-master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/CreativeCalibrationModel.java

@@ -0,0 +1,53 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CreativeCalibrationModel extends Model {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(CreativeCalibrationModel.class);
+
+    private Map<Long, Double> creativeDiffRateMap = new HashMap<>();
+
+    @Override
+    public int getModelSize() {
+        return this.creativeDiffRateMap.size();
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws Exception {
+        Map<Long, Double> initMap = new HashMap<>();
+        try (BufferedReader input = new BufferedReader(in)) {
+            String line;
+            while ((line = input.readLine()) != null) {
+                String[] items = line.split("\t");
+                if (items.length < 2) {
+                    continue;
+                }
+
+                long cid = Long.parseLong(items[0]);
+                double diffRate = Double.parseDouble(items[1]);
+
+                initMap.put(cid, diffRate);
+
+            }
+            this.creativeDiffRateMap = initMap;
+            LOGGER.info("[CreativeCalibrationModel] model load over and size {}", this.creativeDiffRateMap.size());
+        } catch (Exception e) {
+            LOGGER.info("[CreativeCalibrationModel] model load error ", e);
+        } finally {
+            in.close();
+
+        }
+        return true;
+    }
+
+    public double getDiffRate(long cid) {
+        return this.creativeDiffRateMap.getOrDefault(cid, 0d);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/XGBCalibrationModel.java → ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/ValueRangeCalibrationModel.java

@@ -9,9 +9,9 @@ import java.io.BufferedReader;
 import java.io.InputStreamReader;
 import java.util.Objects;
 
-public class XGBCalibrationModel extends Model {
+public class ValueRangeCalibrationModel extends Model {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(XGBCalibrationModel.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ValueRangeCalibrationModel.class);
 
     private Table<Double, Double, Double> table = HashBasedTable.create();
 
@@ -40,12 +40,12 @@ public class XGBCalibrationModel extends Model {
             this.table = initTable;
 
             for (Table.Cell<Double, Double, Double> cell : this.table.cellSet()) {
-                LOGGER.info("cell.row: {}, cell.column: {}, cell.value: {}", cell.getRowKey(), cell.getColumnKey(), cell.getValue());
+                LOGGER.info("[ValueRangeCalibrationModel] cell.row: {}, cell.column: {}, cell.value: {}", cell.getRowKey(), cell.getColumnKey(), cell.getValue());
             }
 
-            LOGGER.info("[XGBCalibrationModel] model load over and size {}", this.table.size());
+            LOGGER.info("[ValueRangeCalibrationModel] model load over and size {}", this.table.size());
         } catch (Exception e) {
-            LOGGER.info("[XGBCalibrationModel] model load error ", e);
+            LOGGER.info("[ValueRangeCalibrationModel] model load error ", e);
         } finally {
             in.close();
 

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/NumUtil.java

@@ -1,5 +1,9 @@
 package com.tzld.piaoquan.ad.engine.commons.util;
 
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.Arrays;
+
 public class NumUtil {
 
     public static double div(double d1, double d2) {
@@ -10,13 +14,41 @@ public class NumUtil {
     }
 
 
-    public static Double log(double a){
-        if (a <= 0){
+    public static double log(double a) {
+        if (a <= 0) {
             return 0D;
         }
         return Math.log(a + 1.0);
     }
 
+    public static double log10(double a) {
+        if (a <= 0) {
+            return 0d;
+        }
+        return Math.log10(a + 1.0);
+    }
+
+    public static double sigmoid(double x) {
+        return 1 / (1 + Math.exp(-x));
+    }
+
+    public static double round(double value, int newScale){
+        return new BigDecimal(value).setScale(newScale, RoundingMode.HALF_UP).doubleValue();
+    }
+
+    public static double[] softmax(double[] inputs) {
+        double max = Arrays.stream(inputs).max().orElse(0);
+        double[] expValues = Arrays.stream(inputs)
+                .map(x -> Math.exp(x - max)) // Subtract max to stabilize
+                .toArray();
+
+        double sum = Arrays.stream(expValues).sum();
+
+        return Arrays.stream(expValues)
+                .map(x -> x / sum)
+                .toArray();
+    }
+
     @SafeVarargs
     public static <T extends Comparable<T>> T min(T... values) {
         if (values == null || values.length == 0) {

ad-engine-server/src/main/resources/ad_score_config_xgboost_20241105.conf

@@ -5,7 +5,7 @@ scorer-config = {
     model-path = "zhangbo/model_xgb_351_1000_v2.tar.gz"
   }
   calibration-score-config = {
-    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.scorer.ScoreCalibrationScorer"
+    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.scorer.CreativeCalibrationScorer"
     scorer-priority = 98
     model-path = "zhangbo/model_xgb_351_1000_v2_calibration.txt"
   }

ad-engine-server/src/main/resources/ad_score_config_xgboost_683.conf

@@ -1,8 +1,7 @@
 scorer-config = {
-  lr-rov-score-config = {
+  xgb-score-config = {
     scorer-name = "com.tzld.piaoquan.ad.engine.service.score.scorer.XGBoostScorer683"
     scorer-priority = 99
-    model-path = "zhangbo/model_xgb_351_1000.tar.gz"
+    model-path = "fengzhoutian/model_xgb_351_1000_30d_v1.tar.gz"
   }
-
-}
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/FeatureService.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.feature;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
 import com.tzld.piaoquan.ad.engine.service.remote.FeatureV2RemoteService;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
@@ -82,9 +83,9 @@ public class FeatureService {
         return this.invokeFeatureService(protos);
     }
 
-    public Feature getPredictFeature(String mid) {
+    public Feature getPredictFeature(PredictContext context) {
         List<FeatureKeyProto> protos = new ArrayList<>();
-        protos.add(genWithMid("alg_ad_crowd_choose_feature", mid));
+        protos.add(genWithMidAndAppType("alg_ad_crowd_choose_feature_v2", context.getMid(), context.getAppType()));
         return this.invokeFeatureService(protos);
     }
 
@@ -224,6 +225,15 @@ public class FeatureService {
                 .build();
     }
 
+    private FeatureKeyProto genWithMidAndAppType(String table, String mid, String appType) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(midUkFormat, table, mid))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .putFieldValue("apptype", appType)
+                .build();
+    }
+
     private Map<String, String> featureStrCover(Map<String, String> metaFeatureMap) {
         Map<String, String> newFeatureMap = new HashMap<>();
         for (Map.Entry<String, String> entry : metaFeatureMap.entrySet()) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -136,7 +136,7 @@ public class LogHubServiceImpl implements LogHubService {
                 logMap.put("allfeature", JSON.toJSONString(context.getLogParam().getAllFeature()));
                 logMap.put("metafeature", JSON.toJSONString(context.getLogParam().getMetaFeature()));
                 logMap.put("scoremap", JSON.toJSONString(context.getLogParam().getScoreMap()));
-
+                logMap.put("isnewuser", context.getLogParam().isBIsNewUser());
                 aliyunLogManager.sendLog(project, crowdChooseStatisticsLogStore, "", logMap);
             }
         });

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/impl/PredictModelServiceImpl.java

@@ -9,6 +9,7 @@ import com.tzld.piaoquan.ad.engine.commons.redis.AdRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.util.AbUtil;
 import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
+import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import com.tzld.piaoquan.ad.engine.service.predict.PredictModelService;
 import com.tzld.piaoquan.ad.engine.service.predict.config.AbConfig;
 import com.tzld.piaoquan.ad.engine.service.predict.config.NewExpUserGroupConfig;
@@ -23,6 +24,7 @@ import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelPa
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.RoiPredictModelRequestParam;
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.ThresholdPredictModelRequestParam;
 import com.tzld.piaoquan.ad.engine.service.predict.v2.ConvertUtil;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
 import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictServiceV2;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -61,6 +63,8 @@ public class PredictModelServiceImpl implements PredictModelService {
 
     @Autowired
     private PredictServiceV2 predictServiceV2;
+    @Autowired
+    private LogHubService logHubService;
 
     @Value("${ad.predict.share0.exp.code:000}")
     private String adPredictNoShareUserExpCode;
@@ -157,8 +161,12 @@ public class PredictModelServiceImpl implements PredictModelService {
                 return result;
             }
 
-            if (AbUtil.isInAbExp(expCodes, requestParam.getAppType(), requestParam.getNewExpGroup(), "713")){
-                return predictServiceV2.adPredict(ConvertUtil.predictParam2Context(requestParam));
+            if (AbUtil.isInAbExp(expCodes, requestParam.getAppType(), requestParam.getNewExpGroup(), "713")) {
+                PredictContext context = ConvertUtil.predictParam2Context(requestParam);
+                Map<String, Object> resultMap = predictServiceV2.adPredict(context);
+                logHubService.crowdChooseLogUpload(context);
+                resultMap.put("pqtId", requestParam.getPqtId());
+                return resultMap;
             }
 
             String abtestId = null;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/RandomPredict673Model.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.ad.engine.service.predict.model.threshold;
 
+import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
 import com.tzld.piaoquan.ad.engine.service.predict.container.RandWContainer;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
@@ -65,8 +66,20 @@ public class RandomPredict673Model extends ThresholdPredictModel {
         result.put("threshold", threshold);
         result.put("model", this.initName());
 
-        log.info("广告跳出选择 -- 673实验结果: {}, 参数: {}, {}, {}, {}",
-                JSONUtils.toJson(result), appType, modelParam.getMid(), abParamKey, thresholdParamKey);
+        JSONObject logJson = new JSONObject();
+        logJson.putAll(result);
+        logJson.put("mid", modelParam.getMid());
+        logJson.put("appType", appType);
+        logJson.put("expId", "673");
+        logJson.put("userExternalSource", modelParam.getUserExternalSource());
+        logJson.put("shareType", modelParam.getUserExtraFuture("shareType").toString());
+        logJson.put("shareLayer", modelParam.getShareLayer());
+        logJson.put("thresholdParamKey", thresholdParamKey);
+        logJson.put("adPlatformType", modelParam.getAdPlatformType());
+
+        log.info("广告跳出选择 -- 673实验结果: {}, 参数: {}",
+                JSONUtils.toJson(result), logJson.toJSONString());
+
 
         return result;
     }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/RandomPredictModel.java

@@ -54,9 +54,10 @@ public class RandomPredictModel extends ThresholdPredictModel {
         logJson.put("expId", "599");
         logJson.put("appType", appType);
         logJson.put("thresholdParamKey", thresholdParamKey);
+        logJson.put("adPlatformType", modelParam.getAdPlatformType());
 
-        log.info("广告跳出选择 -- 599实验结果: {}, 参数: {}, {}, {}",
-                JSONUtils.toJson(result), appType, modelParam.getMid(), thresholdParamKey);
+        log.info("广告跳出选择 -- 599实验结果: {}, 参数: {}",
+                JSONUtils.toJson(result), logJson.toJSONString());
         return result;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/param/ThresholdPredictModelParam.java

@@ -42,6 +42,8 @@ public class ThresholdPredictModelParam {
     String city = "-1";
     MachineInfoParam machineInfo = new MachineInfoParam();
 
+    private String adPlatformType;
+
 
     private String userExternalSource;
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/PredictLogParam.java

@@ -25,4 +25,6 @@ public class PredictLogParam {
     private Map<String, Double> scoreMap = new HashMap<>();
 
     private boolean aIsShowAd;
+
+    private boolean bIsNewUser;
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/PredictServiceV2.java

@@ -4,7 +4,6 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
-import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -19,93 +18,173 @@ public class PredictServiceV2 {
 
     @Autowired
     private FeatureService featureService;
-    @Autowired
-    private LogHubService logHubService;
 
     @ApolloJsonValue("${exp.713.config:{}}")
     private Map<String, Double> exp713Config;
 
     public Map<String, Object> adPredict(PredictContext context) {
-
-        Feature feature = featureService.getPredictFeature(context.getMid());
+        Feature feature = featureService.getPredictFeature(context);
         Map<String, Map<String, String>> userFeature = feature.getUserFeature();
-        Map<String, String> featureMap = userFeature.getOrDefault("alg_ad_crowd_choose_feature", new HashMap<>());
-
-        double score = -1;
-
-        // 没有特征为新用户，随机出广告
-        if (MapUtils.isEmpty(featureMap)) {
-            double newUserShowAdRate = exp713Config.getOrDefault("newUserShowAdRate", 0.8d);
-            double randomRate = Math.random();
-            if (randomRate < newUserShowAdRate) {
-                context.getLogParam().getScoreMap().put("newUserShowAdRate", newUserShowAdRate);
-                context.getLogParam().getScoreMap().put("randomRate", randomRate);
-                score = 0;
-            }
-        } else {
-            context.getLogParam().getMetaFeature().putAll(userFeature);
+        Map<String, String> featureMap = userFeature.getOrDefault("alg_ad_crowd_choose_feature_v2", new HashMap<>());
+        double minScore = exp713Config.getOrDefault("minScore", 0.1d);
+        double maxScore = exp713Config.getOrDefault("maxScore", 0.8d);
+        double score = maxScore;
+
+        context.getLogParam().setBIsNewUser(true);
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+
+            double adViewCnt = Double.parseDouble(featureMap.getOrDefault("ad_view_cnt", "0"));
+            double adClick = Double.parseDouble(featureMap.getOrDefault("ad_click", "0"));
+            double adConver = Double.parseDouble(featureMap.getOrDefault("ad_conver", "0"));
+            double hasAdClick = Double.parseDouble(featureMap.getOrDefault("has_ad_click", "0"));
+            double hasAdShare = Double.parseDouble(featureMap.getOrDefault("has_ad_share", "0"));
+            double hasAdReturnCnt = Double.parseDouble(featureMap.getOrDefault("has_ad_return_cnt", "0"));
+            double noAdClick = Double.parseDouble(featureMap.getOrDefault("no_ad_click", "0"));
+            double noAdShare = Double.parseDouble(featureMap.getOrDefault("no_ad_share", "0"));
+            double noAdReturnCnt = Double.parseDouble(featureMap.getOrDefault("no_ad_return_cnt", "0"));
+
+            // 计算出回流出广告时的收益
+            double adClickValue = NumUtil.div(adClick, adViewCnt) * NumUtil.log10(adClick);
+            double adConverValue = NumUtil.div(adConver, adViewCnt) * NumUtil.log10(adConver);
+            double hasAdShareValue = NumUtil.div(hasAdShare, hasAdClick) * NumUtil.log10(hasAdShare);
+            double hasAdReturnValue = NumUtil.div(hasAdReturnCnt, hasAdClick) * NumUtil.log10(hasAdReturnCnt);
+            double hasAdValue = adClickValue + adConverValue + hasAdShareValue + hasAdReturnValue;
+
+            // 计算回流不出广告时的收益
+            double noAdShareValue = NumUtil.div(noAdShare, noAdClick) * NumUtil.log10(noAdShare);
+            double noAdReturnValue = NumUtil.div(noAdReturnCnt, noAdClick) * NumUtil.log10(noAdReturnCnt + hasAdReturnCnt);
+            double noAdValue = noAdShareValue + noAdReturnValue;
+
+            // 计算最终的收益
+            double hasRate = exp713Config.getOrDefault("hasRate", 1d);
+            double noRate = exp713Config.getOrDefault("noRate", 1d);
+            score = NumUtil.softmax(new double[]{hasAdValue * hasRate, noAdValue * noRate})[0];
+
+
+            context.getLogParam().getMetaFeature().putAll(feature.getUserFeature());
             for (Map.Entry<String, String> entry : featureMap.entrySet()) {
                 context.getLogParam().getAllFeature().put(entry.getKey(), Double.parseDouble(entry.getValue()));
             }
+            context.getLogParam().getScoreMap().put("adClickValue", NumUtil.round(adClickValue, 6));
+            context.getLogParam().getScoreMap().put("adConverValue", NumUtil.round(adConverValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdShareValue", NumUtil.round(hasAdShareValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdReturnValue", NumUtil.round(hasAdReturnValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdValue", NumUtil.round(hasAdValue, 6));
+            context.getLogParam().getScoreMap().put("noAdShareValue", NumUtil.round(noAdShareValue, 6));
+            context.getLogParam().getScoreMap().put("noAdReturnValue", NumUtil.round(noAdReturnValue, 6));
+            context.getLogParam().getScoreMap().put("noAdValue", NumUtil.round(noAdValue, 6));
+            context.getLogParam().getScoreMap().put("originScore", NumUtil.round(score, 6));
+            context.getLogParam().getScoreMap().put("hasRate", NumUtil.round(hasRate, 6));
+            context.getLogParam().getScoreMap().put("noRate", NumUtil.round(noRate, 6));
+
+            context.getLogParam().setBIsNewUser(false);
+        }
 
-            // 获取需要的特征值
-            double showAdClickPv = Double.parseDouble(featureMap.getOrDefault("show_ad_click_pv", "0"));
-            double noShowAdClickPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_click_pv", "0"));
-            double showAdIncome = Double.parseDouble(featureMap.getOrDefault("show_ad_income", "0"));
-            double showAdSharePv = Double.parseDouble(featureMap.getOrDefault("show_ad_share_pv", "0"));
-            double noShowAdSharePv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_share_pv", "0"));
-            double showAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("show_ad_new_return_pv", "0"));
-            double noShowAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_new_return_pv", "0"));
-
-            // 计算中间过程值
-            double singleReturnAdIncome = NumUtil.div(showAdIncome, (showAdClickPv + noShowAdClickPv));
-            double showAdShareRate = NumUtil.div((showAdSharePv + 1), (showAdClickPv + 1));
-            double noShowAdShareRate = NumUtil.div((noShowAdSharePv + 1), (noShowAdClickPv + 1));
-            double returnDivShare = NumUtil.div((showAdNewReturnPv + noShowAdNewReturnPv + 1), (showAdSharePv + noShowAdSharePv + 1));
-
-            double busDauBalanceRate = exp713Config.getOrDefault("busDauBalanceRate", 0.1d);
-            score = singleReturnAdIncome + ((showAdShareRate - noShowAdShareRate) * returnDivShare * busDauBalanceRate);
-
-
-            context.getLogParam().getScoreMap().put("singleReturnAdIncome", singleReturnAdIncome);
-            context.getLogParam().getScoreMap().put("showAdShareRate", showAdShareRate);
-            context.getLogParam().getScoreMap().put("noShowAdShareRate", noShowAdShareRate);
-            context.getLogParam().getScoreMap().put("returnDivShare", returnDivShare);
-            context.getLogParam().getScoreMap().put("busDauBalanceRate", busDauBalanceRate);
-
+        // 分数截断，避免过长或过短
+        if (score < minScore) {
+            score = minScore;
+        } else if (score > maxScore) {
+            score = maxScore;
         }
-        double showAdScoreThreshold = exp713Config.getOrDefault("showAdScoreThreshold", 0d);
-        boolean isShowAd = score >= showAdScoreThreshold;
 
-        context.getLogParam().setAIsShowAd(isShowAd);
+        double random = Math.random();
+        boolean isShowAd = random < score;
         context.getLogParam().setExpId("713");
         context.getLogParam().setScore(score);
-        context.getLogParam().getScoreMap().put("score", score);
-        context.getLogParam().getScoreMap().put("showAdScoreThreshold", showAdScoreThreshold);
-
-
-        logHubService.crowdChooseLogUpload(context);
+        context.getLogParam().getScoreMap().put("score", NumUtil.round(score, 6));
+        context.getLogParam().setAIsShowAd(isShowAd);
+        context.getLogParam().getScoreMap().put("minScore", minScore);
+        context.getLogParam().getScoreMap().put("maxScore", maxScore);
+        context.getLogParam().getScoreMap().put("random", random);
 
 
-        if (isShowAd) {
-            Map<String, Object> rtnMap = rtnAdPredict();
-            rtnMap.putAll(context.getLogParam().getScoreMap());
-            return rtnMap;
-        } else {
-            return rtnNoAdPredict("713_exp");
-        }
+        return isShowAd ? rtnAdPredict(context) : rtnNoAdPredict(context);
     }
 
-    private Map<String, Object> rtnNoAdPredict(String noAdStrategy) {
+
+    private Map<String, Object> rtnNoAdPredict(PredictContext context) {
         Map<String, Object> rtnMap = new HashMap<>();
         rtnMap.put("ad_predict", 1);
-        rtnMap.put("no_ad_strategy", noAdStrategy);
+        rtnMap.put("no_ad_strategy", "713_exp");
         return rtnMap;
     }
 
-    private Map<String, Object> rtnAdPredict() {
+    private Map<String, Object> rtnAdPredict(PredictContext context) {
         Map<String, Object> rtnMap = new HashMap<>();
         rtnMap.put("ad_predict", 2);
+        rtnMap.putAll(context.getLogParam().getScoreMap());
         return rtnMap;
     }
+
+    // public Map<String, Object> adPredictV1(PredictContext context){
+    //
+    //     Feature feature = featureService.getPredictFeature(context);
+    //     Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+    //     Map<String, String> featureMap = userFeature.getOrDefault("alg_ad_crowd_choose_feature", new HashMap<>());
+    //
+    //     double score = -1;
+    //
+    //     // 没有特征为新用户，随机出广告
+    //     if (MapUtils.isEmpty(featureMap)) {
+    //         double newUserShowAdRate = exp713Config.getOrDefault("newUserShowAdRate", 0.8d);
+    //         double randomRate = Math.random();
+    //         if (randomRate < newUserShowAdRate) {
+    //             context.getLogParam().getScoreMap().put("newUserShowAdRate", newUserShowAdRate);
+    //             context.getLogParam().getScoreMap().put("randomRate", randomRate);
+    //             score = 1;
+    //         }
+    //     } else {
+    //         context.getLogParam().getMetaFeature().putAll(userFeature);
+    //         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
+    //             context.getLogParam().getAllFeature().put(entry.getKey(), Double.parseDouble(entry.getValue()));
+    //         }
+    //
+    //         // 获取需要的特征值
+    //         double showAdClickPv = Double.parseDouble(featureMap.getOrDefault("show_ad_click_pv", "0"));
+    //         double noShowAdClickPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_click_pv", "0"));
+    //         double showAdIncome = Double.parseDouble(featureMap.getOrDefault("show_ad_income", "0"));
+    //         double showAdSharePv = Double.parseDouble(featureMap.getOrDefault("show_ad_share_pv", "0"));
+    //         double noShowAdSharePv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_share_pv", "0"));
+    //         double showAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("show_ad_new_return_pv", "0"));
+    //         double noShowAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_new_return_pv", "0"));
+    //
+    //         // 计算中间过程值
+    //         double singleReturnAdIncome = NumUtil.div(showAdIncome, (showAdClickPv + noShowAdClickPv));
+    //         double showAdShareRate = NumUtil.div((showAdSharePv + 1), (showAdClickPv + 1));
+    //         double noShowAdShareRate = NumUtil.div((noShowAdSharePv + 1), (noShowAdClickPv + 1));
+    //         double returnDivShare = NumUtil.div((showAdNewReturnPv + noShowAdNewReturnPv + 1), (showAdSharePv + noShowAdSharePv + 1));
+    //
+    //         double busDauBalanceRate = exp713Config.getOrDefault("busDauBalanceRate", 0.1d);
+    //         score = singleReturnAdIncome + ((showAdShareRate - noShowAdShareRate) * returnDivShare * busDauBalanceRate);
+    //
+    //
+    //         context.getLogParam().getScoreMap().put("singleReturnAdIncome", singleReturnAdIncome);
+    //         context.getLogParam().getScoreMap().put("showAdShareRate", showAdShareRate);
+    //         context.getLogParam().getScoreMap().put("noShowAdShareRate", noShowAdShareRate);
+    //         context.getLogParam().getScoreMap().put("returnDivShare", returnDivShare);
+    //         context.getLogParam().getScoreMap().put("busDauBalanceRate", busDauBalanceRate);
+    //
+    //     }
+    //     double showAdScoreThreshold = exp713Config.getOrDefault("showAdScoreThreshold", 0d);
+    //     boolean isShowAd = score >= showAdScoreThreshold;
+    //
+    //     context.getLogParam().setAIsShowAd(isShowAd);
+    //     context.getLogParam().setExpId("713");
+    //     context.getLogParam().setScore(score);
+    //     context.getLogParam().getScoreMap().put("score", score);
+    //     context.getLogParam().getScoreMap().put("showAdScoreThreshold", showAdScoreThreshold);
+    //
+    //
+    //     logHubService.crowdChooseLogUpload(context);
+    //
+    //
+    //     if (isShowAd) {
+    //         Map<String, Object> rtnMap = rtnAdPredict();
+    //         rtnMap.putAll(context.getLogParam().getScoreMap());
+    //         return rtnMap;
+    //     } else {
+    //         return rtnNoAdPredict("713_exp");
+    //     }
+    // }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/scorer/CreativeCalibrationScorer.java

@@ -0,0 +1,148 @@
+package com.tzld.piaoquan.ad.engine.service.score.scorer;
+
+import com.tzld.piaoquan.ad.engine.commons.score.AbstractScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.CreativeCalibrationModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+public class CreativeCalibrationScorer extends AbstractScorer {
+
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private static final Logger LOGGER = LoggerFactory.getLogger(CreativeCalibrationScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+    public CreativeCalibrationScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        super.doLoadModel(CreativeCalibrationModel.class);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    final List<AdRankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+
+        List<AdRankItem> result = rankByJava(sceneFeatureMap, userFeatureMap, rankItems);
+
+        LOGGER.debug("[CreativeCalibrationScorer] scoring items size={}, time={} ",
+                result.size(), System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                        final Map<String, String> userFeatureMap,
+                                        final List<AdRankItem> items) {
+        long startTime = System.currentTimeMillis();
+        CreativeCalibrationModel model = (CreativeCalibrationModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (AdRankItem item : items) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", item, item);
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("[CreativeCalibrationScorer] items size={}, cost={} ",
+                items.size(), System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final CreativeCalibrationModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex), ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // 等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException: ", e);
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},", sceneFeatureMap.size(), e);
+                }
+            }
+        }
+    }
+
+    public double calcScore(final CreativeCalibrationModel model,
+                            final AdRankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+        double ctcvrScore = item.getLrScore();
+        double newCtcvrScore = ctcvrScore;
+        try {
+
+            double diffRate = model.getDiffRate(item.getAdId());
+            newCtcvrScore = ctcvrScore / (1 + diffRate);
+            item.setLrScore(newCtcvrScore);
+            item.getScoreMap().put("diff_rate", diffRate);
+            item.getScoreMap().put("originCtcvrScore", ctcvrScore);
+            item.getScoreMap().put("ctcvrScore", newCtcvrScore);
+        } catch (Exception e) {
+            LOGGER.error("[score calibration] calcScore error: ", e);
+        }
+
+        return newCtcvrScore;
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/scorer/ScoreCalibrationScorer.java → ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/scorer/ValueRangeCalibrationScorer.java

@@ -3,7 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score.scorer;
 import com.tzld.piaoquan.ad.engine.commons.score.AbstractScorer;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
-import com.tzld.piaoquan.ad.engine.commons.score.model.XGBCalibrationModel;
+import com.tzld.piaoquan.ad.engine.commons.score.model.ValueRangeCalibrationModel;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import org.apache.commons.collections4.CollectionUtils;
@@ -17,20 +17,20 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.*;
 
-public class ScoreCalibrationScorer extends AbstractScorer {
+public class ValueRangeCalibrationScorer extends AbstractScorer {
 
     private static final int LOCAL_TIME_OUT = 150;
-    private static Logger LOGGER = LoggerFactory.getLogger(ScoreCalibrationScorer.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ValueRangeCalibrationScorer.class);
     private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
 
 
-    public ScoreCalibrationScorer(ScorerConfigInfo scorerConfigInfo) {
+    public ValueRangeCalibrationScorer(ScorerConfigInfo scorerConfigInfo) {
         super(scorerConfigInfo);
     }
 
     @Override
     public void loadModel() {
-        super.doLoadModel(XGBCalibrationModel.class);
+        super.doLoadModel(ValueRangeCalibrationModel.class);
     }
 
     @Override
@@ -59,7 +59,7 @@ public class ScoreCalibrationScorer extends AbstractScorer {
                                         final Map<String, String> userFeatureMap,
                                         final List<AdRankItem> items) {
         long startTime = System.currentTimeMillis();
-        XGBCalibrationModel model = (XGBCalibrationModel) this.getModel();
+        ValueRangeCalibrationModel model = (ValueRangeCalibrationModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
 
         // 所有都参与打分，按照ctr排序
@@ -82,7 +82,7 @@ public class ScoreCalibrationScorer extends AbstractScorer {
     private void multipleCtrScore(final List<AdRankItem> items,
                                   final Map<String, String> userFeatureMap,
                                   final Map<String, String> sceneFeatureMap,
-                                  final XGBCalibrationModel model) {
+                                  final ValueRangeCalibrationModel model) {
 
         List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
         for (int index = 0; index < items.size(); index++) {
@@ -124,7 +124,7 @@ public class ScoreCalibrationScorer extends AbstractScorer {
         }
     }
 
-    public double calcScore(final XGBCalibrationModel model,
+    public double calcScore(final ValueRangeCalibrationModel model,
                             final AdRankItem item,
                             final Map<String, String> userFeatureMap,
                             final Map<String, String> sceneFeatureMap) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -298,11 +298,14 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
+                double ecpm = NumUtil.div(income * 1000, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -328,11 +331,14 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
+                double ecpm = NumUtil.div(income * 1000, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -370,6 +376,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
         featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
         featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
 
         return midActionList;
     }
@@ -423,6 +430,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
             featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
             featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
             featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
         }
     }
 
@@ -431,7 +439,8 @@ public class RankStrategyBy679 extends RankStrategyBasic {
             return;
         }
 
-        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
         for (String prefix1 : prefixes1) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -47,6 +47,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
 
         Map<String, Double> weightParam = ObjUtil.nullOrDefault(weightMap, new HashMap<>());
 
+
         Map<Long, Double> creativeScoreCoefficient = getCreativeScoreCoefficient();
         Set<String> noApiAdVerIds = getNoApiAdVerIds();
 
@@ -306,11 +307,14 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
+                double ecpm = NumUtil.div(income * 1000, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -338,11 +342,12 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
                 double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
+                double ecpm = NumUtil.div(income * 1000, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -380,7 +385,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
         featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
         featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
-        // featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
 
         return midActionList;
     }
@@ -434,7 +439,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
             featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
             featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
-            // featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
         }
     }
 
@@ -443,8 +448,8 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             return;
         }
 
-        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
-        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
         for (String prefix1 : prefixes1) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
@@ -9,8 +10,10 @@ import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
@@ -33,12 +36,32 @@ public class RankStrategyBy683 extends RankStrategyBasic {
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
+    @Value("${word2vec.exp:694}")
+    private String word2vecExp;
+
+    // FIXME(zhoutian): 可能需要独立配置
+    @ApolloJsonValue("${rank.score.weight.680:{}}")
+    private Map<String, Double> weightMap;
+
+    @ApolloJsonValue("${rank.score.neg_sample_rate:0.01}")
+    Double negSampleRate;
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+
+        Map<String, Double> weightParam = ObjUtil.nullOrDefault(weightMap, new HashMap<>());
+
+
+        Map<Long, Double> creativeScoreCoefficient = getCreativeScoreCoefficient();
         Set<String> noApiAdVerIds = getNoApiAdVerIds();
 
         long ts = System.currentTimeMillis() / 1000;
 
+        String brand = scoreParam.getRequestContext().getMachineinfoBrand();
+        if (StringUtils.isNotEmpty(brand)) {
+            scoreParam.getRequestContext().setMachineinfoBrand(brand + "-n");
+        }
+
         long start = System.currentTimeMillis();
         // 特征处理
         // feature1
@@ -67,7 +90,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
-        List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
+        List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
         CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
@@ -89,6 +112,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     } else {
                         adRankItem.getExt().put("isApi", "1");
                     }
+
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
 
                     String cidStr = dto.getCreativeId().toString();
@@ -138,27 +162,25 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             String title = b1Feature.getOrDefault("cidtitle", "");
             ThreadPoolFactory.defaultPool().submit(() -> {
                 try {
-                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap(), scoreParam);
                 } finally {
                     cdl2.countDown();
                 }
             });
             ThreadPoolFactory.defaultPool().submit(() -> {
                 try {
-                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap(), scoreParam);
                 } finally {
                     cdl2.countDown();
                 }
             });
         }
-        long time31 = System.currentTimeMillis();
         try {
             cdl2.await(150, TimeUnit.MILLISECONDS);
         } catch (Exception e) {
             log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
         }
 
-        // feature4
         long time3 = System.currentTimeMillis();
         // 分桶
         this.readBucketFile();
@@ -184,52 +206,73 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         // getScorerPipeline
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_683).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
         long time5 = System.currentTimeMillis();
+
+        // calibrate score for negative sampling
+        for (AdRankItem item : result) {
+            double originalScore = item.getLrScore();
+            double calibratedScore = originalScore / (originalScore + (1 - originalScore) / negSampleRate);
+            item.setLrScore(calibratedScore);
+            item.getScoreMap().put("originCtcvrScore", originalScore);
+            item.getScoreMap().put("ctcvrScore", calibratedScore);
+        }
+
         // loop
+        double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
+
         for (AdRankItem item : result) {
-            item.setScore(item.getLrScore() * item.getCpa());
+
+            double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
+            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
+
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
+            item.getScoreMap().put("cpmCoefficient", cpmCoefficient);
+            item.getScoreMap().put("scoreCoefficient", scoreCoefficient);
             item.getFeatureMap().putAll(userFeatureMap);
             item.getFeatureMap().putAll(sceneFeatureMap);
 
             // 没有转化回传的广告主，使用后台配置的CPM
             if (noApiAdVerIds.contains(item.getAdVerId())) {
-                item.setScore(item.getCpm() / 1000);
+                item.setScore(item.getCpm() * cpmCoefficient / 1000);
             }
+        }
+
+
+        result.sort(ComparatorUtil.equalsRandomComparator());
 
+        if (CollectionUtils.isNotEmpty(result)) {
+            AdRankItem top1Item = result.get(0);
             for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
                 if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
 
             for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
                 if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
 
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(item.getAdVerId(), new HashMap<>());
+            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(top1Item.getAdVerId(), new HashMap<>());
             for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
                 if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
 
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(top1Item.getAdId()), new HashMap<>());
             for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
                 if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
         }
-
-        log.info("cost={}, feature1={}, feature2={}, feature31={}, feature32={}, feature4={}, getScorerPipeline={}, " +
-                        "adIdSize={}, adRankItemsSize={}",
-                time5 - start, time1 - start, time2 - time1, time31 - time2, time3 - time31, time4 - time3,
-                time5 - time4, request.getAdIdList().size(), adRankItems.size());
-
-        result.sort(ComparatorUtil.equalsRandomComparator());
+        long time6 = System.currentTimeMillis();
+        log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +
+                        "other={}, adIdSize={}, adRankItemsSize={}",
+                time6 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3,
+                time5 - time4, time6 - time5, request.getAdIdList().size(), adRankItems.size());
 
         return result;
     }
@@ -278,11 +321,14 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
+                double ecpm = NumUtil.div(income * 1000, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -310,11 +356,12 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
                 double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
+                double ecpm = NumUtil.div(income * 1000, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -352,7 +399,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
         featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
         featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
-        // featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
 
         return midActionList;
     }
@@ -387,17 +434,11 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             featureMap.put("actionstatic_ctr", String.valueOf(ctr));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_conver_" + cid)) {
-            double ctcvr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
-            );
+            double ctcvr = NumUtil.div(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0), midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0));
             featureMap.put("actionstatic_ctcvr", String.valueOf(ctcvr));
         }
         if (midActionStatic.containsKey("actionstatic_conver_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            double cvr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)
-            );
+            double cvr = NumUtil.div(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0), midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0));
             featureMap.put("actionstatic_cvr", String.valueOf(cvr));
         }
     }
@@ -412,7 +453,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
             featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
             featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
-            // featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
         }
     }
 
@@ -421,8 +462,8 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             return;
         }
 
-        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
-        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
         for (String prefix1 : prefixes1) {
@@ -438,24 +479,28 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         }
     }
 
-    private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap) {
+    private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap,
+                                      ScoreParam scoreParam) {
         if (MapUtils.isEmpty(d3Feature) || !d3Feature.containsKey("title") || StringUtils.isEmpty(cTitle)) {
             return;
         }
         String vTitle = d3Feature.get("title");
-        double score = Similarity.conceptSimilarity(cTitle, vTitle);
+        double score;
+        if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
+            score = SimilarityUtils.word2VecSimilarity(cTitle, vTitle);
+        } else {
+            score = Similarity.conceptSimilarity(cTitle, vTitle);
+        }
         featureMap.put("ctitle_vtitle_similarity", String.valueOf(score));
     }
 
-    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title, Map<String, String> featureMap) {
+    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title,
+                                      Map<String, String> featureMap, ScoreParam scoreParam) {
         if (StringUtils.isEmpty(title)) {
             return;
         }
 
-        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(
-                new Tuple2<>(e1Feature, "e1"),
-                new Tuple2<>(e2Feature, "e2")
-        );
+        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(new Tuple2<>(e1Feature, "e1"), new Tuple2<>(e2Feature, "e2"));
 
         List<String> tagsFieldList = Arrays.asList("tags_3d", "tags_7d", "tags_14d");
         for (Tuple2<Map<String, String>, String> tuple2 : tuple2List) {
@@ -468,7 +513,13 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             for (String tagsField : tagsFieldList) {
                 if (StringUtils.isNotEmpty(feature.get(tagsField))) {
                     String tags = feature.get(tagsField);
-                    Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    // Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    Double[] doubles;
+                    if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
+                        doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
+                    } else {
+                        doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    }
                     featureMap.put(prefix + "_" + tagsField + "_matchnum", String.valueOf(doubles[0]));
                     featureMap.put(prefix + "_" + tagsField + "_maxscore", String.valueOf(doubles[1]));
                     featureMap.put(prefix + "_" + tagsField + "_avgscore", String.valueOf(doubles[2]));
@@ -527,9 +578,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
             String key = entry.getKey();
             String value = entry.getValue();
-            Map<String, Double> valueMap = Arrays.stream(value.split(","))
-                    .map(r -> r.split(":"))
-                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
+            Map<String, Double> valueMap = Arrays.stream(value.split(",")).map(r -> r.split(":")).collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
             vidRankMaps.put(key, valueMap);
         }
         return vidRankMaps;
@@ -562,9 +611,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                                 String key = rList[0];
                                 double value1 = Double.parseDouble(rList[1]);
                                 bucketsLen.put(key, value1);
-                                double[] value2 = Arrays.stream(rList[2].split(","))
-                                        .mapToDouble(Double::valueOf)
-                                        .toArray();
+                                double[] value2 = Arrays.stream(rList[2].split(",")).mapToDouble(Double::valueOf).toArray();
                                 bucketsMap.put(key, value2);
                             }
                         }
@@ -601,4 +648,4 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         return newFeatureMap;
     }
 
-}
+}