Merge branch 'dev-xym-calibration-dnn1' of algorithm/ad-engine into master

ad-engine-commons/pom.xml

@@ -26,7 +26,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.1.23</version>
+            <version>1.1.24</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/CalibrationModelCtcvrData.java

@@ -0,0 +1,13 @@
+package com.tzld.piaoquan.ad.engine.service.entity;
+
+import lombok.Data;
+
+@Data
+public class CalibrationModelCtcvrData {
+
+    private Integer view;
+
+    private Double realCtcvr;
+
+    private Double pCtcvr;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

@@ -14,6 +14,7 @@ import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.commons.util.ObjUtil;
+import com.tzld.piaoquan.ad.engine.service.entity.CalibrationModelCtcvrData;
 import com.tzld.piaoquan.ad.engine.service.entity.CorrectCpaParam;
 import com.tzld.piaoquan.ad.engine.service.entity.CorrectCtcvrScoreParam;
 import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
@@ -46,6 +47,9 @@ public abstract class RankStrategyBasic implements RankStrategy {
     @ApolloJsonValue("${alpha:1.0}")
     protected Double alpha;
 
+    @Value("${calibration.ctcvr.exp:779}")
+    protected String calibrationCtcvrExp;
+
     @Value("${correct.cpa.exp.1:}")
     protected String correctCpaExp1;
 
@@ -104,6 +108,11 @@ public abstract class RankStrategyBasic implements RankStrategy {
 
     String adPlatformGuaranteeKey = "ad:platform:guarantee:data:{date}:{adverId}";
 
+    String realLayerCtcvrKey = "ad:platform:real:ctcvr:{model}:{layer}:{class}";
+
+    String adCustomerLayerHourKey = "ad:platform:customer:hour:{layer}:{clazz}:{customerId}";
+
+    String adVerLayerHourKey = "ad:platform:adver:hour:{layer}:{clazz}:{adverId}";
     String adLayerHourKey = "ad:platform:ad:hour:{layer}:{clazz}:{adId}";
 
     String adLayerDayKey = "ad:platform:ad:day:{layer}:{clazz}:{adId}";
@@ -506,9 +515,22 @@ public abstract class RankStrategyBasic implements RankStrategy {
         }
     }
 
-    protected void calculateCtcvrScore(List<AdRankItem> items, RankRecommendRequestParam request, ScoreParam scoreParam) {
+    protected void calculateCtcvrScore(List<AdRankItem> items, RankRecommendRequestParam request, ScoreParam scoreParam, String modelName) {
+        //判断是否走校准试验
+        if (scoreParam.getExpCodeSet().contains(calibrationCtcvrExp)) {
+            try {
+                calibrationDnnCtcvrScore(items, request.getMid(), request.getIsFilterUser(), modelName);
+            } catch (Exception e) {
+                log.error("calibrationDnnCtcvrScore error", e);
+            }
+        }
+
         if (scoreParam.getExpCodeSet().contains(calibrationCoefficientExp)) {
-            calibrationCtcvrScore(items, request);
+            try {
+                calibrationCtcvrScore(items, request);
+            } catch (Exception e) {
+                log.error("calibrationCtcvrScore error", e);
+            }
         }
     }
 
@@ -676,6 +698,51 @@ public abstract class RankStrategyBasic implements RankStrategy {
         }
     }
 
+    protected void calibrationDnnCtcvrScore(List<AdRankItem> adRankItems, String mid, boolean isFilterUser, String modelName) {
+        if (StringUtils.isEmpty(modelName)) {
+            return;
+        }
+        // 1. 获取用户分层信息
+        Map<String, String> userLayer = getUserLayer(mid);
+        String layer = userLayer.getOrDefault("layer", "无曝光");
+        String clazz = userLayer.getOrDefault("class", "近期未出现");
+        if (Objects.equals(layer, "已转化")) {
+            layer = "有转化";
+        }
+        if (isFilterUser) {
+            layer += "-炸";
+        }
+        // 2. 构建Key模板
+        String layerKeyTemplate = realLayerCtcvrKey
+                .replace("{model}", modelName)
+                .replace("{layer}", layer)
+                .replace("{class}", clazz);
+
+
+        String value = adRedisHelper.get(layerKeyTemplate);
+        JSONObject json = JSONObject.parseObject(value);
+        Integer view = json.getInteger("view");
+        Double realCtcvr = json.getDouble("ctcvr");
+        Double pCtcvr = json.getDouble("pCtcvr");
+        CalibrationModelCtcvrData calibrationModelCtcvrData = new CalibrationModelCtcvrData();
+        calibrationModelCtcvrData.setRealCtcvr(realCtcvr);
+        calibrationModelCtcvrData.setView(view);
+        calibrationModelCtcvrData.setPCtcvr(pCtcvr);
+        for (AdRankItem item : adRankItems) {
+            item.getExt().put("calibrationModelCtcvrData", JSONObject.toJSONString(calibrationModelCtcvrData));
+            if (pCtcvr == null || pCtcvr == 0.0 || realCtcvr == null || realCtcvr == 0.0) {
+                continue;
+            }
+            double diff = realCtcvr / pCtcvr;
+            if (Math.abs(diff - 1) < 0.1) {
+                continue;
+            }
+            double correctCtcvrScore = item.getLrScore() * diff;
+            item.getScoreMap().put("correctCtcvrScore", correctCtcvrScore);
+            item.setLrScore(correctCtcvrScore);
+        }
+    }
+
     // 安全的数值转换
     private double safeDouble(Double value, Double min) {
         return (value == null || value == 0) ? min : value;
@@ -685,4 +752,4 @@ public abstract class RankStrategyBasic implements RankStrategy {
     private boolean isValidViews(Integer views, int threshold) {
         return views != null && views >= threshold;
     }
-}
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -118,6 +118,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
                     adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setCustomerId(dto.getCustomerId());
+                    adRankItem.setProfession(dto.getProfession());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -221,7 +222,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         // getScorerPipeline
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20241105).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
         long time5 = System.currentTimeMillis();
-        calculateCtcvrScore(result, request, scoreParam);
+        calculateCtcvrScore(result, request, scoreParam, null);
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -242,7 +242,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             item.getScoreMap().put("ctcvrScore", calibratedScore);
         }
 
-        calculateCtcvrScore(result, request, scoreParam);
+        calculateCtcvrScore(result, request, scoreParam, null);
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
 
@@ -281,6 +281,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             AdRankItem top1Item = result.get(0);
             top1Item.getExt().put("isGuaranteeType", isGuaranteeType);
             putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+            top1Item.getExt().put("model", "xgb");
         }
         long time6 = System.currentTimeMillis();
         log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -239,7 +239,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             item.getScoreMap().put("ctcvrScore", calibratedScore);
         }
 
-        calculateCtcvrScore(result, request, scoreParam);
+        calculateCtcvrScore(result, request, scoreParam, null);
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
         boolean isGuaranteeType = false;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -294,7 +294,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             item.getScoreMap().put("ctcvrScore", calibratedScore);
         }
 
-        calculateCtcvrScore(result, request, scoreParam);
+        calculateCtcvrScore(result, request, scoreParam, "dnn");
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
         boolean isGuaranteeType = false;
@@ -335,6 +335,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             AdRankItem top1Item = result.get(0);
             top1Item.getExt().put("isGuaranteeType", isGuaranteeType);
             putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+            top1Item.getExt().put("model", "dnn");
         }
         long time6 = System.currentTimeMillis();
         log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +