增加dnn模型校准

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/CalibrationCtcvrData.java → ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/CalibrationModelCtcvrData.java

@@ -3,7 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.entity;
 import lombok.Data;
 
 @Data
-public class CalibrationCtcvrData {
+public class CalibrationModelCtcvrData {
 
     private Integer view;
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

@@ -14,9 +14,9 @@ import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.commons.util.ObjUtil;
+import com.tzld.piaoquan.ad.engine.service.entity.CalibrationModelCtcvrData;
 import com.tzld.piaoquan.ad.engine.service.entity.CorrectCpaParam;
 import com.tzld.piaoquan.ad.engine.service.entity.CorrectCtcvrScoreParam;
-import com.tzld.piaoquan.ad.engine.service.entity.CalibrationCtcvrData;
 import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
@@ -104,10 +104,7 @@ public abstract class RankStrategyBasic implements RankStrategy {
 
     String adPlatformGuaranteeKey = "ad:platform:guarantee:data:{date}:{adverId}";
 
-    String realCtcvrCustomerKey = "ad:platform:real:ctcvr:{model}:{layer}:{class}:{profession}:{customerId}";
-
-    String realCtcvrProfessionKey = "ad:platform:real:ctcvr:{model}:{layer}:{class}:{profession}";
-
+    String realLayerCtcvrKey = "ad:platform:real:ctcvr:{model}:{layer}:{class}";
 
     String adCustomerLayerHourKey = "ad:platform:customer:hour:{layer}:{clazz}:{customerId}";
 
@@ -538,7 +535,12 @@ public abstract class RankStrategyBasic implements RankStrategy {
         }
     }
 
-    protected void calculateCtcvrScore(List<AdRankItem> items, RankRecommendRequestParam request, ScoreParam scoreParam) {
+    protected void calculateCtcvrScore(List<AdRankItem> items, RankRecommendRequestParam request, ScoreParam scoreParam, String modelName) {
+        //判断是否走校准试验
+        if (scoreParam.getExpCodeSet().contains(calibrationCtcvrExp)) {
+            calibrationDnnCtcvrScore(scoreParam, items, request.getMid(), request.getIsFilterUser(), modelName);
+        }
+
         if (scoreParam.getExpCodeSet().contains(calibrationCoefficientExp)) {
             calibrationCtcvrScore(items, request);
         }
@@ -708,113 +710,50 @@ public abstract class RankStrategyBasic implements RankStrategy {
         }
     }
 
-    protected void calibrationCtcvrScore(ScoreParam scoreParam, List<AdRankItem> adRankItems, String mid, boolean isFilterUser, String modelName) {
-        //判断是否走校准试验
-        if (!scoreParam.getExpCodeSet().contains(calibrationCtcvrExp)) {
+    protected void calibrationDnnCtcvrScore(ScoreParam scoreParam, List<AdRankItem> adRankItems, String mid, boolean isFilterUser, String modelName) {
+        if (StringUtils.isEmpty(modelName)) {
             return;
         }
         // 1. 获取用户分层信息
         Map<String, String> userLayer = getUserLayer(mid);
         String layer = userLayer.getOrDefault("layer", "无曝光");
         String clazz = userLayer.getOrDefault("class", "近期未出现");
+        if (Objects.equals(layer, "已转化")) {
+            layer = "有转化";
+        }
         if (isFilterUser) {
             layer += "-炸";
         }
-
         // 2. 构建Key模板
-        String customerKeyTemplate = realCtcvrCustomerKey
-                .replace("{model}", modelName)
-                .replace("{layer}", layer)
-                .replace("{class}", clazz);
-
-        String professionKeyTemplate = realCtcvrProfessionKey
+        String layerKeyTemplate = realLayerCtcvrKey
                 .replace("{model}", modelName)
                 .replace("{layer}", layer)
                 .replace("{class}", clazz);
 
-        // 3. 定义Key生成器
-        Function<AdRankItem, String> customerKeyFunc = e ->
-                (e.getProfession() != null && e.getCustomerId() != null)
-                        ? customerKeyTemplate.replace("{profession}", e.getProfession())
-                        .replace("{customerId}", String.valueOf(e.getCustomerId()))
-                        : null;
-
-        Function<AdRankItem, String> professionKeyFunc = e ->
-                (e.getProfession() != null)
-                        ? professionKeyTemplate.replace("{profession}", e.getProfession())
-                        : null;
-
-        // 4. 获取校准数据
-        Map<String, CalibrationCtcvrData> customerMap = getCtcvrMap(adRankItems, customerKeyFunc);
-        Map<String, CalibrationCtcvrData> professionMap = getCtcvrMap(adRankItems, professionKeyFunc);
-        // 5. 校准分数
+        String value = adRedisHelper.get(layerKeyTemplate);
+        JSONObject json = JSONObject.parseObject(value);
+        Integer view = json.getInteger("view");
+        Double realCtcvr = json.getDouble("ctcvr");
+        Double pCtcvr = json.getDouble("pCtcvr");
+        CalibrationModelCtcvrData calibrationModelCtcvrData = new CalibrationModelCtcvrData();
+        calibrationModelCtcvrData.setRealCtcvr(realCtcvr);
+        calibrationModelCtcvrData.setView(view);
+        calibrationModelCtcvrData.setPCtcvr(pCtcvr);
         for (AdRankItem item : adRankItems) {
-            CalibrationCtcvrData data = null;
-            String customerKey = customerKeyFunc.apply(item);
-            String professionKey = professionKeyFunc.apply(item);
-
-            if (customerKey != null) {
-                data = customerMap.get(customerKey);
+            item.getExt().put("calibrationModelCtcvrData", JSONObject.toJSONString(calibrationModelCtcvrData));
+            if (view == null || view < calibrationViewCount) {
+                continue;
             }
-            if (data == null && professionKey != null) {
-                data = professionMap.get(professionKey);
+            if (pCtcvr == null || pCtcvr == 0.0 || realCtcvr == null || realCtcvr == 0.0) {
+                continue;
             }
-            item.getExt().put("calibrationCtcvrData", JSONObject.toJSONString(data));
-            if (data == null || data.getView() == null || data.getView() < calibrationViewCount) continue;
-
-            Map<String, Double> scoreMap = item.getScoreMap();
-            Double pCtcvr = data.getPCtcvr();
-            Double realCtcvr = data.getRealCtcvr();
-
-            if (pCtcvr != null && pCtcvr != 0.0 && realCtcvr != null && realCtcvr != 0.0) {
-                double diff = realCtcvr / pCtcvr;
-                if (Math.abs(diff - 1) >= 0.1) {
-                    Double ctcvrScore = scoreMap.get("ctcvrScore");
-                    if (ctcvrScore == null) {
-                        continue;
-                    }
-                    scoreMap.put("modelCtcvrScore", ctcvrScore);
-                    scoreMap.put("ctcvrScore", ctcvrScore * diff);
-                    item.setLrScore(ctcvrScore * diff);
-                }
+            double diff = realCtcvr / pCtcvr;
+            if (Math.abs(diff - 1) < 0.1) {
+                continue;
             }
+            double correctCtcvrScore = item.getLrScore() * diff;
+            item.getScoreMap().put("correctCtcvrScore", correctCtcvrScore);
+            item.setLrScore(correctCtcvrScore);
         }
     }
-
-    // 通用CTR数据获取方法
-    private Map<String, CalibrationCtcvrData> getCtcvrMap(
-            List<AdRankItem> items,
-            Function<AdRankItem, String> keyGenerator
-    ) {
-        // 生成有效Key集合
-        List<String> redisKeys = items.stream()
-                .map(keyGenerator)
-                .filter(Objects::nonNull)
-                .distinct()
-                .collect(Collectors.toList());
-
-        // 批量查询Redis
-        List<String> redisValues = adRedisHelper.mget(redisKeys);
-        Map<String, CalibrationCtcvrData> resultMap = new HashMap<>();
-
-        for (int i = 0; i < redisKeys.size(); i++) {
-            String val = redisValues.get(i);
-            if (StringUtils.isEmpty(val)) continue;
-
-            try {
-                JSONObject json = JSONObject.parseObject(val);
-                Integer view = json.getInteger("view");
-                Double ctcvr = json.getDouble("ctcvr");
-                Double pCtcvr = json.getDouble("pCtcvr");
-                CalibrationCtcvrData data = new CalibrationCtcvrData();
-                data.setView(view);
-                data.setRealCtcvr(ctcvr);
-                data.setPCtcvr(pCtcvr);
-                resultMap.put(redisKeys.get(i), data);
-            } catch (Exception e) {
-                log.error("getCtcvrMap error", e);
-            }
-        }
-        return resultMap;
-    }
-}
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -222,7 +222,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         // getScorerPipeline
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20241105).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
         long time5 = System.currentTimeMillis();
-        calculateCtcvrScore(result, request, scoreParam);
+        calculateCtcvrScore(result, request, scoreParam, null);
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -242,7 +242,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             item.getScoreMap().put("ctcvrScore", calibratedScore);
         }
 
-        calculateCtcvrScore(result, request, scoreParam);
+        calculateCtcvrScore(result, request, scoreParam, null);
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -239,7 +239,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             item.getScoreMap().put("ctcvrScore", calibratedScore);
         }
 
-        calculateCtcvrScore(result, request, scoreParam);
+        calculateCtcvrScore(result, request, scoreParam, null);
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
         boolean isGuaranteeType = false;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -294,7 +294,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             item.getScoreMap().put("ctcvrScore", calibratedScore);
         }
 
-        calculateCtcvrScore(result, request, scoreParam);
+        calculateCtcvrScore(result, request, scoreParam, "dnn");
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
         boolean isGuaranteeType = false;