Merge branch 'refs/heads/dev-xym-calibration-dnn' into pre-master

# Conflicts:
#	ad-engine-commons/pom.xml
#	ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java
#	ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java
#	ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java
#	ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java
#	ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

ad-engine-commons/pom.xml

@@ -26,7 +26,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.1.23</version>
+            <version>1.1.24</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/CalibrationCtcvrData.java

@@ -0,0 +1,13 @@
+package com.tzld.piaoquan.ad.engine.service.entity;
+
+import lombok.Data;
+
+@Data
+public class CalibrationCtcvrData {
+
+    private Integer view;
+
+    private Double realCtcvr;
+
+    private Double pCtcvr;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

@@ -13,6 +13,7 @@ import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.commons.util.ObjUtil;
+import com.tzld.piaoquan.ad.engine.service.entity.CalibrationCtcvrData;
 import com.tzld.piaoquan.ad.engine.service.entity.CorrectCpaParam;
 import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
@@ -40,10 +41,15 @@ public abstract class RankStrategyBasic implements RankStrategy {
 
     @Value("${guarantee.exp:742}")
     protected String guaranteeExp;
-
     @ApolloJsonValue("${alpha:1.0}")
     protected Double alpha;
 
+    @Value("${calibration:ctcvr.exp:779}")
+    protected String calibrationCtcvrExp;
+
+    @ApolloJsonValue("${calibration.view.count:5000}")
+    protected Integer calibrationViewCount;
+
     @Value("${correct.cpa.exp.1:}")
     protected String correctCpaExp1;
 
@@ -74,6 +80,11 @@ public abstract class RankStrategyBasic implements RankStrategy {
 
     String adPlatformGuaranteeKey = "ad:platform:guarantee:data:{date}:{adverId}";
 
+    String realCtcvrCustomerKey = "ad:platform:real:ctcvr:{model}:{layer}:{class}:{profession}:{customerId}";
+
+    String realCtcvrProfessionKey = "ad:platform:real:ctcvr:{model}:{layer}:{class}:{profession}";
+
+
     String adCustomerLayerHourKey = "ad:platform:customer:hour:{layer}:{clazz}:{customerId}";
 
     String adVerLayerHourKey = "ad:platform:adver:hour:{layer}:{clazz}:{adverId}";
@@ -306,166 +317,6 @@ public abstract class RankStrategyBasic implements RankStrategy {
         }
     }
 
-    protected Map<Long, CorrectCpaParam> getCorrectCpaParamMap(RankRecommendRequestParam request, ScoreParam scoreParam) {
-        Map<String, String> userLayer = this.getUserLayer(request.getMid());
-
-        String layer = userLayer.getOrDefault("layer", "无曝光");
-        String clazz = userLayer.getOrDefault("class", "近期未出现");
-        if (StringUtils.isNotEmpty(layer) && layer.equals("已转化")) {
-            layer = "有转化";
-        }
-        if (request.getIsFilterUser()) {
-            layer = layer + "-炸";
-        }
-
-        String redisAdCustomerLayerHourKey = adCustomerLayerHourKey.replace("{layer}", layer).replace("{clazz}", clazz);
-        String redisAdVerLayerHourKey = adVerLayerHourKey.replace("{layer}", layer).replace("{clazz}", clazz);
-        String redisAadCustomerLayerDayKey = adCustomerLayerDayKey.replace("{layer}", layer).replace("{clazz}", clazz);
-        String redisAdVerLayerDayKey = adVerLayerDayKey.replace("{layer}", layer).replace("{clazz}", clazz);
-
-
-        Map<Long, CorrectCpaParam> resultMap = new HashMap<>();
-        try {
-            if (CollectionUtils.isEmpty(request.getAdIdList())) {
-                return resultMap;
-            }
-
-            // 抽取公共方法获取数据
-            List<Long> customerIds = request.getAdIdList().stream()
-                    .map(AdPlatformCreativeDTO::getCustomerId)
-                    .distinct()
-                    .collect(Collectors.toList());
-            Map<Long, JSONObject> customerLayerMap = getRedisData(
-                    customerIds,
-                    id -> redisAdCustomerLayerHourKey.replace("{customerId}", String.valueOf(id)),
-                    id -> redisAadCustomerLayerDayKey.replace("{customerId}", String.valueOf(id))
-            );
-
-            List<String> adVerIds = request.getAdIdList().stream()
-                    .map(AdPlatformCreativeDTO::getAdVerId)
-                    .distinct()
-                    .collect(Collectors.toList());
-            Map<String, JSONObject> adVerLayerMap = getRedisData(
-                    adVerIds,
-                    id -> redisAdVerLayerHourKey.replace("{adverId}", id),
-                    id -> redisAdVerLayerDayKey.replace("{adverId}", id)
-            );
-            log.info("getCorrectCpaParamMap customerLayerMap={}", customerLayerMap);
-            log.info("getCorrectCpaParamMap adVerLayerMap={}", adVerLayerMap);
-
-            for (AdPlatformCreativeDTO adPlatformCreativeDTO : request.getAdIdList()) {
-                Long creativeId = adPlatformCreativeDTO.getCreativeId();
-                Long customerId = adPlatformCreativeDTO.getCustomerId();
-                JSONObject jsonObject;
-                if (customerId != null && customerId != 0 && customerLayerMap.containsKey(customerId)) {
-                    jsonObject = customerLayerMap.get(customerId);
-                } else {
-                    jsonObject = adVerLayerMap.get(adPlatformCreativeDTO.getAdVerId());
-                }
-                log.info("getCorrectCpaParamMap cid={}, jsonObject={}", adPlatformCreativeDTO.getCreativeId(), jsonObject);
-                CorrectCpaParam correctCpaParam = new CorrectCpaParam();
-                if (jsonObject == null) {
-                    correctCpaParam.setCorrectionFactor(1.0);
-                    resultMap.put(creativeId, correctCpaParam);
-                    continue;
-                }
-                Integer views = jsonObject.getInteger("viewsHour");
-                Double realCtcvrHour = jsonObject.getDouble("realCtcvrHour");
-                Double pCtcvrHour = jsonObject.getDouble("pCtcvrHour");
-                Double realCtcvrDay = jsonObject.getDouble("realCtcvrDay");
-                Double pCtcvrDay = jsonObject.getDouble("pCtcvrDay");
-                correctCpaParam.setRealCtcvrHour(realCtcvrHour);
-                correctCpaParam.setPCtcvrHour(pCtcvrHour);
-                correctCpaParam.setRealCtcvrDay(realCtcvrDay);
-                correctCpaParam.setPCtcvrDay(pCtcvrDay);
-                correctCpaParam.setView(views);
-                double correctionFactor = 1.0;
-                //曝光数小于目标曝光数，不进行修正
-                if (views == null || views < correctCpaView) {
-                    correctCpaParam.setCorrectionFactor(correctionFactor);
-                    resultMap.put(creativeId, correctCpaParam);
-                    continue;
-                }
-                if (realCtcvrHour != null && pCtcvrHour != null && realCtcvrDay != null && pCtcvrDay != null) {
-                    if (scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
-                        correctionFactor = (1 - correctCpaAlpha2 - correctCpaBeta2) +
-                                NumUtil.div(realCtcvrHour, pCtcvrHour) * correctCpaAlpha2 +
-                                NumUtil.div(realCtcvrDay, pCtcvrDay) * correctCpaBeta2;
-                    } else {
-                        correctionFactor = Math.pow(NumUtil.div(realCtcvrHour, pCtcvrHour), correctCpaAlpha1) *
-                                Math.pow(NumUtil.div(realCtcvrDay, pCtcvrDay), correctCpaBeta1);
-                    }
-                    log.info("calculate correctionFactor={}", correctionFactor);
-                    if (correctionFactor <= 0) {
-                        correctionFactor = 1;
-                    }
-                }
-                correctCpaParam.setCorrectionFactor(correctionFactor);
-                resultMap.put(creativeId, correctCpaParam);
-            }
-        } catch (Exception e) {
-            log.error("getCorrectCpaParamMap error", e);
-        }
-        return resultMap;
-    }
-
-    private <T> Map<T, JSONObject> getRedisData(
-            Collection<T> ids,
-            Function<T, String> hourKeyBuilder,
-            Function<T, String> dayKeyBuilder) {
-
-        Map<T, JSONObject> resultMap = new HashMap<>();
-        if (CollectionUtils.isEmpty(ids)) {
-            return resultMap;
-        }
-
-        // 构建Keys
-        List<String> hourKeys = ids.stream()
-                .map(hourKeyBuilder)
-                .collect(Collectors.toList());
-        List<String> dayKeys = ids.stream()
-                .map(dayKeyBuilder)
-                .collect(Collectors.toList());
-
-        // 批量获取Redis值
-        List<String> hourValues = adRedisHelper.mget(hourKeys);
-        List<String> dayValues = adRedisHelper.mget(dayKeys);
-
-        // 解析数据
-        Iterator<T> idIter = ids.iterator();
-        for (int i = 0; i < ids.size(); i++) {
-            T id = idIter.next();
-            JSONObject jsonObj = new JSONObject();
-
-            // 解析小时数据
-            parseRedisValue(hourValues.get(i), jsonObj, "Hour", "ctcvr", "pCtcvr", "views");
-            // 解析天数据
-            parseRedisValue(dayValues.get(i), jsonObj, "Day", "ctcvr", "pCtcvr", "views");
-
-            resultMap.put(id, jsonObj);
-        }
-        return resultMap;
-    }
-
-    private void parseRedisValue(String value, JSONObject target,
-                                 String suffix, String... keys) {
-        if (StringUtils.isEmpty(value)) return;
-
-        try {
-            JSONObject source = JSONObject.parseObject(value);
-            for (String key : keys) {
-                Object val = source.get(key);
-                if (val != null) {
-                    String newKey = (key.equals("ctcvr") ? "realCtcvr" : key) + suffix;
-                    target.put(newKey, val);
-                }
-            }
-        } catch (Exception e) {
-            log.warn("Parse redis value failed: {}", value, e);
-        }
-    }
-
-
     protected AdRankItem creativeCovertRankItem(AdPlatformCreativeDTO dto, RankRecommendRequestParam request, Set<String> noApiAdVerIds) {
         AdRankItem adRankItem = new AdRankItem();
         adRankItem.setAdId(dto.getCreativeId());
@@ -561,4 +412,114 @@ public abstract class RankStrategyBasic implements RankStrategy {
             return new HashMap<>();
         }
     }
+
+    protected void calibrationCtcvrScore(ScoreParam scoreParam, List<AdRankItem> adRankItems, String mid, boolean isFilterUser, String modelName) {
+        //判断是否走校准试验
+        if (!scoreParam.getExpCodeSet().contains(calibrationCtcvrExp)) {
+            return;
+        }
+        // 1. 获取用户分层信息
+        Map<String, String> userLayer = getUserLayer(mid);
+        String layer = userLayer.getOrDefault("layer", "无曝光");
+        String clazz = userLayer.getOrDefault("class", "近期未出现");
+        if (isFilterUser) {
+            layer += "-炸";
+        }
+
+        // 2. 构建Key模板
+        String customerKeyTemplate = realCtcvrCustomerKey
+                .replace("{model}", modelName)
+                .replace("{layer}", layer)
+                .replace("{class}", clazz);
+
+        String professionKeyTemplate = realCtcvrProfessionKey
+                .replace("{model}", modelName)
+                .replace("{layer}", layer)
+                .replace("{class}", clazz);
+
+        // 3. 定义Key生成器
+        Function<AdRankItem, String> customerKeyFunc = e ->
+                (e.getProfession() != null && e.getCustomerId() != null)
+                        ? customerKeyTemplate.replace("{profession}", e.getProfession())
+                        .replace("{customerId}", String.valueOf(e.getCustomerId()))
+                        : null;
+
+        Function<AdRankItem, String> professionKeyFunc = e ->
+                (e.getProfession() != null)
+                        ? professionKeyTemplate.replace("{profession}", e.getProfession())
+                        : null;
+
+        // 4. 获取校准数据
+        Map<String, CalibrationCtcvrData> customerMap = getCtcvrMap(adRankItems, customerKeyFunc);
+        Map<String, CalibrationCtcvrData> professionMap = getCtcvrMap(adRankItems, professionKeyFunc);
+        // 5. 校准分数
+        for (AdRankItem item : adRankItems) {
+            CalibrationCtcvrData data = null;
+            String customerKey = customerKeyFunc.apply(item);
+            String professionKey = professionKeyFunc.apply(item);
+
+            if (customerKey != null) {
+                data = customerMap.get(customerKey);
+            }
+            if (data == null && professionKey != null) {
+                data = professionMap.get(professionKey);
+            }
+            item.getExt().put("calibrationCtcvrData", JSONObject.toJSONString(data));
+            if (data == null || data.getView() == null || data.getView() < calibrationViewCount) continue;
+
+            Map<String, Double> scoreMap = item.getScoreMap();
+            Double pCtcvr = data.getPCtcvr();
+            Double realCtcvr = data.getRealCtcvr();
+
+            if (pCtcvr != null && pCtcvr != 0.0 && realCtcvr != null && realCtcvr != 0.0) {
+                double diff = realCtcvr / pCtcvr;
+                if (Math.abs(diff - 1) >= 0.1) {
+                    Double ctcvrScore = scoreMap.get("ctcvrScore");
+                    if (ctcvrScore == null) {
+                        continue;
+                    }
+                    scoreMap.put("modelCtcvrScore", ctcvrScore);
+                    scoreMap.put("ctcvrScore", ctcvrScore * diff);
+                    item.setLrScore(ctcvrScore * diff);
+                }
+            }
+        }
+    }
+
+    // 通用CTR数据获取方法
+    private Map<String, CalibrationCtcvrData> getCtcvrMap(
+            List<AdRankItem> items,
+            Function<AdRankItem, String> keyGenerator
+    ) {
+        // 生成有效Key集合
+        List<String> redisKeys = items.stream()
+                .map(keyGenerator)
+                .filter(Objects::nonNull)
+                .distinct()
+                .collect(Collectors.toList());
+
+        // 批量查询Redis
+        List<String> redisValues = adRedisHelper.mget(redisKeys);
+        Map<String, CalibrationCtcvrData> resultMap = new HashMap<>();
+
+        for (int i = 0; i < redisKeys.size(); i++) {
+            String val = redisValues.get(i);
+            if (StringUtils.isEmpty(val)) continue;
+
+            try {
+                JSONObject json = JSONObject.parseObject(val);
+                Integer view = json.getInteger("view");
+                Double ctcvr = json.getDouble("ctcvr");
+                Double pCtcvr = json.getDouble("pCtcvr");
+                CalibrationCtcvrData data = new CalibrationCtcvrData();
+                data.setView(view);
+                data.setRealCtcvr(ctcvr);
+                data.setPCtcvr(pCtcvr);
+                resultMap.put(redisKeys.get(i), data);
+            } catch (Exception e) {
+                log.error("getCtcvrMap error", e);
+            }
+        }
+        return resultMap;
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -115,6 +115,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
                     adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setCustomerId(dto.getCustomerId());
+                    adRankItem.setProfession(dto.getProfession());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -151,6 +151,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
                     adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setCustomerId(dto.getCustomerId());
+                    adRankItem.setProfession(dto.getProfession());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -259,6 +260,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             double calibratedScore = originalScore / (originalScore + (1 - originalScore) / negSampleRate);
             item.setLrScore(calibratedScore);
             item.getScoreMap().put("originCtcvrScore", originalScore);
+            item.getScoreMap().put("modelCtcvrScore", calibratedScore);
             item.getScoreMap().put("ctcvrScore", calibratedScore);
         }
 
@@ -290,11 +292,13 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             item.setScore(score);
         }
 
+
         result.sort(ComparatorUtil.equalsRandomComparator());
 
         if (CollectionUtils.isNotEmpty(result)) {
             AdRankItem top1Item = result.get(0);
             putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+            top1Item.getExt().put("model", "xgb");
         }
         long time6 = System.currentTimeMillis();
         log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -124,6 +124,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
                     adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setCustomerId(dto.getCustomerId());
+                    adRankItem.setProfession(dto.getProfession());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -188,6 +188,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
                     adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setCustomerId(dto.getCustomerId());
+                    adRankItem.setProfession(dto.getProfession());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -317,6 +318,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             item.getScoreMap().put("originCtcvrScore", originalScore);
             item.getScoreMap().put("ctcvrScore", calibratedScore);
         }
+        //校准Ctcvr
+        calibrationCtcvrScore(scoreParam, result, request.getMid(), request.getIsFilterUser(), "dnn");
 
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
@@ -352,6 +355,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         if (CollectionUtils.isNotEmpty(result)) {
             AdRankItem top1Item = result.get(0);
             putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+            top1Item.getExt().put("model", "dnn");
         }
         long time6 = System.currentTimeMillis();
         log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +