Merge branch 'feature_20250725_peopleAddWeight' into pre-master

# Conflicts:
#	ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/CorrectCpaParam.java

@@ -4,8 +4,9 @@ import lombok.Data;
 
 @Data
 public class CorrectCpaParam {
-    private Integer view;
     private Double correctionFactor;
+    private Integer viewsHour;
+    private Integer viewsDay;
     private Double realCtcvrHour;
     private Double pCtcvrHour;
     private Double realCtcvrDay;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

@@ -68,8 +68,15 @@ public abstract class RankStrategyBasic implements RankStrategy {
     @Value("${correct.cpa.beta.2:0.2}")
     protected Double correctCpaBeta2;
 
-    @Value("${correct.cpa.view:1000}")
-    protected Integer correctCpaView;
+    @Value("${correct.cpa.view.hour:300}")
+    protected Integer correctCpaViewHour;
+
+    @Value("${correct.cpa.view.day:300}")
+    protected Integer correctCpaViewDay;
+
+    @Value("${correct.cpa.min.ctcvr:0.000001}")
+    protected Double minCtcvr;
+
 
     @Value("${exclude.exp:772}")
     protected String excludeExp;
@@ -106,13 +113,14 @@ public abstract class RankStrategyBasic implements RankStrategy {
     String adCustomerLayerHourKey = "ad:platform:customer:hour:{layer}:{clazz}:{customerId}";
 
     String adVerLayerHourKey = "ad:platform:adver:hour:{layer}:{clazz}:{adverId}";
+    String adLayerHourKey = "ad:platform:ad:hour:{layer}:{clazz}:{adId}";
 
-    String adCustomerLayerDayKey = "ad:platform:customer:day:{layer}:{clazz}:{customerId}";
-
-    String adVerLayerDayKey = "ad:platform:adver:day:{layer}:{clazz}:{adverId}";
+    String adLayerDayKey = "ad:platform:ad:day:{layer}:{clazz}:{adId}";
 
     String cidLayerKey = "ad:engine:cid:layer:info:{cid}:{userLayer}";
 
+    private static final double DEFAULT_CORRECTION = 1.0;
+
 
     protected static final List<String> hasChannelScenes = new ArrayList<String>() {{
         add("DaiTou");
@@ -388,94 +396,69 @@ public abstract class RankStrategyBasic implements RankStrategy {
         if (request.getIsFilterUser()) {
             layer = layer + "-炸";
         }
-
-        String redisAdCustomerLayerHourKey = adCustomerLayerHourKey.replace("{layer}", layer).replace("{clazz}", clazz);
-        String redisAdVerLayerHourKey = adVerLayerHourKey.replace("{layer}", layer).replace("{clazz}", clazz);
-        String redisAadCustomerLayerDayKey = adCustomerLayerDayKey.replace("{layer}", layer).replace("{clazz}", clazz);
-        String redisAdVerLayerDayKey = adVerLayerDayKey.replace("{layer}", layer).replace("{clazz}", clazz);
-
-
         Map<Long, CorrectCpaParam> resultMap = new HashMap<>();
         try {
             if (CollectionUtils.isEmpty(request.getAdIdList())) {
                 return resultMap;
             }
-
-            // 抽取公共方法获取数据
-            List<Long> customerIds = request.getAdIdList().stream()
-                    .map(AdPlatformCreativeDTO::getCustomerId)
+            String redisAdLayerHourKey = adLayerHourKey.replace("{layer}", layer).replace("{clazz}", clazz);
+            String redisAdLayerDayKey = adLayerDayKey.replace("{layer}", layer).replace("{clazz}", clazz);
+            Function<Long, String> hourKeyFunc = id -> redisAdLayerHourKey.replace("{adId}", String.valueOf(id));
+            Function<Long, String> dayKeyFunc = id -> redisAdLayerDayKey.replace("{adId}", String.valueOf(id));
+            List<Long> adIds = request.getAdIdList().stream()
+                    .map(AdPlatformCreativeDTO::getAdId)
                     .distinct()
                     .collect(Collectors.toList());
-            Map<Long, JSONObject> customerLayerMap = getRedisData(
-                    customerIds,
-                    id -> redisAdCustomerLayerHourKey.replace("{customerId}", String.valueOf(id)),
-                    id -> redisAadCustomerLayerDayKey.replace("{customerId}", String.valueOf(id))
-            );
-
-            List<String> adVerIds = request.getAdIdList().stream()
-                    .map(AdPlatformCreativeDTO::getAdVerId)
-                    .distinct()
-                    .collect(Collectors.toList());
-            Map<String, JSONObject> adVerLayerMap = getRedisData(
-                    adVerIds,
-                    id -> redisAdVerLayerHourKey.replace("{adverId}", id),
-                    id -> redisAdVerLayerDayKey.replace("{adverId}", id)
-            );
-
-            for (AdPlatformCreativeDTO adPlatformCreativeDTO : request.getAdIdList()) {
-                Long creativeId = adPlatformCreativeDTO.getCreativeId();
-                Long customerId = adPlatformCreativeDTO.getCustomerId();
-                JSONObject jsonObject;
-                if (customerId != null && customerId != 0 && customerLayerMap.containsKey(customerId)) {
-                    jsonObject = customerLayerMap.get(customerId);
-                } else {
-                    jsonObject = adVerLayerMap.get(adPlatformCreativeDTO.getAdVerId());
-                }
-                CorrectCpaParam correctCpaParam = new CorrectCpaParam();
+            Map<Long, JSONObject> adLayerMap = getRedisData(adIds, hourKeyFunc, dayKeyFunc);
+            adIds.forEach(adId -> {
+                JSONObject jsonObject = adLayerMap.get(adId);
+                CorrectCpaParam param = new CorrectCpaParam();
+
                 if (jsonObject == null) {
-                    correctCpaParam.setCorrectionFactor(1.0);
-                    resultMap.put(creativeId, correctCpaParam);
-                    continue;
-                }
-                Integer views = jsonObject.getInteger("viewsHour");
-                Double realCtcvrHour = jsonObject.getDouble("realCtcvrHour");
-                Double pCtcvrHour = jsonObject.getDouble("pCtcvrHour");
-                Double realCtcvrDay = jsonObject.getDouble("realCtcvrDay");
-                Double pCtcvrDay = jsonObject.getDouble("pCtcvrDay");
-                correctCpaParam.setRealCtcvrHour(realCtcvrHour);
-                correctCpaParam.setPCtcvrHour(pCtcvrHour);
-                correctCpaParam.setRealCtcvrDay(realCtcvrDay);
-                correctCpaParam.setPCtcvrDay(pCtcvrDay);
-                correctCpaParam.setView(views);
-                double correctionFactor = 1.0;
-                //曝光数小于目标曝光数，不进行修正
-                if (views == null || views < correctCpaView) {
-                    correctCpaParam.setCorrectionFactor(correctionFactor);
-                    resultMap.put(creativeId, correctCpaParam);
-                    continue;
-                }
-                if (realCtcvrHour != null && pCtcvrHour != null && realCtcvrDay != null && pCtcvrDay != null) {
-                    if (scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
-                        correctionFactor = (1 - correctCpaAlpha2 - correctCpaBeta2) +
-                                NumUtil.div(realCtcvrHour, pCtcvrHour) * correctCpaAlpha2 +
-                                NumUtil.div(realCtcvrDay, pCtcvrDay) * correctCpaBeta2;
-                    } else {
-                        correctionFactor = Math.pow(NumUtil.div(realCtcvrHour, pCtcvrHour), correctCpaAlpha1) *
-                                Math.pow(NumUtil.div(realCtcvrDay, pCtcvrDay), correctCpaBeta1);
-                    }
-                    if (correctionFactor <= 0) {
-                        correctionFactor = 1;
-                    }
+                    param.setCorrectionFactor(DEFAULT_CORRECTION);
+                } else {
+                    // 设置基础参数
+                    param.setViewsHour(jsonObject.getInteger("viewsHour"));
+                    param.setViewsDay(jsonObject.getInteger("viewsDay"));
+                    param.setRealCtcvrHour(jsonObject.getDouble("realCtcvrHour"));
+                    param.setPCtcvrHour(jsonObject.getDouble("pCtcvrHour"));
+                    param.setRealCtcvrDay(jsonObject.getDouble("realCtcvrDay"));
+                    param.setPCtcvrDay(jsonObject.getDouble("pCtcvrDay"));
+                    // 计算校正因子
+                    double factor = calculateCorrection(jsonObject, scoreParam);
+                    param.setCorrectionFactor(factor > 0 ? factor : DEFAULT_CORRECTION);
                 }
-                correctCpaParam.setCorrectionFactor(correctionFactor);
-                resultMap.put(creativeId, correctCpaParam);
-            }
+                resultMap.put(adId, param);
+            });
         } catch (Exception e) {
             log.error("getCorrectCpaParamMap error", e);
         }
         return resultMap;
     }
 
+    private double calculateCorrection(JSONObject jsonObject, ScoreParam scoreParam) {
+        // 安全获取值
+        Integer viewsHour = jsonObject.getInteger("viewsHour");
+        Integer viewsDay = jsonObject.getInteger("viewsDay");
+        double realCtcvrHour = safeDouble(jsonObject.getDouble("realCtcvrHour"), minCtcvr);
+        double pCtcvrHour = safeDouble(jsonObject.getDouble("pCtcvrHour"), minCtcvr);
+        double realCtcvrDay = safeDouble(jsonObject.getDouble("realCtcvrDay"), minCtcvr);
+        double pCtcvrDay = safeDouble(jsonObject.getDouble("pCtcvrDay"), minCtcvr);
+
+        if (scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
+            double alpha = isValidViews(viewsHour, correctCpaViewHour) ? correctCpaAlpha2 : 0;
+            double beta = isValidViews(viewsDay, correctCpaViewDay) ? correctCpaBeta2 : 0;
+            return (1 - alpha - beta) +
+                    (realCtcvrHour / pCtcvrHour) * alpha +
+                    (realCtcvrDay / pCtcvrDay) * beta;
+        } else {
+            double alpha = isValidViews(viewsHour, correctCpaViewHour) ? correctCpaAlpha1 : 0;
+            double beta = isValidViews(viewsDay, correctCpaViewDay) ? correctCpaBeta1 : 0;
+            return Math.pow(realCtcvrHour / pCtcvrHour, alpha) *
+                    Math.pow(realCtcvrDay / pCtcvrDay, beta);
+        }
+    }
+
     private <T> Map<T, JSONObject> getRedisData(
             Collection<T> ids,
             Function<T, String> hourKeyBuilder,
@@ -538,11 +521,19 @@ public abstract class RankStrategyBasic implements RankStrategy {
         log.info("calibrationCtcvrExp={}", calibrationCtcvrExp);
         log.info("scoreParam.getExpCodeSet={}", scoreParam.getExpCodeSet());
         if (scoreParam.getExpCodeSet().contains(calibrationCtcvrExp)) {
-            calibrationDnnCtcvrScore(items, request.getMid(), request.getIsFilterUser(), modelName);
+            try {
+                calibrationDnnCtcvrScore(items, request.getMid(), request.getIsFilterUser(), modelName);
+            } catch (Exception e) {
+                log.error("calibrationDnnCtcvrScore error", e);
+            }
         }
 
         if (scoreParam.getExpCodeSet().contains(calibrationCoefficientExp)) {
-            calibrationCtcvrScore(items, request);
+            try {
+                calibrationCtcvrScore(items, request);
+            } catch (Exception e) {
+                log.error("calibrationCtcvrScore error", e);
+            }
         }
     }
 
@@ -602,8 +593,8 @@ public abstract class RankStrategyBasic implements RankStrategy {
             double realCtcvr = Optional.ofNullable(param.getRealCtcvr()).orElse(0.0);
             double calibratedScore = item.getLrScore() * calibrationAlpha + (1 - calibrationAlpha) * realCtcvr;
             item.getExt().put("correctCtcvrScoreParam", JSONObject.toJSONString(param));
+            item.getScoreMap().put("cidCorrectCtcvrScore", calibratedScore);
             item.getScoreMap().put("ctcvrScore", calibratedScore);
-            item.getScoreMap().put("realCtcvr", realCtcvr);
             item.setLrScore(calibratedScore);
         }
     }
@@ -752,8 +743,19 @@ public abstract class RankStrategyBasic implements RankStrategy {
                 continue;
             }
             double correctCtcvrScore = item.getLrScore() * diff;
-            item.getScoreMap().put("correctCtcvrScore", correctCtcvrScore);
+            item.getScoreMap().put("layerCorrectCtcvrScore", correctCtcvrScore);
+            item.getScoreMap().put("ctcvrScore", correctCtcvrScore);
             item.setLrScore(correctCtcvrScore);
         }
     }
+
+    // 安全的数值转换
+    private double safeDouble(Double value, Double min) {
+        return (value == null || value == 0) ? min : value;
+    }
+
+    // 视图数校验封装
+    private boolean isValidViews(Integer views, int threshold) {
+        return views != null && views >= threshold;
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -126,8 +126,8 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                         adRankItem.getExt().put("isApi", "1");
                     }
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getCreativeId())));
-                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getCreativeId()).getCorrectionFactor());
+                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
                     setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -138,8 +138,8 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                     }
 
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getCreativeId())));
-                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getCreativeId()).getCorrectionFactor());
+                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
                     setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -135,8 +135,8 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     }
 
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getCreativeId())));
-                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getCreativeId()).getCorrectionFactor());
+                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
                     setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -51,6 +51,13 @@ public class RankStrategyBy688 extends RankStrategyBasic {
     @ApolloJsonValue("${rank.score.weight.680:{}}")
     private Map<String, Double> weightMap;
 
+    /**
+     * 人群分层&创意的权重
+     * 格式：{layer_creativeId: weight}
+     */
+    @ApolloJsonValue("${rank.score.weight.layer.and.creative:{}}")
+    private Map<String, Double> layerAndCreativeWeightMap;
+
     @ApolloJsonValue("${rank.score.neg_sample_rate:0.01}")
     Double negSampleRate;
 
@@ -172,8 +179,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                         adRankItem.getExt().put("isApi", "1");
                     }
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getCreativeId())));
-                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getCreativeId()).getCorrectionFactor());
+                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
                     setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
@@ -300,6 +307,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
         boolean isGuaranteeType = false;
+        // 查询人群分层信息
+        String peopleLayer = getUserLayerAfterHandle(request);
         for (AdRankItem item : result) {
             double bid = item.getCpa();
             if (scoreParam.getExpCodeSet().contains(correctCpaExp1) || scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
@@ -311,9 +320,13 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             if (isGuaranteedFlow && item.getExt().get("isGuaranteed") != null && (boolean) item.getExt().get("isGuaranteed")) {
                 isGuaranteeType = true;
             }
+
+            String layerAndCreativeWeightMapKey = getLayerAndCreativeWeightMapKey(peopleLayer, String.valueOf(item.getAdId()));
+            // 人群分层&创意的权重
+            double layerAndCreativeWeight = getLayerAndCreativeWeight(layerAndCreativeWeightMapKey);
             double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
             double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(isGuaranteedFlow, item.getExt());
-            double score = item.getLrScore() * bid * scoreCoefficient * guaranteeScoreCoefficient;
+            double score = item.getLrScore() * bid * scoreCoefficient * guaranteeScoreCoefficient * layerAndCreativeWeight;
             item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
@@ -337,6 +350,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             AdRankItem top1Item = result.get(0);
             top1Item.getExt().put("isGuaranteeType", isGuaranteeType);
             putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+            top1Item.getExt().put("model", "dnn");
         }
         long time6 = System.currentTimeMillis();
         log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +
@@ -347,6 +361,47 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         return result;
     }
 
+    /**
+     * 获取人群分层和创意的权重
+     * @param key
+     * @return
+     */
+    private Double getLayerAndCreativeWeight(String key){
+        if (StringUtils.isBlank(key)) {
+            return 1d;
+        }
+        return layerAndCreativeWeightMap.getOrDefault(key, 1d);
+    }
+
+    /**
+     * 获取人群分层和创意的权重key
+     * @param layer
+     * @param creativeId
+     * @return
+     */
+    private String getLayerAndCreativeWeightMapKey(String layer, String creativeId){
+        if ( StringUtils.isBlank(layer) || StringUtils.isBlank(creativeId)) {
+            return null;
+        }
+        return layer + "_" + creativeId;
+    }
+
+    /**
+     * 获取处理后的人群分层
+     * @param request
+     * @return
+     */
+    private String getUserLayerAfterHandle(RankRecommendRequestParam request){
+        Map<String, String> userLayer = this.getUserLayer(request.getMid());
+        String layer = userLayer.getOrDefault("layer", "无曝光");
+        if (StringUtils.isNotEmpty(layer) && layer.equals("已转化")) {
+            layer = "有转化";
+        }
+        return layer;
+    }
+
+
+
     private void handleB1Feature(Map<String, String> b1Feature, Map<String, String> cidFeatureMap, String cid) {
         cidFeatureMap.put("cid_" + cid, "0.1");
         // if (StringUtils.isNotBlank(b1Feature.get("adid"))) {