Merge branch '20250729_feature_fjy_guarantee' into test

merge 20250729_feature_fjy_guarantee

.gitignore

@@ -37,4 +37,6 @@ weblog
 logs
 LOG_PATH_IS_UNDEFINED
 
-xgboost
+xgboost
+xgboost351
+word2vec.bin

ad-engine-commons/pom.xml

@@ -11,7 +11,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>ad-engine-commons</artifactId>
-    <version>1.1.0</version>
+    <version>1.1.1</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
@@ -26,7 +26,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.1.21</version>
+            <version>1.1.24</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/config/OssConfig.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.ad.engine.commons.config;
+
+import com.aliyun.oss.ClientBuilderConfiguration;
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.CredentialsProviderFactory;
+import com.aliyun.oss.common.auth.DefaultCredentialProvider;
+import com.aliyun.oss.common.comm.SignVersion;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class OssConfig {
+
+    @Value("${oss.endpoint}")
+    private String endpoint;
+
+    @Value("${oss.adplatform.accessKey}")
+    private String accessKey;
+
+    @Value("${oss.adplatform.secretKey}")
+    private String secretKey;
+
+    @Bean
+    public OSS ossClient() {
+        DefaultCredentialProvider defaultCredentialProvider = CredentialsProviderFactory.newDefaultCredentialProvider(accessKey, secretKey);
+        ClientBuilderConfiguration config = new ClientBuilderConfiguration();
+        config.setSignatureVersion(SignVersion.V4);
+        return OSSClientBuilder.create()
+                .endpoint(endpoint)
+                .credentialsProvider(defaultCredentialProvider)
+                .clientConfiguration(config)
+                .region("cn-hangzhou")
+                .build();
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/dto/AdPlatformCreativeDTO.java

@@ -43,4 +43,13 @@ public class AdPlatformCreativeDTO {
 
     // 召回源，记录召回用
     private Object recallSources;
+
+    // 广告主行业
+    private String profession;
+
+    private Long skuId;
+
+    private String customer;
+
+    private Long customerId;
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/enums/CrowdLayerEnum.java

@@ -0,0 +1,53 @@
+package com.tzld.piaoquan.ad.engine.commons.enums;
+
+/**
+ * 人群分层枚举
+ */
+public enum CrowdLayerEnum {
+
+
+    NO_EXPOSURE(1, "无曝光"),
+    WITH_CONVERSION(2, "有转化"),
+    WITH_EXPOSURE_NO_CONVERSION(3, "有曝光无转化"),
+    ;
+    private Integer code;
+    private String desc;
+
+    CrowdLayerEnum(Integer code, String desc) {
+        this.code = code;
+        this.desc = desc;
+    }
+
+    public Integer getCode() {
+        return code;
+    }
+
+    public String getDesc() {
+        return desc;
+    }
+
+    public static CrowdLayerEnum getByCode(Integer code) {
+        if (code == null) {
+            return null;
+        }
+        for (CrowdLayerEnum constants : values()) {
+            if (constants.getCode().equals(code)) {
+                return constants;
+            }
+        }
+        return null;
+    }
+
+    public static CrowdLayerEnum getBytDesc( String desc) {
+        if (desc == null) {
+            return null;
+        }
+        for (CrowdLayerEnum constants : values()) {
+            if (constants.getDesc().equals(desc)) {
+                return constants;
+            }
+        }
+        return null;
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/enums/ExpCodeEnum.java → ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/enums/ExpCodeEnum.java

@@ -1,4 +1,4 @@
-package com.tzld.piaoquan.ad.engine.service.predict.enums;
+package com.tzld.piaoquan.ad.engine.commons.enums;
 
 public enum ExpCodeEnum {
 

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/enums/GuaranteedTypeEnum.java

@@ -0,0 +1,24 @@
+package com.tzld.piaoquan.ad.engine.commons.enums;
+
+public enum GuaranteedTypeEnum {
+
+    IS_GUARANTEED(1, "保量广告"),
+    NOT_GUARANTEED(0, "非保量广告"),
+
+    ;
+    private final Integer type;
+    private final String desc;
+
+    GuaranteedTypeEnum(Integer type, String desc) {
+        this.type = type;
+        this.desc = desc;
+    }
+
+    public Integer getType() {
+        return type;
+    }
+
+    public String getDesc() {
+        return desc;
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/helper/DnnCidDataHelper.java

@@ -0,0 +1,149 @@
+package com.tzld.piaoquan.ad.engine.commons.helper;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.model.ListObjectsRequest;
+import com.aliyun.oss.model.OSSObject;
+import com.aliyun.oss.model.OSSObjectSummary;
+import com.aliyun.oss.model.ObjectListing;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+@Slf4j
+@Component
+public class DnnCidDataHelper {
+
+
+    @Autowired
+    private OSS ossClient;
+
+    private static final String BUCKET_NAME = "art-recommend";
+
+    private static final String modelVersionPath = "fengzhoutian/pai_model_trained_cids/model_version.json";
+
+    private static final String cidPath = "fengzhoutian/pai_model_trained_cids/";
+
+
+    // 提供获取CID集合的方法
+    // 全局变量，存储CSV文件中的数据
+    @Getter
+    private volatile static Set<Long> cidSet = Collections.emptySet();
+
+    // 服务启动时初始化数据
+    @PostConstruct
+    public void init() {
+        updateCidSet();
+    }
+
+    // 每10分钟更新一次数据
+    @Scheduled(fixedRate = 10 * 60 * 1000)
+    public void scheduledUpdate() {
+        updateCidSet();
+    }
+
+    // 更新CID集合的方法
+    private synchronized void updateCidSet() {
+        try {
+            log.info("start init cid data");
+            String modelVersion = readCsvPathFromOss();
+            if (StringUtils.isEmpty(modelVersion)) {
+                log.error("modelVersion is null");
+                return;
+            }
+            String csvPath = cidPath + modelVersion;
+
+            // 创建临时Set存储新数据
+            Set<Long> newCidSet = new HashSet<>();
+            // 列出指定文件夹下的所有对象
+
+            ListObjectsRequest listObjectsRequest = new ListObjectsRequest(BUCKET_NAME);
+            listObjectsRequest.setPrefix(csvPath);
+            listObjectsRequest.setMaxKeys(10);
+            ObjectListing objectListing = ossClient.listObjects(listObjectsRequest);
+
+            // 查找第一个CSV文件
+            String firstCsvFilePath = null;
+            for (OSSObjectSummary objectSummary : objectListing.getObjectSummaries()) {
+                String filePath = objectSummary.getKey();
+                if (filePath.equals(csvPath)) {
+                    continue;
+                }
+                if (filePath.toLowerCase().endsWith(".csv")) {
+                    firstCsvFilePath = filePath;
+                    log.info("find csv file: {}", firstCsvFilePath);
+                    break;
+                }
+            }
+
+            // 如果找到CSV文件，则读取内容
+            if (firstCsvFilePath != null) {
+                // 读取CSV文件内容到临时Set
+                try (OSSObject ossObject = ossClient.getObject(BUCKET_NAME, firstCsvFilePath);
+                     BufferedReader reader = new BufferedReader(new InputStreamReader(
+                             ossObject.getObjectContent(), StandardCharsets.UTF_8))) {
+                    String line;
+                    while ((line = reader.readLine()) != null) {
+                        if (!line.trim().isEmpty()) {
+                            newCidSet.add(Long.valueOf(line));
+                        }
+                    }
+                }
+                log.info("success get csv file size = {}", newCidSet.size());
+            } else {
+                log.error("not found csv file");
+            }
+            // 使用volatile保证可见性，一次性替换整个集合
+            cidSet = Collections.unmodifiableSet(newCidSet);
+            log.info("cid set init success size={}", cidSet.size());
+        } catch (Exception e) {
+            log.error("update cid error", e);
+            // 发生异常时保持原数据不变
+        }
+    }
+
+    /**
+     * 从OSS配置文件中读取CSV路径（第一行）
+     */
+    private String readCsvPathFromOss() {
+        try {
+            // 检查配置文件是否存在
+            if (!ossClient.doesObjectExist(BUCKET_NAME, modelVersionPath)) {
+                log.error("modelVersionPath not exist: {}", modelVersionPath);
+                return null;
+            }
+            // 获取配置文件内容
+            OSSObject ossObject = ossClient.getObject(BUCKET_NAME, modelVersionPath);
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(
+                    ossObject.getObjectContent(), StandardCharsets.UTF_8))) {
+                // 读取整个文件内容
+                StringBuilder content = new StringBuilder();
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    content.append(line);
+                }
+                JSONObject jsonObject = JSON.parseObject(content.toString());
+                String modelName = jsonObject.getString("modelName");
+                String dtVersion = jsonObject.getString("dtVersion");
+                if (StringUtils.isNotEmpty(modelName) && StringUtils.isNotEmpty(dtVersion)) {
+                    return modelName + "/" + dtVersion;
+                }
+            }
+        } catch (Exception e) {
+            log.error("get modelVersionPath error", e);
+        }
+        return null;
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/math/Const.java

@@ -0,0 +1,8 @@
+package com.tzld.piaoquan.ad.engine.commons.math;
+
+public class Const {
+    public static final double WILSON_ZSCORE = 1.96;
+    public static final double CTR_SMOOTH_BETA_FACTOR = 25;
+    public static final double CVR_SMOOTH_BETA_FACTOR = 10;
+    public static final double CTCVR_SMOOTH_BETA_FACTOR = 100;
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/param/LayerParam.java

@@ -0,0 +1,10 @@
+package com.tzld.piaoquan.ad.engine.commons.param;
+
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class LayerParam {
+    private String mid;
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/param/RankRecommendRequestParam.java

@@ -3,11 +3,18 @@ package com.tzld.piaoquan.ad.engine.commons.param;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 import java.util.List;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
+@ToString(callSuper = true)
 public class RankRecommendRequestParam extends RecommendRequestParam{
      List<AdPlatformCreativeDTO> adIdList;
+     String sessionId;
+     String subSessionId;
+     String rootSessionId;
+     String rootSourceId;
+     Boolean isFilterUser;
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/redis/AdRedisHelper.java

@@ -7,6 +7,7 @@ import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -51,4 +52,8 @@ public class AdRedisHelper {
         return Boolean.TRUE.equals(adRedisTemplate.hasKey(key));
     }
 
+    public List<String> mget(List<String> keys) {
+        return adRedisTemplate.opsForValue().multiGet(keys);
+    }
+
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/redis/AlgorithmRedisHelper.java

@@ -21,9 +21,6 @@ public class AlgorithmRedisHelper {
     @Resource(name = "algorithmRedisTemplate")
     private RedisTemplate<String, String> redisTemplate;
 
-
-
-
     public boolean containsKey(String key) {
         return redisTemplate.hasKey(key);
     }
@@ -207,4 +204,11 @@ public class AlgorithmRedisHelper {
         }
         return null;
     }
+
+    public List<String> mget(Collection<String> keys) {
+        if (CollectionUtils.isEmpty(keys)) {
+            return new ArrayList<>();
+        }
+        return redisTemplate.opsForValue().multiGet(keys);
+    }
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScoreParam.java

@@ -28,5 +28,7 @@ public class ScoreParam {
      * 本次命中的实验
      */
     private String expCode;
+
+    private Boolean isFilterUser;
 }
 

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerUtils.java

@@ -29,6 +29,7 @@ public final class ScorerUtils {
     public static String XGBOOST_SCORE_CONF_683 = "ad_score_config_xgboost_683.conf";
     public static String XGBOOST_SCORE_CONF_20240909 = "ad_score_config_xgboost_20240909.conf";
     public static String XGBOOST_SCORE_CONF_20241105 = "ad_score_config_xgboost_20241105.conf";
+    public static String PAI_SCORE_CONF_20250214 = "ad_score_config_pai_20250214.conf";
 
     public static void warmUp() {
         log.info("scorer warm up ");
@@ -39,6 +40,7 @@ public final class ScorerUtils {
         ScorerUtils.init(XGBOOST_SCORE_CONF);
         ScorerUtils.init(XGBOOST_SCORE_CONF_20240909);
         ScorerUtils.init(XGBOOST_SCORE_CONF_20241105);
+        ScorerUtils.init(PAI_SCORE_CONF_20250214);
     }
 
     private ScorerUtils() {

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/CreativeCalibrationModel.java

@@ -0,0 +1,53 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CreativeCalibrationModel extends Model {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(CreativeCalibrationModel.class);
+
+    private Map<Long, Double> creativeDiffRateMap = new HashMap<>();
+
+    @Override
+    public int getModelSize() {
+        return this.creativeDiffRateMap.size();
+    }
+
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws Exception {
+        Map<Long, Double> initMap = new HashMap<>();
+        try (BufferedReader input = new BufferedReader(in)) {
+            String line;
+            while ((line = input.readLine()) != null) {
+                String[] items = line.split("\t");
+                if (items.length < 2) {
+                    continue;
+                }
+
+                long cid = Long.parseLong(items[0]);
+                double diffRate = Double.parseDouble(items[1]);
+
+                initMap.put(cid, diffRate);
+
+            }
+            this.creativeDiffRateMap = initMap;
+            LOGGER.info("[CreativeCalibrationModel] model load over and size {}", this.creativeDiffRateMap.size());
+        } catch (Exception e) {
+            LOGGER.info("[CreativeCalibrationModel] model load error ", e);
+        } finally {
+            in.close();
+
+        }
+        return true;
+    }
+
+    public double getDiffRate(long cid) {
+        return this.creativeDiffRateMap.getOrDefault(cid, 0d);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/PAIModelV1.java

@@ -0,0 +1,210 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+import com.aliyun.openservices.eas.predict.http.HttpConfig;
+import com.aliyun.openservices.eas.predict.http.PredictClient;
+import com.aliyun.openservices.eas.predict.request.TFDataType;
+import com.aliyun.openservices.eas.predict.request.TFRequest;
+import com.aliyun.openservices.eas.predict.response.TFResponse;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.CollectionUtils;
+
+import java.util.*;
+
+
+public class PAIModelV1 {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(PAIModelV1.class);
+
+    private PAIModelV1() {
+    }
+
+    private static final PAIModelV1 model;
+
+    public static PAIModelV1 getModel() {
+        return model;
+    }
+
+    private static final PredictClient client;
+
+    static {
+        model = new PAIModelV1();
+        client = new PredictClient(new HttpConfig());
+        client.setEndpoint("1894469520484605.vpc.cn-hangzhou.pai-eas.aliyuncs.com");
+        client.setToken("ODI1MmUxODgzZDc3ODM0ZmQwZWU0YTVjZjdlOWVlMGFlZGJjNTlkYQ==");
+        client.setModelName("ad_rank_dnn_v11_easyrec");
+    }
+
+    private static final String[] sparseUserStrFeatures = {
+            "brand", "region", "city", "cate1", "cate2", "user_cid_click_list", "user_cid_conver_list",
+            "user_vid_return_tags_2h", "user_vid_return_tags_1d", "user_vid_return_tags_3d", "user_vid_return_tags_7d",
+            "user_vid_return_tags_14d", "root_source_scene", "root_source_channel", "title_split", "user_vid_share_tags_1d",
+            "user_vid_share_tags_14d", "user_vid_return_cate1_14d", "user_vid_return_cate2_14d", "user_vid_share_cate1_14d",
+            "user_vid_share_cate2_14d", "user_conver_ad_class", "user_layer_class"
+    };
+
+    private static final String[] sparseUserLongFeatures = {
+            "vid", "apptype", "is_first_layer", "user_has_conver_1y"
+    };
+
+    private static final String[] sparseSceneLongFeatures = {
+            "hour", "hour_quarter"
+    };
+
+    private static final String[] sparseAdLongFeatures = {
+            "cid", "adid", "adverid",
+            "user_adverid_view_3d", "user_adverid_view_7d", "user_adverid_view_30d",
+            "user_adverid_click_3d", "user_adverid_click_7d", "user_adverid_click_30d",
+            "user_adverid_conver_3d", "user_adverid_conver_7d", "user_adverid_conver_30d",
+            "user_skuid_view_3d", "user_skuid_view_7d", "user_skuid_view_30d",
+            "user_skuid_click_3d", "user_skuid_click_7d", "user_skuid_click_30d",
+            "user_skuid_conver_3d", "user_skuid_conver_7d", "user_skuid_conver_30d"
+    };
+
+    private static final String[] sparseAdStrFeatures = {
+            "profession"
+    };
+
+    private final String[] userFeatures = {
+            "viewAll", "clickAll", "converAll", "incomeAll", "ctr_all", "ctcvr_all", "cvr_all", "ecpm_all"
+    };
+
+    private final String[] itemFeatures = {
+            "actionstatic_click", "actionstatic_ctcvr", "actionstatic_ctr", "actionstatic_view", "b2_12h_click", "b2_12h_conver",
+            "b2_12h_conver_x_ctcvr", "b2_12h_conver_x_log_view", "b2_12h_ctcvr", "b2_12h_ctr", "b2_12h_cvr", "b2_12h_ecpm",
+            "b2_1d_click", "b2_1d_conver", "b2_1d_conver_x_ctcvr", "b2_1d_conver_x_log_view", "b2_1d_ctcvr", "b2_1d_ctr",
+            "b2_1d_cvr", "b2_1d_ecpm", "b2_3d_click", "b2_3d_conver", "b2_3d_conver_x_ctcvr", "b2_3d_conver_x_log_view",
+            "b2_3d_ctcvr", "b2_3d_ctr", "b2_3d_cvr", "b2_3d_ecpm", "b2_3h_click", "b2_3h_conver", "b2_3h_conver_x_ctcvr",
+            "b2_3h_conver_x_log_view", "b2_3h_ctcvr", "b2_3h_ctr", "b2_3h_cvr", "b2_3h_ecpm", "b2_6h_click", "b2_6h_conver",
+            "b2_6h_conver_x_ctcvr", "b2_6h_conver_x_log_view", "b2_6h_ctcvr", "b2_6h_ctr", "b2_6h_cvr", "b2_6h_ecpm", "b2_7d_click",
+            "b2_7d_conver", "b2_7d_conver_x_ctcvr", "b2_7d_conver_x_log_view", "b2_7d_ctcvr", "b2_7d_ctr", "b2_7d_cvr",
+            "b2_7d_ecpm", "b3_12h_click", "b3_12h_conver", "b3_12h_conver_x_ctcvr", "b3_12h_ctcvr", "b3_12h_ctr", "b3_12h_cvr",
+            "b3_12h_ecpm", "b3_1d_click", "b3_1d_conver", "b3_1d_conver_x_ctcvr", "b3_1d_conver_x_log_view", "b3_1d_ctcvr",
+            "b3_1d_ctr", "b3_1d_cvr", "b3_1d_ecpm", "b3_3d_click", "b3_3d_conver", "b3_3d_conver_x_ctcvr",
+            "b3_3d_conver_x_log_view", "b3_3d_ctcvr", "b3_3d_ctr", "b3_3d_cvr", "b3_3d_ecpm", "b3_3h_click", "b3_3h_conver",
+            "b3_3h_conver_x_ctcvr", "b3_3h_ctcvr", "b3_3h_ctr", "b3_3h_cvr", "b3_3h_ecpm", "b3_6h_click", "b3_6h_conver_x_ctcvr",
+            "b3_6h_ctcvr", "b3_6h_ctr", "b3_6h_cvr", "b3_6h_ecpm", "b3_7d_click", "b3_7d_conver", "b3_7d_conver_x_ctcvr",
+            "b3_7d_conver_x_log_view", "b3_7d_ctcvr", "b3_7d_ctr", "b3_7d_cvr", "b3_7d_ecpm", "b4_12h_click",
+            "b4_12h_conver_x_ctcvr", "b4_12h_conver_x_log_view", "b4_12h_ctcvr", "b4_12h_ctr", "b4_12h_cvr", "b4_12h_ecpm",
+            "b4_1d_click", "b4_1d_conver_x_ctcvr", "b4_1d_conver_x_log_view", "b4_1d_ctcvr", "b4_1d_ctr", "b4_1d_cvr", "b4_1d_ecpm",
+            "b4_3d_click", "b4_3d_conver_x_ctcvr", "b4_3d_conver_x_log_view", "b4_3d_ctcvr", "b4_3d_ctr", "b4_3d_cvr", "b4_3d_ecpm",
+            "b4_3h_click", "b4_3h_conver_x_ctcvr", "b4_3h_conver_x_log_view", "b4_3h_ctcvr", "b4_3h_ctr", "b4_3h_cvr", "b4_3h_ecpm",
+            "b4_6h_click", "b4_6h_conver_x_ctcvr", "b4_6h_conver_x_log_view", "b4_6h_ctcvr", "b4_6h_ctr", "b4_6h_cvr", "b4_6h_ecpm",
+            "b4_7d_click", "b4_7d_conver", "b4_7d_conver_x_ctcvr", "b4_7d_conver_x_log_view", "b4_7d_ctcvr", "b4_7d_ctr",
+            "b4_7d_cvr", "b4_7d_ecpm", "b5_12h_click", "b5_12h_conver", "b5_12h_conver_x_ctcvr", "b5_12h_ctcvr", "b5_12h_ctr",
+            "b5_12h_cvr", "b5_12h_ecpm", "b5_1d_click", "b5_1d_conver", "b5_1d_conver_x_ctcvr", "b5_1d_conver_x_log_view",
+            "b5_1d_ctcvr", "b5_1d_ctr", "b5_1d_cvr", "b5_1d_ecpm", "b5_3d_click", "b5_3d_conver", "b5_3d_conver_x_ctcvr",
+            "b5_3d_conver_x_log_view", "b5_3d_ctcvr", "b5_3d_ctr", "b5_3d_cvr", "b5_3d_ecpm", "b5_3h_click", "b5_3h_conver_x_ctcvr",
+            "b5_3h_conver_x_log_view", "b5_3h_ctcvr", "b5_3h_ctr", "b5_3h_cvr", "b5_3h_ecpm", "b5_6h_click",
+            "b5_6h_conver_x_log_view", "b5_6h_ctcvr", "b5_6h_ctr", "b5_6h_cvr", "b5_6h_ecpm", "b5_7d_click", "b5_7d_conver",
+            "b5_7d_conver_x_ctcvr", "b5_7d_conver_x_log_view", "b5_7d_ctcvr", "b5_7d_ctr", "b5_7d_cvr", "b5_7d_ecpm", "b6_7d_click",
+            "b6_7d_conver", "b6_7d_conver_x_log_view", "b6_7d_ctcvr", "b6_7d_ctr", "b6_7d_cvr", "b6_7d_ecpm", "b7_14d_ctr",
+            "b7_7d_click", "b7_7d_conver_x_ctcvr", "b7_7d_conver_x_log_view", "b7_7d_ctcvr", "b7_7d_ctr", "b7_7d_cvr", "b7_7d_ecpm",
+            "b8_12h_click", "b8_12h_conver_x_ctcvr", "b8_12h_ctcvr", "b8_12h_ctr", "b8_12h_cvr", "b8_12h_ecpm", "b8_1d_click",
+            "b8_1d_conver", "b8_1d_conver_x_ctcvr", "b8_1d_conver_x_log_view", "b8_1d_ctcvr", "b8_1d_ctr", "b8_1d_cvr",
+            "b8_1d_ecpm", "b8_3d_click", "b8_3d_conver", "b8_3d_conver_x_ctcvr", "b8_3d_ctcvr", "b8_3d_ctr", "b8_3d_cvr",
+            "b8_3d_ecpm", "b8_3h_click", "b8_3h_conver_x_ctcvr", "b8_3h_ctcvr", "b8_3h_ctr", "b8_3h_cvr", "b8_3h_ecpm",
+            "b8_6h_click", "b8_6h_conver_x_ctcvr", "b8_6h_ctcvr", "b8_6h_ctr", "b8_6h_cvr", "b8_6h_ecpm", "b8_7d_click",
+            "b8_7d_conver_x_ctcvr", "b8_7d_conver_x_log_view", "b8_7d_ctcvr", "b8_7d_ctr", "b8_7d_cvr", "b8_7d_ecpm", "cpa",
+            "d1_feature_12h_conver", "d1_feature_12h_ctcvr", "d1_feature_12h_ctr", "d1_feature_12h_cvr", "d1_feature_12h_ecpm",
+            "d1_feature_1d_conver", "d1_feature_1d_ctcvr", "d1_feature_1d_ctr", "d1_feature_1d_cvr", "d1_feature_1d_ecpm",
+            "d1_feature_3d_conver", "d1_feature_3d_ctcvr", "d1_feature_3d_ctr", "d1_feature_3d_ecpm", "d1_feature_3h_conver",
+            "d1_feature_3h_ctcvr", "d1_feature_3h_ctr", "d1_feature_3h_cvr", "d1_feature_3h_ecpm", "d1_feature_6h_ctcvr",
+            "d1_feature_6h_ctr", "d1_feature_6h_ecpm", "d1_feature_7d_conver", "d1_feature_7d_ctcvr", "d1_feature_7d_ctr",
+            "d1_feature_7d_cvr", "d1_feature_7d_ecpm", "e1_tags_14d_avgscore", "e1_tags_14d_maxscore", "e1_tags_3d_avgscore",
+            "e1_tags_3d_maxscore", "e1_tags_7d_avgscore", "e1_tags_7d_maxscore", "e2_tags_14d_avgscore", "e2_tags_14d_maxscore",
+            "e2_tags_3d_avgscore", "e2_tags_3d_maxscore", "e2_tags_7d_avgscore", "e2_tags_7d_maxscore", "timediff_conver",
+            "timediff_view", "vid_rank_ctcvr_14d", "vid_rank_ctcvr_1d", "vid_rank_ctcvr_3d", "vid_rank_ctcvr_7d",
+            "vid_rank_ctr_14d", "vid_rank_ctr_1d", "vid_rank_ctr_3d", "vid_rank_ctr_7d", "vid_rank_ecpm_14d", "vid_rank_ecpm_1d",
+            "vid_rank_ecpm_3d", "vid_rank_ecpm_7d"
+    };
+
+
+    public List<Float> score(final List<AdRankItem> items,
+                             final Map<String, String> userFeatureMap,
+                             final Map<String, String> sceneFeatureMap) {
+        try {
+            TFRequest request = new TFRequest();
+
+            for (String feature : sparseUserStrFeatures) {
+                String v = userFeatureMap.getOrDefault(feature, "");
+                request.addFeed(feature, TFDataType.DT_STRING, new long[]{1}, new String[]{v});
+            }
+
+            for (String feature : sparseUserLongFeatures) {
+                long v = NumberUtils.toLong(userFeatureMap.getOrDefault(feature, "0"), 0);
+                request.addFeed(feature, TFDataType.DT_INT64, new long[]{1}, new long[]{v});
+            }
+
+            for (String feature : sparseSceneLongFeatures) {
+                long v = NumberUtils.toLong(sceneFeatureMap.getOrDefault(feature, "0"), 0);
+                request.addFeed(feature, TFDataType.DT_INT64, new long[]{1}, new long[]{v});
+            }
+
+
+            for (String feature : userFeatures) {
+                double v = NumberUtils.toDouble(userFeatureMap.getOrDefault(feature, "0.0"), 0.0);
+                request.addFeed(feature.toLowerCase(), TFDataType.DT_DOUBLE, new long[]{1}, new double[]{v});
+            }
+            Map<String, double[]> doubleFeed = new HashMap<>();
+            Map<String, long[]> longFeed = new HashMap<>();
+            Map<String, String[]> strFeed = new HashMap<>();
+            for (int i = 0; i < items.size(); i++) {
+                for (String feature : itemFeatures) {
+                    String key = feature.replace("_x_", "*").replace("_view", "(view)");
+                    double[] doubles = doubleFeed.computeIfAbsent(feature, k -> new double[items.size()]);
+                    if (MapUtils.isEmpty(items.get(i).getFeatureMap())) {
+                        doubles[i] = 0.0;
+                        continue;
+                    }
+                    double v = NumberUtils.toDouble(items.get(i).getFeatureMap().getOrDefault(key, "0.0"), 0.0);
+                    doubles[i] = v;
+                }
+
+                for (String feature : sparseAdLongFeatures) {
+                    long[] longs = longFeed.computeIfAbsent(feature, k -> new long[items.size()]);
+                    if (MapUtils.isEmpty(items.get(i).getFeatureMap())) {
+                        longs[i] = 0L;
+                        continue;
+                    }
+                    long v = NumberUtils.toLong(items.get(i).getFeatureMap().getOrDefault(feature, "0"), 0L);
+                    longs[i] = v;
+                }
+
+                for (String feature : sparseAdStrFeatures) {
+                    String[] strs = strFeed.computeIfAbsent(feature, k -> new String[items.size()]);
+                    if (MapUtils.isEmpty(items.get(i).getFeatureMap())) {
+                        strs[i] = "";
+                        continue;
+                    }
+                    String v = items.get(i).getFeatureMap().getOrDefault(feature, "");
+                    strs[i] = v;
+                }
+            }
+            for (Map.Entry<String, double[]> entry : doubleFeed.entrySet()) {
+                request.addFeed(entry.getKey(), TFDataType.DT_DOUBLE, new long[]{items.size()}, entry.getValue());
+            }
+
+            for (Map.Entry<String, long[]> entry : longFeed.entrySet()) {
+                request.addFeed(entry.getKey(), TFDataType.DT_INT64, new long[]{items.size()}, entry.getValue());
+            }
+
+            for (Map.Entry<String, String[]> entry : strFeed.entrySet()) {
+                request.addFeed(entry.getKey(), TFDataType.DT_STRING, new long[]{items.size()}, entry.getValue());
+            }
+            request.addFetch("probs");
+            TFResponse response = client.predict(request);
+            List<Float> result = response.getFloatVals("probs");
+            if (!CollectionUtils.isEmpty(result)) {
+                return result;
+            }
+        } catch (Exception e) {
+            LOGGER.error("PAIModel score error", e);
+        }
+        return new ArrayList<>(Collections.nCopies(items.size(), 0.0f));
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/XGBCalibrationModel.java → ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/ValueRangeCalibrationModel.java

@@ -9,9 +9,9 @@ import java.io.BufferedReader;
 import java.io.InputStreamReader;
 import java.util.Objects;
 
-public class XGBCalibrationModel extends Model {
+public class ValueRangeCalibrationModel extends Model {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(XGBCalibrationModel.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ValueRangeCalibrationModel.class);
 
     private Table<Double, Double, Double> table = HashBasedTable.create();
 
@@ -40,12 +40,12 @@ public class XGBCalibrationModel extends Model {
             this.table = initTable;
 
             for (Table.Cell<Double, Double, Double> cell : this.table.cellSet()) {
-                LOGGER.info("cell.row: {}, cell.column: {}, cell.value: {}", cell.getRowKey(), cell.getColumnKey(), cell.getValue());
+                LOGGER.info("[ValueRangeCalibrationModel] cell.row: {}, cell.column: {}, cell.value: {}", cell.getRowKey(), cell.getColumnKey(), cell.getValue());
             }
 
-            LOGGER.info("[XGBCalibrationModel] model load over and size {}", this.table.size());
+            LOGGER.info("[ValueRangeCalibrationModel] model load over and size {}", this.table.size());
         } catch (Exception e) {
-            LOGGER.info("[XGBCalibrationModel] model load error ", e);
+            LOGGER.info("[ValueRangeCalibrationModel] model load error ", e);
         } finally {
             in.close();
 

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/DateUtils.java

@@ -1,9 +1,8 @@
 package com.tzld.piaoquan.ad.engine.commons.util;
 
 import java.text.SimpleDateFormat;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
+import java.time.*;
+import java.time.format.DateTimeFormatter;
 import java.util.Calendar;
 import java.util.Date;
 
@@ -43,4 +42,18 @@ public final class DateUtils {
                 .getDayOfWeek()
                 .getValue();
     }
+
+    public static int getHourQuarter(long timestamp) {
+        Instant instant = Instant.ofEpochSecond(timestamp);
+        ZonedDateTime zonedDateTime = instant.atZone(ZoneId.systemDefault());
+        int hour = zonedDateTime.getHour();
+        int minute = zonedDateTime.getMinute();
+        return minute / 15 + hour * 4;
+    }
+
+    public static String getThatDayDateString() {
+        DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern("yyyyMMdd");
+        LocalDate today = LocalDate.now();
+        return dateFormat.format(today);
+    }
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/NumUtil.java

@@ -1,5 +1,9 @@
 package com.tzld.piaoquan.ad.engine.commons.util;
 
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.Arrays;
+
 public class NumUtil {
 
     public static double div(double d1, double d2) {
@@ -9,14 +13,60 @@ public class NumUtil {
         return d1 / d2;
     }
 
+    public static double divSmoothV1(double a, double b, double zscore) {
+        // Wilson Smoothing
+        if (a == 0 || b == 0) {
+            return 0d;
+        }
+        double zscore2 = zscore * zscore;
+        double p = a / b;
+        double numerator = p + zscore2 / (2 * b) - zscore * Math.sqrt((p * (1 - p) + zscore2 / (4 * b)) / b);
+        double denominator = 1 + zscore2 / b;
+        return numerator / denominator;
+    }
+
+    public static double divSmoothV2(double a, double b, double beta) {
+        if (a == 0 || b == 0) {
+            return 0d;
+        }
+        return a / (b + beta);
+    }
 
-    public static Double log(double a){
-        if (a <= 0){
+    public static double log(double a) {
+        if (a <= 0) {
             return 0D;
         }
         return Math.log(a + 1.0);
     }
 
+    public static double log10(double a) {
+        if (a <= 0) {
+            return 0d;
+        }
+        return Math.log10(a + 1.0);
+    }
+
+    public static double sigmoid(double x) {
+        return 1 / (1 + Math.exp(-x));
+    }
+
+    public static double round(double value, int newScale){
+        return new BigDecimal(value).setScale(newScale, RoundingMode.HALF_UP).doubleValue();
+    }
+
+    public static double[] softmax(double[] inputs) {
+        double max = Arrays.stream(inputs).max().orElse(0);
+        double[] expValues = Arrays.stream(inputs)
+                .map(x -> Math.exp(x - max)) // Subtract max to stabilize
+                .toArray();
+
+        double sum = Arrays.stream(expValues).sum();
+
+        return Arrays.stream(expValues)
+                .map(x -> x / sum)
+                .toArray();
+    }
+
     @SafeVarargs
     public static <T extends Comparable<T>> T min(T... values) {
         if (values == null || values.length == 0) {

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/AdRecommendController.java

@@ -1,11 +1,18 @@
 package com.tzld.piaoquan.ad.engine.server.controller;
 
-import com.tzld.piaoquan.ad.engine.service.score.RankService;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.TypeReference;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
-import com.tzld.piaoquan.ad.engine.service.score.deprecated.BidRankRecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.commons.enums.GuaranteedTypeEnum;
+import com.tzld.piaoquan.ad.engine.commons.param.LayerParam;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
+import com.tzld.piaoquan.ad.engine.service.score.RankService;
+import com.tzld.piaoquan.ad.engine.service.score.deprecated.BidRankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -14,9 +21,7 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 
 @RestController
 @RequestMapping("/recommend")
@@ -27,6 +32,9 @@ public class AdRecommendController {
     @Qualifier("rankServiceImpl")
     RankService rankService;
 
+    @Autowired
+    protected AlgorithmRedisHelper algRedisHelper;
+
     @RequestMapping("/top1/basic")
     public Map<String, Object> adRecommendTop1Basic(@RequestBody RankRecommendRequestParam request) {
         Map<String, Object> map = new HashMap<>();
@@ -43,6 +51,32 @@ public class AdRecommendController {
                 Map<String, Object> contentMap = new HashMap<>();
                 contentMap.put("adId", rankResult.getAdId());
                 contentMap.put("adScore", rankResult.getScore());
+                Double ctcvrScore = rankResult.getScoreMap().get("ctcvrScore");
+                Double modelCtcvrScore = rankResult.getScoreMap().get("modelCtcvrScore");
+                if (ctcvrScore != null && ctcvrScore > 0) {
+                    contentMap.put("ecpm", modelCtcvrScore * rankResult.getCpa() * 1000);
+                    if (rankResult.getExt().get("correctionFactor") != null) {
+                        contentMap.put("revisedBid", ctcvrScore * rankResult.getCpa() * (double) rankResult.getExt().get("correctionFactor"));
+                    } else {
+                        contentMap.put("revisedBid", ctcvrScore * rankResult.getCpa());
+                    }
+                } else {
+                    contentMap.put("revisedBid", rankResult.getCpm() / 1000);
+                }
+                List<String> participateCompetitionType = new ArrayList<>();
+                participateCompetitionType.add("engine");
+                if (rankResult.getExt().get("isGuaranteed") != null && rankResult.getExt().get("isGuaranteed").equals(true)) {
+                    contentMap.put("type", GuaranteedTypeEnum.IS_GUARANTEED.getType());
+                } else if (rankResult.getExt().containsKey("isHotRank") && Boolean.TRUE.equals(rankResult.getExt().get("isHotRank"))) {
+                    contentMap.put("type", 2);
+                    participateCompetitionType.add("weight");
+                } else {
+                    contentMap.put("type", GuaranteedTypeEnum.NOT_GUARANTEED.getType());
+                }
+                if (rankResult.getExt().containsKey("isGuaranteeType") && (boolean) rankResult.getExt().get("isGuaranteeType")) {
+                    participateCompetitionType.add("guarantee");
+                }
+                contentMap.put("participateCompetitionType", StringUtils.join(participateCompetitionType, ","));
                 map.put("content", contentMap);
                 return map;
             }
@@ -88,4 +122,40 @@ public class AdRecommendController {
         return map;
     }
 
+    @RequestMapping("/layer")
+    public Map<String, Object> getMidWithLayer(@RequestBody LayerParam request) {
+        Map<String, Object> map = new HashMap<>();
+        if (request == null || StringUtils.isEmpty(request.getMid())) {
+            map.put("code", "1");
+            map.put("msg", "mid is null");
+            return map;
+        }
+        map.put("code", "0");
+        map.put("msg", "success");
+        Map<String, Object> result = new HashMap<>();
+        map.put("content", result);
+        String key = String.format("ad:engine:mid:layer:%s", request.getMid());
+        String value = algRedisHelper.get(key);
+        if (StringUtils.isEmpty(value)) {
+            result.put("layer", "无曝光");
+            return map;
+        }
+        try {
+            Map<String, String> layerMap = JSON.parseObject(value, new TypeReference<Map<String, String>>() {
+            });
+            String layer = layerMap.getOrDefault("layer", "无曝光");
+            if (Objects.equals(layer, "已转化")) {
+                layer = "有转化";
+            }
+            result.put("layer", layer);
+            return map;
+        } catch (Exception e) {
+            log.error("getMidWithLayer error. mid: {}, \n", request.getMid(), e);
+            map.put("code", "1");
+            map.put("msg", "get mid layer error");
+            return map;
+        }
+    }
+
+
 }

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/LogUploadController.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.ad.engine.server.controller;
+
+import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
+import com.tzld.piaoquan.ad.engine.service.score.convert.RequestConvert;
+import com.tzld.piaoquan.ad.engine.service.score.strategy.FeaturePrinterStrategy;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/log")
+public class LogUploadController {
+
+    @Autowired
+    private FeaturePrinterStrategy featurePrinterStrategy;
+
+    @Autowired
+    private LogHubService logHubService;
+
+    @RequestMapping("/printFeatureOnly")
+    public Map<String, Object> printFeatureOnly(@RequestBody RankRecommendRequestParam request) {
+        ScoreParam scoreParam = RequestConvert.requestConvert(request);
+        List<AdRankItem> adRankItems = featurePrinterStrategy.adItemRank(request, scoreParam);
+        logHubService.scoreLogUpload(scoreParam, request.getAdIdList(), adRankItems, request, scoreParam.getExpCode());
+        return new HashMap<String, Object>() {{
+            put("result", "success");
+        }};
+    }
+
+
+}

ad-engine-server/src/main/resources/ad_score_config_pai_20250214.conf

@@ -0,0 +1,6 @@
+scorer-config = {
+  pai-score-config = {
+    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.scorer.PAIScorer"
+    scorer-priority = 99
+  }
+}

ad-engine-server/src/main/resources/ad_score_config_xgboost_20240909.conf

@@ -2,7 +2,7 @@ scorer-config = {
   xgb-score-config = {
     scorer-name = "com.tzld.piaoquan.ad.engine.service.score.scorer.XGBoostScorer683"
     scorer-priority = 99
-    model-path = "zhangbo/model_xgb_351_1000_v2.tar.gz"
+    model-path = "fengzhoutian/model_xgb_351_1000_14d_v1.tar.gz"
   }
 
 }

ad-engine-server/src/main/resources/ad_score_config_xgboost_20241105.conf

@@ -5,7 +5,7 @@ scorer-config = {
     model-path = "zhangbo/model_xgb_351_1000_v2.tar.gz"
   }
   calibration-score-config = {
-    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.scorer.ScoreCalibrationScorer"
+    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.scorer.CreativeCalibrationScorer"
     scorer-priority = 98
     model-path = "zhangbo/model_xgb_351_1000_v2_calibration.txt"
   }

ad-engine-server/src/main/resources/ad_score_config_xgboost_683.conf

@@ -1,8 +1,7 @@
 scorer-config = {
-  lr-rov-score-config = {
+  xgb-score-config = {
     scorer-name = "com.tzld.piaoquan.ad.engine.service.score.scorer.XGBoostScorer683"
     scorer-priority = 99
-    model-path = "zhangbo/model_xgb_351_1000.tar.gz"
+    model-path = "fengzhoutian/model_xgb_351_1000_14d_v1.tar.gz"
   }
-
 }

ad-engine-server/src/main/resources/application-dev.yml

@@ -4,7 +4,7 @@ server:
 eureka:
   client:
     serviceUrl:
-      defaultZone: http://deveureka-internal.piaoquantv.com/eureka/
+      defaultZone: http://127.0.0.1:7000/eureka/
 
 datalog: .
 

ad-engine-server/src/main/resources/application.yml

@@ -86,7 +86,7 @@ logging:
 
 aliyun:
   log:
-    endpoint: cn-hangzhou.log.aliyuncs.com
+    endpoint: cn-hangzhou-intranet.log.aliyuncs.com
     accessKeyId: LTAIP6x1l3DXfSxm
     accessKeySecret: KbTaM9ars4OX3PMS6Xm7rtxGr1FLon
     logstore:
@@ -94,6 +94,7 @@ aliyun:
       info: info-log
       error: error-log
       statistics: statistics-log
+      crowdChooseStatistics: crowd-choose-statistics-log
       landingTypeFilter: landingpagetype-filter-error-log
       adPackageData: ad-package-data
     topic:

ad-engine-service/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>ad-engine-commons</artifactId>
-            <version>1.1.0</version>
+            <version>1.1.1</version>
         </dependency>
         <dependency>
             <groupId>com.aliyun.odps</groupId>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/CalibrationModelCtcvrData.java

@@ -0,0 +1,13 @@
+package com.tzld.piaoquan.ad.engine.service.entity;
+
+import lombok.Data;
+
+@Data
+public class CalibrationModelCtcvrData {
+
+    private Integer view;
+
+    private Double realCtcvr;
+
+    private Double pCtcvr;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/CorrectCpaParam.java

@@ -0,0 +1,14 @@
+package com.tzld.piaoquan.ad.engine.service.entity;
+
+import lombok.Data;
+
+@Data
+public class CorrectCpaParam {
+    private Double correctionFactor;
+    private Integer viewsHour;
+    private Integer viewsDay;
+    private Double realCtcvrHour;
+    private Double pCtcvrHour;
+    private Double realCtcvrDay;
+    private Double pCtcvrDay;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/CorrectCtcvrScoreParam.java

@@ -0,0 +1,11 @@
+package com.tzld.piaoquan.ad.engine.service.entity;
+
+import lombok.Data;
+
+@Data
+public class CorrectCtcvrScoreParam {
+
+    private Integer view;
+
+    private Double realCtcvr;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/GuaranteeView.java

@@ -0,0 +1,25 @@
+package com.tzld.piaoquan.ad.engine.service.entity;
+
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class GuaranteeView {
+
+    private String adrId;
+
+    private Integer adrAlgoViewNum;
+
+    private Integer allAlgoViewNum;
+
+    private Integer guaranteeNum;
+
+    private Double guaranteeRate;
+
+    /**
+     * @see com.tzld.piaoquan.ad.engine.commons.enums.CrowdLayerEnum
+     * CrowdLayerEnum.code,保量人群code 码，(1,2.。。)中间用英文逗号隔开
+      */
+    private String guaranteeCrowdCode;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/Feature.java

@@ -20,4 +20,7 @@ public class Feature {
     // k1:mid、k2:表、v:特征值
     private Map<String, Map<String, String>> userFeature = new HashMap<>();
 
+    // k1:skuid、k2:表、v:特征值
+    private Map<String, Map<String, Map<String, String>>> skuFeature = new HashMap<>();
+
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/FeatureService.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.feature;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
 import com.tzld.piaoquan.ad.engine.service.remote.FeatureV2RemoteService;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
@@ -25,8 +26,9 @@ public class FeatureService {
     private static final String avUkFormat = "av:%s:%s";
     private static final String vidUkFormat = "v:%s:%s";
     private static final String midUkFormat = "m:%s:%s";
+    private static final String skuUkFormat = "s:%s:%s";
 
-    public Feature getFeature(Collection<String> cidList, Collection<String> adVerIdList, ScoreParam param) {
+    public Feature getFeature(Collection<String> cidList, Collection<String> adVerIdList, List<Long> skuIdList, ScoreParam param) {
         AdRequestContext context = param.getRequestContext();
 
         List<FeatureKeyProto> protos = new ArrayList<>();
@@ -62,9 +64,19 @@ public class FeatureService {
             }
         }
 
+        // skuid
+        for (Long skuId : skuIdList) {
+            if (StringUtils.isNotEmpty(param.getMid())) {
+                protos.add(genWithMidAndSkuId("alg_mid_feature_sku_action", param.getMid(), String.valueOf(skuId)));
+            }
+        }
+
         for (String adVerId : adVerIdList) {
             // adverid
             protos.add(genWithAdVerId("alg_cid_feature_adver_action", adVerId));
+            if (StringUtils.isNotEmpty(param.getMid())) {
+                protos.add(genWithMidAndAdVerId("alg_mid_feature_adver_action", param.getMid(), adVerId));
+            }
         }
 
         // vid
@@ -78,7 +90,19 @@ public class FeatureService {
             protos.add(genWithMid("alg_mid_feature_ad_action", param.getMid()));
             protos.add(genWithMid("alg_mid_feature_return_tags", param.getMid()));
             protos.add(genWithMid("alg_mid_feature_share_tags", param.getMid()));
+            protos.add(genWithMid("mid_return_video_cate", param.getMid()));
+            protos.add(genWithMid("mid_share_video_cate", param.getMid()));
         }
+        return this.invokeFeatureService(protos);
+    }
+
+    public Feature getPredictFeature(PredictContext context) {
+        List<FeatureKeyProto> protos = new ArrayList<>();
+        protos.add(genWithMidAndAppType("alg_ad_crowd_choose_feature_v2", context.getMid(), context.getAppType()));
+        return this.invokeFeatureService(protos);
+    }
+
+    public Feature invokeFeatureService(List<FeatureKeyProto> protos) {
 
         Map<String, String> featureMap = remoteService.getFeature(protos);
         featureMap = this.featureStrCover(featureMap);
@@ -113,6 +137,12 @@ public class FeatureService {
                     feature.getVideoFeature().put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
                     }, Collections.emptyMap()));
                     break;
+                case "s":
+                    Map<String, Map<String, String>> skuFeatureMap = feature.getSkuFeature().getOrDefault(id, new HashMap<>());
+                    skuFeatureMap.put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getSkuFeature().put(id, skuFeatureMap);
+                    break;
             }
         }
 
@@ -135,6 +165,24 @@ public class FeatureService {
                 .build();
     }
 
+    private FeatureKeyProto genWithMidAndAdVerId(String table, String mid, String adVerId) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(avUkFormat, table, adVerId))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .putFieldValue("adverid", adVerId)
+                .build();
+    }
+
+    private FeatureKeyProto genWithMidAndSkuId(String table, String mid, String skuId) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(skuUkFormat, table, skuId))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .putFieldValue("skuid", skuId)
+                .build();
+    }
+
     private FeatureKeyProto genWithCidAndRegion(String table, String cid, String region) {
         return FeatureKeyProto.newBuilder()
                 .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, region))
@@ -214,6 +262,15 @@ public class FeatureService {
                 .build();
     }
 
+    private FeatureKeyProto genWithMidAndAppType(String table, String mid, String appType) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(midUkFormat, table, mid))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .putFieldValue("apptype", appType)
+                .build();
+    }
+
     private Map<String, String> featureStrCover(Map<String, String> metaFeatureMap) {
         Map<String, String> newFeatureMap = new HashMap<>();
         for (Map.Entry<String, String> entry : metaFeatureMap.entrySet()) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/LogHubService.java

@@ -3,11 +3,15 @@ package com.tzld.piaoquan.ad.engine.service.log;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 
 import java.util.List;
+import java.util.Map;
 
 public interface LogHubService {
 
     void scoreLogUpload(ScoreParam param, List<AdPlatformCreativeDTO> adIdList, List<AdRankItem> rankItems, RecommendRequestParam requestParam, String abCode);
+
+    void crowdChooseLogUpload(PredictContext context);
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -4,10 +4,11 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Sets;
 import com.tzld.commons.aliyun.log.AliyunLogManager;
-import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
-import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import lombok.extern.slf4j.Slf4j;
@@ -30,10 +31,13 @@ public class LogHubServiceImpl implements LogHubService {
             TimeUnit.MINUTES, new LinkedBlockingDeque<>(), new ThreadPoolExecutor.CallerRunsPolicy());
 
     @Value("${aliyun.log.logstore.statistics}")
-    private String logStore;
+    private String scoreStatisticsLogStore;
     @Value("${aliyun.log.project}")
     private String project;
 
+    @Value("${aliyun.log.logstore.crowdChooseStatistics}")
+    private String crowdChooseStatisticsLogStore;
+
     @Autowired
     private AliyunLogManager aliyunLogManager;
 
@@ -110,8 +114,31 @@ public class LogHubServiceImpl implements LogHubService {
                 logMap.put("allfeature", JSON.toJSONString(top1.getFeatureMap()));
                 logMap.put("metafeature", JSON.toJSONString(top1.getMetaFeatureMap()));
 
-                aliyunLogManager.sendLog(project, logStore, "", logMap);
+                aliyunLogManager.sendLog(project, scoreStatisticsLogStore, "", logMap);
+            }
+        });
+    }
+
+    @Override
+    public void crowdChooseLogUpload(PredictContext context) {
+        logUploadThreadPool.execute(new Runnable() {
+            @Override
+            public void run() {
+                Map<String, Object> logMap = new HashMap<>();
+                logMap.put("mid", context.getMid());
+                logMap.put("vid", context.getVideoId());
+                logMap.put("apptype", context.getAppType());
+                logMap.put("abcode", context.getAdAbCode());
+                logMap.put("pqtid", context.getPqtId());
+                logMap.put("isshowad", context.getLogParam().isAIsShowAd());
+                logMap.put("expid", context.getLogParam().getExpId());
+                logMap.put("score", context.getLogParam().getScore());
+                logMap.put("allfeature", JSON.toJSONString(context.getLogParam().getAllFeature()));
+                logMap.put("metafeature", JSON.toJSONString(context.getLogParam().getMetaFeature()));
+                logMap.put("scoremap", JSON.toJSONString(context.getLogParam().getScoreMap()));
+                logMap.put("isnewuser", context.getLogParam().isBIsNewUser());
+                aliyunLogManager.sendLog(project, crowdChooseStatisticsLogStore, "", logMap);
             }
         });
     }
-}
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/config/AbConfig.java

@@ -3,7 +3,6 @@ package com.tzld.piaoquan.ad.engine.service.predict.config;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.gson.Gson;
-import com.tzld.piaoquan.ad.engine.service.predict.model.threshold.ThresholdPredictModel;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.PostConstruct;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/container/AbTestConfigContainer.java

@@ -1,10 +1,7 @@
 package com.tzld.piaoquan.ad.engine.service.predict.container;
 
 import com.alibaba.fastjson.JSONObject;
-import com.tzld.piaoquan.ad.engine.commons.feign.CommonResponse;
 import com.tzld.piaoquan.ad.engine.commons.feign.longvideo.abtest.LongVideoFeign;
-import com.tzld.piaoquan.ad.engine.commons.feign.longvideo.abtest.request.AbTestConfigRequest;
-import com.tzld.piaoquan.ad.engine.commons.math.Tuple;
 import com.tzld.piaoquan.ad.engine.service.predict.impl.PredictModelServiceImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -12,10 +9,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
-import javax.annotation.PostConstruct;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 
 @Component
 public class AbTestConfigContainer {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/container/ThresholdModelContainer.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.ad.engine.service.predict.container;
 
-import com.tdunning.math.stats.Centroid;
 import com.tzld.piaoquan.ad.engine.service.predict.model.threshold.ThresholdPredictModel;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/impl/PredictModelServiceImpl.java

@@ -7,7 +7,9 @@ import com.tzld.piaoquan.ad.engine.commons.enums.AppTypeEnum;
 import com.tzld.piaoquan.ad.engine.commons.enums.RedisPrefixEnum;
 import com.tzld.piaoquan.ad.engine.commons.redis.AdRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
+import com.tzld.piaoquan.ad.engine.commons.util.AbUtil;
 import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
+import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import com.tzld.piaoquan.ad.engine.service.predict.PredictModelService;
 import com.tzld.piaoquan.ad.engine.service.predict.config.AbConfig;
 import com.tzld.piaoquan.ad.engine.service.predict.config.NewExpUserGroupConfig;
@@ -21,6 +23,10 @@ import com.tzld.piaoquan.ad.engine.service.predict.param.RoiThresholdPredictMode
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.RoiPredictModelRequestParam;
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.ThresholdPredictModelRequestParam;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.ConvertUtil;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictServiceV2;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,6 +61,11 @@ public class PredictModelServiceImpl implements PredictModelService {
     @Autowired
     RoiModelConfig roiModelConfig;
 
+    @Autowired
+    private PredictServiceV2 predictServiceV2;
+    @Autowired
+    private LogHubService logHubService;
+
     @Value("${ad.predict.share0.exp.code:000}")
     private String adPredictNoShareUserExpCode;
 
@@ -78,6 +89,9 @@ public class PredictModelServiceImpl implements PredictModelService {
     @Value("${advance.show.ad.switch:true}")
     private boolean advanceShowAdSwitch;
 
+    @Value("#{'${show.ad.whitelist.mid:}'.split(',')}")
+    private Set<String> showAdWhitelistMidSet;
+
     @Autowired
     private AdRedisHelper adRedisHelper;
 
@@ -87,6 +101,14 @@ public class PredictModelServiceImpl implements PredictModelService {
     public Map<String, Object> adPredict(ThresholdPredictModelRequestParam requestParam) {
         Map<String, Object> result = new HashMap<>();
         result.put("pqtId", requestParam.getPqtId());
+
+        // 白名单用户固定出广告
+        if (CollectionUtils.isNotEmpty(showAdWhitelistMidSet) && showAdWhitelistMidSet.contains(requestParam.getMid())) {
+            result.put("ad_predict", 2);
+            result.put("ad_strategy", "show_ad_whitelist");
+            return result;
+        }
+
         try {
             String[] withoutAdVideoIdsArr = withoutAdVideoIds.split(",");
             for (String videoId : withoutAdVideoIdsArr) {
@@ -138,6 +160,15 @@ public class PredictModelServiceImpl implements PredictModelService {
                 result.put("no_ad_strategy", "no_ad_time_with_fixed_time");
                 return result;
             }
+
+            if (AbUtil.isInAbExp(expCodes, requestParam.getAppType(), requestParam.getNewExpGroup(), "713")) {
+                PredictContext context = ConvertUtil.predictParam2Context(requestParam);
+                Map<String, Object> resultMap = predictServiceV2.adPredict(context);
+                logHubService.crowdChooseLogUpload(context);
+                resultMap.put("pqtId", requestParam.getPqtId());
+                return resultMap;
+            }
+
             String abtestId = null;
             String abTestConfigTag = null;
             Map<String, Object> abtestParam = null;
@@ -219,7 +250,6 @@ public class PredictModelServiceImpl implements PredictModelService {
                 }
             }
 
-
             // 市-中文
             requestParam.setRegion(requestParam.getRegion().replace("省", ""));
             requestParam.setCity(requestParam.getCity().replace("市", ""));
@@ -281,7 +311,7 @@ public class PredictModelServiceImpl implements PredictModelService {
             return result;
         } catch (Exception e) {
             log.error("svc=adPredict appType={} group={} newGroup={} pqtId={}"
-                    , requestParam.getAppType(), requestParam.getAbTestCode(), requestParam.getNewExpGroup(), requestParam.getPqtId());
+                    , requestParam.getAppType(), requestParam.getAbTestCode(), requestParam.getNewExpGroup(), requestParam.getPqtId(), e);
             result.put("ad_predict", 1);
             result.put("no_ad_strategy", "error");
             return result;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/AddThresholdPredictModel.java

@@ -2,7 +2,6 @@ package com.tzld.piaoquan.ad.engine.service.predict.model.threshold;
 
 
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
-import com.tzld.piaoquan.ad.engine.commons.util.TimerWatchUtil;
 import com.tzld.piaoquan.ad.engine.service.predict.calculator.ThresholdPredictCalculator;
 import com.tzld.piaoquan.ad.engine.service.predict.helper.RuleParamHelper;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdCalculateParam;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/BasicThresholdPredictModel.java

@@ -1,7 +1,6 @@
 package com.tzld.piaoquan.ad.engine.service.predict.model.threshold;
 
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
-import com.tzld.piaoquan.ad.engine.commons.util.TimerWatchUtil;
 import com.tzld.piaoquan.ad.engine.service.predict.calculator.ThresholdPredictCalculator;
 import com.tzld.piaoquan.ad.engine.service.predict.helper.RuleParamHelper;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdCalculateParam;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/MultiplyThresholdPredictModel.java

@@ -1,7 +1,6 @@
 package com.tzld.piaoquan.ad.engine.service.predict.model.threshold;
 
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
-import com.tzld.piaoquan.ad.engine.commons.util.TimerWatchUtil;
 import com.tzld.piaoquan.ad.engine.service.predict.calculator.ThresholdPredictCalculator;
 import com.tzld.piaoquan.ad.engine.service.predict.helper.RuleParamHelper;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdCalculateParam;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/RandomPredict667Model.java

@@ -1,6 +1,8 @@
 package com.tzld.piaoquan.ad.engine.service.predict.model.threshold;
 
+import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
+import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
 import com.tzld.piaoquan.ad.engine.service.predict.container.RandWContainer;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
 import org.slf4j.Logger;
@@ -22,23 +24,45 @@ public class RandomPredict667Model extends ThresholdPredictModel {
 
     @Override
     public Map<String, Object> predict(ThresholdPredictModelParam modelParam) {
-        int hash=modelParam.getMid().hashCode();
-        hash=hash<0?-hash:hash;
-        double score=(hash+ RandWContainer.getRandW())%100/100d;
-        double threshold=Double.parseDouble(
-                modelParam.getExtraParam().getOrDefault(modelParam.getAppType()+"_"+modelParam.getUserExtraFuture("shareType").toString().replace("return","").replace("mids",""),-1
-                ).toString());
-        if(threshold<0d){
-            threshold=Double.parseDouble(
-                    modelParam.getExtraParam().getOrDefault("default_threshold","0.5")
-                    .toString());
+        int hash = modelParam.getMid().hashCode();
+        hash = hash < 0 ? -hash : hash;
+        double score = (hash + RandWContainer.getRandW()) % 100 / 100d;
+
+        String keySuffix = modelParam.getUserExtraFuture("shareType").toString()
+                .replace("return", "")
+                .replace("mids", "");
+
+        Integer appType = modelParam.getAppType();
+        String thresholdParamKey = appType + "_" + keySuffix;
+
+        double threshold = Double.parseDouble(modelParam.getExtraParam().getOrDefault(thresholdParamKey, -1).toString());
+        if (threshold < 0d) {
+            thresholdParamKey = "default_" + keySuffix;
+            threshold = Double.parseDouble(modelParam.getExtraParam().getOrDefault(thresholdParamKey, "-1").toString());
         }
+        double correction = Double.parseDouble(modelParam.getExtraParam().getOrDefault("E620_" + appType + "_" + DateUtils.getCurrentHour(), 1).toString());
+        threshold = threshold * correction;
+
         Map<String, Object> result = new HashMap<>();
-        result.put("ad_predict", score<threshold?2:1);
+        result.put("ad_predict", score < threshold ? 2 : 1);
         result.put("score", score);
         result.put("threshold", threshold);
         result.put("model", initName());
 
+        JSONObject logJson = new JSONObject();
+        logJson.putAll(result);
+        logJson.put("mid", modelParam.getMid());
+        logJson.put("expId", "667");
+        logJson.put("appType", modelParam.getAppType());
+        logJson.put("thresholdParamKey", thresholdParamKey);
+        logJson.put("adPlatformType", modelParam.getAdPlatformType());
+        logJson.put("abCode", modelParam.getAbTestCode());
+        logJson.put("extraParam", modelParam.getExtraParam());
+        logJson.put("shareType", modelParam.getUserExtraFuture("shareType").toString());
+
+        log.info("广告跳出选择 -- 667实验结果: {}, 参数: {}",
+                JSONUtils.toJson(result), logJson.toJSONString());
+
         return result;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/RandomPredict673Model.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.ad.engine.service.predict.model.threshold;
 
+import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
 import com.tzld.piaoquan.ad.engine.service.predict.container.RandWContainer;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
@@ -65,8 +66,20 @@ public class RandomPredict673Model extends ThresholdPredictModel {
         result.put("threshold", threshold);
         result.put("model", this.initName());
 
-        log.info("广告跳出选择 -- 673实验结果: {}, 参数: {}, {}, {}, {}",
-                JSONUtils.toJson(result), appType, modelParam.getMid(), abParamKey, thresholdParamKey);
+        JSONObject logJson = new JSONObject();
+        logJson.putAll(result);
+        logJson.put("mid", modelParam.getMid());
+        logJson.put("appType", appType);
+        logJson.put("expId", "673");
+        logJson.put("userExternalSource", modelParam.getUserExternalSource());
+        logJson.put("shareType", modelParam.getUserExtraFuture("shareType").toString());
+        logJson.put("shareLayer", modelParam.getShareLayer());
+        logJson.put("thresholdParamKey", thresholdParamKey);
+        logJson.put("adPlatformType", modelParam.getAdPlatformType());
+
+        log.info("广告跳出选择 -- 673实验结果: {}, 参数: {}",
+                JSONUtils.toJson(result), logJson.toJSONString());
+
 
         return result;
     }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/RandomPredictModel.java

@@ -54,9 +54,14 @@ public class RandomPredictModel extends ThresholdPredictModel {
         logJson.put("expId", "599");
         logJson.put("appType", appType);
         logJson.put("thresholdParamKey", thresholdParamKey);
+        logJson.put("adPlatformType", modelParam.getAdPlatformType());
+        logJson.put("abCode", modelParam.getAbTestCode());
+        logJson.put("extraParam", modelParam.getExtraParam());
+        logJson.put("shareType", modelParam.getUserExtraFuture("shareType").toString());
 
-        log.info("广告跳出选择 -- 599实验结果: {}, 参数: {}, {}, {}",
-                JSONUtils.toJson(result), appType, modelParam.getMid(), thresholdParamKey);
+
+        log.info("广告跳出选择 -- 599实验结果: {}, 参数: {}",
+                JSONUtils.toJson(result), logJson.toJSONString());
         return result;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/model/threshold/ScoreThresholdPredictModel.java

@@ -2,7 +2,6 @@ package com.tzld.piaoquan.ad.engine.service.predict.model.threshold;
 
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
-import com.tzld.piaoquan.ad.engine.commons.util.TimerWatchUtil;
 import com.tzld.piaoquan.ad.engine.service.predict.config.AdOutV1OnlineWeightConfig;
 import com.tzld.piaoquan.ad.engine.service.predict.constant.RuleRedisKeyConst;
 import com.tzld.piaoquan.ad.engine.service.predict.container.ThresholdModelContainer;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/param/ThresholdPredictModelParam.java

@@ -42,6 +42,8 @@ public class ThresholdPredictModelParam {
     String city = "-1";
     MachineInfoParam machineInfo = new MachineInfoParam();
 
+    private String adPlatformType;
+
 
     private String userExternalSource;
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/param/request/ThresholdPredictModelRequestParam.java

@@ -27,6 +27,8 @@ public class ThresholdPredictModelRequestParam {
     String pqtId;
     MachineInfoParam machineInfo = new MachineInfoParam();
 
+    private String adPlatformType;
+
     private String userExternalSource;
 
     /**

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/ConvertUtil.java

@@ -0,0 +1,58 @@
+package com.tzld.piaoquan.ad.engine.service.predict.v2;
+
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
+import com.tzld.piaoquan.ad.engine.service.predict.param.request.ThresholdPredictModelRequestParam;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+@Slf4j
+public class ConvertUtil {
+
+    public static PredictContext predictParam2Context(ThresholdPredictModelRequestParam param) {
+        PredictContext context = new PredictContext();
+        context.setMid(param.getMid());
+        context.setVideoId(Optional.ofNullable(param.getVideoId()).map(Object::toString).orElse(""));
+        context.setAppType(Optional.ofNullable(param.getAppType()).map(Object::toString).orElse(""));
+        context.setAdAbCode(param.getAbTestCode());
+
+        context.setRegion(param.getRegion());
+        context.setCity(param.getCity());
+        context.setPqtId(param.getPqtId());
+        context.setUserExternalSource(param.getUserExternalSource());
+        context.setShareLayer(param.getShareLayer());
+        context.setMachineInfo(param.getMachineInfo());
+
+        JSONObject abExpInfo = param.getAbExpInfo();
+        Map<String, JSONObject> expConfigMap = ConvertUtil.parseAbTest002Info(abExpInfo);
+        context.setExpCodeSet(expConfigMap.keySet());
+        context.setExpConfigMap(expConfigMap);
+
+        return context;
+    }
+
+
+    private static Map<String, JSONObject> parseAbTest002Info(JSONObject abExpInfo) {
+        Map<String, JSONObject> abTest002Info = new HashMap<>();
+        try {
+            List<JSONObject> abTest002 = abExpInfo.getJSONArray("ab_test002").toJavaObject(new TypeReference<List<JSONObject>>() {
+            });
+
+            for (JSONObject json : abTest002) {
+                String expCode = json.getString("abExpCode");
+                JSONObject configValue = json.getJSONObject("configValue");
+                abTest002Info.put(expCode, configValue);
+            }
+
+        } catch (Exception e) {
+            log.error("ConvertUtil解析广告层实验配置异常: ", e);
+        }
+
+        return abTest002Info;
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/PredictContext.java

@@ -0,0 +1,68 @@
+package com.tzld.piaoquan.ad.engine.service.predict.v2;
+
+import com.alibaba.fastjson.JSONObject;
+import com.tzld.piaoquan.ad.engine.commons.param.MachineInfoParam;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class PredictContext {
+
+    private String mid;
+
+    private String videoId;
+
+    private String appType;
+
+    /**
+     * 用户在广告层所属的实验组
+     */
+    private String adAbCode;
+
+    /**
+     * 用户命中的广告层所有实验
+     */
+    private Set<String> expCodeSet = new HashSet<>();
+
+    private Map<String, JSONObject> expConfigMap = new HashMap<>();
+
+    /**
+     * 省份
+     */
+    private String region;
+
+    /**
+     * 城市
+     */
+    private String city;
+
+    private String pqtId;
+
+    /**
+     * 设备信息
+     */
+    MachineInfoParam machineInfo = new MachineInfoParam();
+
+    private String userExternalSource;
+
+    /**
+     * 分享所属层
+     * <br />
+     * 1. firstLayer: 首层
+     * <br />
+     * 2. otherLayer: 裂变层
+     */
+    private String shareLayer;
+
+    private PredictLogParam logParam = new PredictLogParam();
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/PredictLogParam.java

@@ -0,0 +1,30 @@
+package com.tzld.piaoquan.ad.engine.service.predict.v2;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class PredictLogParam {
+
+    private String expId;
+
+    private Map<String, Double> allFeature = new HashMap<>();
+
+    private Map<String, Map<String, String>> metaFeature = new HashMap<>();
+
+    private Double score;
+
+    private Map<String, Double> scoreMap = new HashMap<>();
+
+    private boolean aIsShowAd;
+
+    private boolean bIsNewUser;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/PredictServiceV2.java

@@ -0,0 +1,190 @@
+package com.tzld.piaoquan.ad.engine.service.predict.v2;
+
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
+import com.tzld.piaoquan.ad.engine.service.feature.Feature;
+import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Slf4j
+@Service
+public class PredictServiceV2 {
+
+    @Autowired
+    private FeatureService featureService;
+
+    @ApolloJsonValue("${exp.713.config:{}}")
+    private Map<String, Double> exp713Config;
+
+    public Map<String, Object> adPredict(PredictContext context) {
+        Feature feature = featureService.getPredictFeature(context);
+        Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+        Map<String, String> featureMap = userFeature.getOrDefault("alg_ad_crowd_choose_feature_v2", new HashMap<>());
+        double minScore = exp713Config.getOrDefault("minScore", 0.1d);
+        double maxScore = exp713Config.getOrDefault("maxScore", 0.8d);
+        double score = maxScore;
+
+        context.getLogParam().setBIsNewUser(true);
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+
+            double adViewCnt = Double.parseDouble(featureMap.getOrDefault("ad_view_cnt", "0"));
+            double adClick = Double.parseDouble(featureMap.getOrDefault("ad_click", "0"));
+            double adConver = Double.parseDouble(featureMap.getOrDefault("ad_conver", "0"));
+            double hasAdClick = Double.parseDouble(featureMap.getOrDefault("has_ad_click", "0"));
+            double hasAdShare = Double.parseDouble(featureMap.getOrDefault("has_ad_share", "0"));
+            double hasAdReturnCnt = Double.parseDouble(featureMap.getOrDefault("has_ad_return_cnt", "0"));
+            double noAdClick = Double.parseDouble(featureMap.getOrDefault("no_ad_click", "0"));
+            double noAdShare = Double.parseDouble(featureMap.getOrDefault("no_ad_share", "0"));
+            double noAdReturnCnt = Double.parseDouble(featureMap.getOrDefault("no_ad_return_cnt", "0"));
+
+            // 计算出回流出广告时的收益
+            double adClickValue = NumUtil.div(adClick, adViewCnt) * NumUtil.log10(adClick);
+            double adConverValue = NumUtil.div(adConver, adViewCnt) * NumUtil.log10(adConver);
+            double hasAdShareValue = NumUtil.div(hasAdShare, hasAdClick) * NumUtil.log10(hasAdShare);
+            double hasAdReturnValue = NumUtil.div(hasAdReturnCnt, hasAdClick) * NumUtil.log10(hasAdReturnCnt);
+            double hasAdValue = adClickValue + adConverValue + hasAdShareValue + hasAdReturnValue;
+
+            // 计算回流不出广告时的收益
+            double noAdShareValue = NumUtil.div(noAdShare, noAdClick) * NumUtil.log10(noAdShare);
+            double noAdReturnValue = NumUtil.div(noAdReturnCnt, noAdClick) * NumUtil.log10(noAdReturnCnt + hasAdReturnCnt);
+            double noAdValue = noAdShareValue + noAdReturnValue;
+
+            // 计算最终的收益
+            double hasRate = exp713Config.getOrDefault("hasRate", 1d);
+            double noRate = exp713Config.getOrDefault("noRate", 1d);
+            score = NumUtil.softmax(new double[]{hasAdValue * hasRate, noAdValue * noRate})[0];
+
+
+            context.getLogParam().getMetaFeature().putAll(feature.getUserFeature());
+            for (Map.Entry<String, String> entry : featureMap.entrySet()) {
+                context.getLogParam().getAllFeature().put(entry.getKey(), Double.parseDouble(entry.getValue()));
+            }
+            context.getLogParam().getScoreMap().put("adClickValue", NumUtil.round(adClickValue, 6));
+            context.getLogParam().getScoreMap().put("adConverValue", NumUtil.round(adConverValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdShareValue", NumUtil.round(hasAdShareValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdReturnValue", NumUtil.round(hasAdReturnValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdValue", NumUtil.round(hasAdValue, 6));
+            context.getLogParam().getScoreMap().put("noAdShareValue", NumUtil.round(noAdShareValue, 6));
+            context.getLogParam().getScoreMap().put("noAdReturnValue", NumUtil.round(noAdReturnValue, 6));
+            context.getLogParam().getScoreMap().put("noAdValue", NumUtil.round(noAdValue, 6));
+            context.getLogParam().getScoreMap().put("originScore", NumUtil.round(score, 6));
+            context.getLogParam().getScoreMap().put("hasRate", NumUtil.round(hasRate, 6));
+            context.getLogParam().getScoreMap().put("noRate", NumUtil.round(noRate, 6));
+
+            context.getLogParam().setBIsNewUser(false);
+        }
+
+        // 分数截断，避免过长或过短
+        if (score < minScore) {
+            score = minScore;
+        } else if (score > maxScore) {
+            score = maxScore;
+        }
+
+        double random = Math.random();
+        boolean isShowAd = random < score;
+        context.getLogParam().setExpId("713");
+        context.getLogParam().setScore(score);
+        context.getLogParam().getScoreMap().put("score", NumUtil.round(score, 6));
+        context.getLogParam().setAIsShowAd(isShowAd);
+        context.getLogParam().getScoreMap().put("minScore", minScore);
+        context.getLogParam().getScoreMap().put("maxScore", maxScore);
+        context.getLogParam().getScoreMap().put("random", random);
+
+
+        return isShowAd ? rtnAdPredict(context) : rtnNoAdPredict(context);
+    }
+
+
+    private Map<String, Object> rtnNoAdPredict(PredictContext context) {
+        Map<String, Object> rtnMap = new HashMap<>();
+        rtnMap.put("ad_predict", 1);
+        rtnMap.put("no_ad_strategy", "713_exp");
+        return rtnMap;
+    }
+
+    private Map<String, Object> rtnAdPredict(PredictContext context) {
+        Map<String, Object> rtnMap = new HashMap<>();
+        rtnMap.put("ad_predict", 2);
+        rtnMap.putAll(context.getLogParam().getScoreMap());
+        return rtnMap;
+    }
+
+    // public Map<String, Object> adPredictV1(PredictContext context){
+    //
+    //     Feature feature = featureService.getPredictFeature(context);
+    //     Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+    //     Map<String, String> featureMap = userFeature.getOrDefault("alg_ad_crowd_choose_feature", new HashMap<>());
+    //
+    //     double score = -1;
+    //
+    //     // 没有特征为新用户，随机出广告
+    //     if (MapUtils.isEmpty(featureMap)) {
+    //         double newUserShowAdRate = exp713Config.getOrDefault("newUserShowAdRate", 0.8d);
+    //         double randomRate = Math.random();
+    //         if (randomRate < newUserShowAdRate) {
+    //             context.getLogParam().getScoreMap().put("newUserShowAdRate", newUserShowAdRate);
+    //             context.getLogParam().getScoreMap().put("randomRate", randomRate);
+    //             score = 1;
+    //         }
+    //     } else {
+    //         context.getLogParam().getMetaFeature().putAll(userFeature);
+    //         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
+    //             context.getLogParam().getAllFeature().put(entry.getKey(), Double.parseDouble(entry.getValue()));
+    //         }
+    //
+    //         // 获取需要的特征值
+    //         double showAdClickPv = Double.parseDouble(featureMap.getOrDefault("show_ad_click_pv", "0"));
+    //         double noShowAdClickPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_click_pv", "0"));
+    //         double showAdIncome = Double.parseDouble(featureMap.getOrDefault("show_ad_income", "0"));
+    //         double showAdSharePv = Double.parseDouble(featureMap.getOrDefault("show_ad_share_pv", "0"));
+    //         double noShowAdSharePv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_share_pv", "0"));
+    //         double showAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("show_ad_new_return_pv", "0"));
+    //         double noShowAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_new_return_pv", "0"));
+    //
+    //         // 计算中间过程值
+    //         double singleReturnAdIncome = NumUtil.div(showAdIncome, (showAdClickPv + noShowAdClickPv));
+    //         double showAdShareRate = NumUtil.div((showAdSharePv + 1), (showAdClickPv + 1));
+    //         double noShowAdShareRate = NumUtil.div((noShowAdSharePv + 1), (noShowAdClickPv + 1));
+    //         double returnDivShare = NumUtil.div((showAdNewReturnPv + noShowAdNewReturnPv + 1), (showAdSharePv + noShowAdSharePv + 1));
+    //
+    //         double busDauBalanceRate = exp713Config.getOrDefault("busDauBalanceRate", 0.1d);
+    //         score = singleReturnAdIncome + ((showAdShareRate - noShowAdShareRate) * returnDivShare * busDauBalanceRate);
+    //
+    //
+    //         context.getLogParam().getScoreMap().put("singleReturnAdIncome", singleReturnAdIncome);
+    //         context.getLogParam().getScoreMap().put("showAdShareRate", showAdShareRate);
+    //         context.getLogParam().getScoreMap().put("noShowAdShareRate", noShowAdShareRate);
+    //         context.getLogParam().getScoreMap().put("returnDivShare", returnDivShare);
+    //         context.getLogParam().getScoreMap().put("busDauBalanceRate", busDauBalanceRate);
+    //
+    //     }
+    //     double showAdScoreThreshold = exp713Config.getOrDefault("showAdScoreThreshold", 0d);
+    //     boolean isShowAd = score >= showAdScoreThreshold;
+    //
+    //     context.getLogParam().setAIsShowAd(isShowAd);
+    //     context.getLogParam().setExpId("713");
+    //     context.getLogParam().setScore(score);
+    //     context.getLogParam().getScoreMap().put("score", score);
+    //     context.getLogParam().getScoreMap().put("showAdScoreThreshold", showAdScoreThreshold);
+    //
+    //
+    //     logHubService.crowdChooseLogUpload(context);
+    //
+    //
+    //     if (isShowAd) {
+    //         Map<String, Object> rtnMap = rtnAdPredict();
+    //         rtnMap.putAll(context.getLogParam().getScoreMap());
+    //         return rtnMap;
+    //     } else {
+    //         return rtnNoAdPredict("713_exp");
+    //     }
+    // }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/RequestConvert.java

@@ -1,9 +1,9 @@
 package com.tzld.piaoquan.ad.engine.service.score.convert;
 
-import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
-import com.tzld.piaoquan.ad.engine.commons.util.AbUtil;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.util.AbUtil;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 
@@ -45,7 +45,6 @@ public class RequestConvert {
         // 本次命中的实验
         String expCode = AbUtil.chooseAdAlgExpCode(expCodeSet, request.getAppType(), request.getNewExpGroup());
         scoreParam.setExpCode(expCode);
-
         return scoreParam;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/scorer/CreativeCalibrationScorer.java

@@ -0,0 +1,148 @@
+package com.tzld.piaoquan.ad.engine.service.score.scorer;
+
+import com.tzld.piaoquan.ad.engine.commons.score.AbstractScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.CreativeCalibrationModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+public class CreativeCalibrationScorer extends AbstractScorer {
+
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private static final Logger LOGGER = LoggerFactory.getLogger(CreativeCalibrationScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+    public CreativeCalibrationScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        super.doLoadModel(CreativeCalibrationModel.class);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    final List<AdRankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+
+        List<AdRankItem> result = rankByJava(sceneFeatureMap, userFeatureMap, rankItems);
+
+        LOGGER.debug("[CreativeCalibrationScorer] scoring items size={}, time={} ",
+                result.size(), System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                        final Map<String, String> userFeatureMap,
+                                        final List<AdRankItem> items) {
+        long startTime = System.currentTimeMillis();
+        CreativeCalibrationModel model = (CreativeCalibrationModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (AdRankItem item : items) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", item, item);
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("[CreativeCalibrationScorer] items size={}, cost={} ",
+                items.size(), System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final CreativeCalibrationModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex), ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // 等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException: ", e);
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},", sceneFeatureMap.size(), e);
+                }
+            }
+        }
+    }
+
+    public double calcScore(final CreativeCalibrationModel model,
+                            final AdRankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+        double ctcvrScore = item.getLrScore();
+        double newCtcvrScore = ctcvrScore;
+        try {
+
+            double diffRate = model.getDiffRate(item.getAdId());
+            newCtcvrScore = ctcvrScore / (1 + diffRate);
+            item.setLrScore(newCtcvrScore);
+            item.getScoreMap().put("diff_rate", diffRate);
+            item.getScoreMap().put("originCtcvrScore", ctcvrScore);
+            item.getScoreMap().put("ctcvrScore", newCtcvrScore);
+        } catch (Exception e) {
+            LOGGER.error("[score calibration] calcScore error: ", e);
+        }
+
+        return newCtcvrScore;
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/scorer/PAIScorer.java

@@ -0,0 +1,94 @@
+package com.tzld.piaoquan.ad.engine.service.score.scorer;
+
+
+import com.google.common.collect.Lists;
+import com.tzld.piaoquan.ad.engine.commons.score.AbstractScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.BaseXGBoostModelScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.PAIModelV1;
+import com.tzld.piaoquan.ad.engine.commons.score.model.XGBoostModel683;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+public class PAIScorer extends AbstractScorer {
+
+    private final static Logger LOGGER = LoggerFactory.getLogger(PAIScorer.class);
+
+
+    public PAIScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userAdFeature,
+                                    final List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    final List<AdRankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+
+        List<AdRankItem> result = rankByJava(sceneFeatureMap, userFeatureMap, rankItems);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                        final Map<String, String> userFeatureMap,
+                                        final List<AdRankItem> items) {
+        long startTime = System.currentTimeMillis();
+        PAIModelV1 model = PAIModelV1.getModel();
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final PAIModelV1 model) {
+
+        List<Float> score = model.score(items, userFeatureMap, sceneFeatureMap);
+        LOGGER.debug("PAIScorer score={}", score);
+        for (int i = 0; i < items.size(); i++) {
+            Double pro = Double.valueOf(score.get(i));
+            items.get(i).setLrScore(pro);
+            items.get(i).getScoreMap().put("ctcvrScore", pro);
+        }
+    }
+
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/scorer/ScoreCalibrationScorer.java → ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/scorer/ValueRangeCalibrationScorer.java

@@ -3,7 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score.scorer;
 import com.tzld.piaoquan.ad.engine.commons.score.AbstractScorer;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
-import com.tzld.piaoquan.ad.engine.commons.score.model.XGBCalibrationModel;
+import com.tzld.piaoquan.ad.engine.commons.score.model.ValueRangeCalibrationModel;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import org.apache.commons.collections4.CollectionUtils;
@@ -17,20 +17,20 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.*;
 
-public class ScoreCalibrationScorer extends AbstractScorer {
+public class ValueRangeCalibrationScorer extends AbstractScorer {
 
     private static final int LOCAL_TIME_OUT = 150;
-    private static Logger LOGGER = LoggerFactory.getLogger(ScoreCalibrationScorer.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(ValueRangeCalibrationScorer.class);
     private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
 
 
-    public ScoreCalibrationScorer(ScorerConfigInfo scorerConfigInfo) {
+    public ValueRangeCalibrationScorer(ScorerConfigInfo scorerConfigInfo) {
         super(scorerConfigInfo);
     }
 
     @Override
     public void loadModel() {
-        super.doLoadModel(XGBCalibrationModel.class);
+        super.doLoadModel(ValueRangeCalibrationModel.class);
     }
 
     @Override
@@ -59,7 +59,7 @@ public class ScoreCalibrationScorer extends AbstractScorer {
                                         final Map<String, String> userFeatureMap,
                                         final List<AdRankItem> items) {
         long startTime = System.currentTimeMillis();
-        XGBCalibrationModel model = (XGBCalibrationModel) this.getModel();
+        ValueRangeCalibrationModel model = (ValueRangeCalibrationModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
 
         // 所有都参与打分，按照ctr排序
@@ -82,7 +82,7 @@ public class ScoreCalibrationScorer extends AbstractScorer {
     private void multipleCtrScore(final List<AdRankItem> items,
                                   final Map<String, String> userFeatureMap,
                                   final Map<String, String> sceneFeatureMap,
-                                  final XGBCalibrationModel model) {
+                                  final ValueRangeCalibrationModel model) {
 
         List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
         for (int index = 0; index < items.size(); index++) {
@@ -124,7 +124,7 @@ public class ScoreCalibrationScorer extends AbstractScorer {
         }
     }
 
-    public double calcScore(final XGBCalibrationModel model,
+    public double calcScore(final ValueRangeCalibrationModel model,
                             final AdRankItem item,
                             final Map<String, String> userFeatureMap,
                             final Map<String, String> sceneFeatureMap) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/FeaturePrinterStrategy.java

@@ -0,0 +1,53 @@
+package com.tzld.piaoquan.ad.engine.service.score.strategy;
+
+import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
+import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.util.ObjUtil;
+import com.tzld.piaoquan.ad.engine.service.feature.Feature;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Component
+public class FeaturePrinterStrategy extends RankStrategyBasic {
+    @Override
+    public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+        Set<String> noApiAdVerIds = getNoApiAdVerIds();
+        long ts = System.currentTimeMillis() / 1000;
+
+        String brand = scoreParam.getRequestContext().getMachineinfoBrand();
+        if (StringUtils.isNotEmpty(brand)) {
+            scoreParam.getRequestContext().setMachineinfoBrand(brand + "-n");
+        }
+
+        Feature feature = this.getFeature(scoreParam, request);
+        Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
+        Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
+
+        AdPlatformCreativeDTO dto = request.getAdIdList().get(0);
+        AdRankItem adRankItem = new AdRankItem();
+        adRankItem.setAdId(dto.getCreativeId());
+        adRankItem.setCreativeCode(dto.getCreativeCode());
+        adRankItem.setAdVerId(dto.getAdVerId());
+        adRankItem.setVideoId(request.getVideoId());
+        adRankItem.setCpa(dto.getCpa());
+        adRankItem.setId(dto.getAdId());
+        adRankItem.setCampaignId(dto.getCampaignId());
+        adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+        adRankItem.setSkuId(dto.getSkuId());
+        if (noApiAdVerIds.contains(dto.getAdVerId())) {
+            adRankItem.getExt().put("isApi", "0");
+        } else {
+            adRankItem.getExt().put("isApi", "1");
+        }
+        adRankItem.getExt().put("recallsources", dto.getRecallSources());
+        putMetaFeature(adRankItem, feature, reqFeature, sceneFeatureMap, request);
+        return Collections.singletonList(adRankItem);
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

@@ -1,19 +1,34 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
+import com.tzld.piaoquan.ad.engine.commons.enums.CrowdLayerEnum;
 import com.tzld.piaoquan.ad.engine.commons.enums.RedisPrefixEnum;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.ad.engine.commons.redis.AdRedisHelper;
+import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
+import com.tzld.piaoquan.ad.engine.commons.util.ObjUtil;
+import com.tzld.piaoquan.ad.engine.service.entity.CalibrationModelCtcvrData;
+import com.tzld.piaoquan.ad.engine.service.entity.CorrectCpaParam;
+import com.tzld.piaoquan.ad.engine.service.entity.CorrectCtcvrScoreParam;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 
 import java.util.*;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -25,10 +40,105 @@ public abstract class RankStrategyBasic implements RankStrategy {
     @ApolloJsonValue("${creative.model.score.coefficient:{}}")
     private Map<Long, Double> creativeScoreCoefficient;
 
+    @Value("${guarantee.exp:742}")
+    protected String guaranteeExp;
+
+    @ApolloJsonValue("${alpha:1.0}")
+    protected Double alpha;
+
+    @Value("${calibration.ctcvr.exp:779}")
+    protected String calibrationCtcvrExp;
+
+    @Value("${correct.cpa.exp.1:}")
+    protected String correctCpaExp1;
+
+    @Value("${correct.cpa.exp.2:}")
+    protected String correctCpaExp2;
+
+    @Value("${correct.cpa.alpha.1:1.0}")
+    protected Double correctCpaAlpha1;
+
+    @Value("${correct.cpa.beta.1:1.0}")
+    protected Double correctCpaBeta1;
+
+    @Value("${correct.cpa.alpha.2:0.7}")
+    protected Double correctCpaAlpha2;
+
+    @Value("${correct.cpa.beta.2:0.2}")
+    protected Double correctCpaBeta2;
+
+    @Value("${correct.cpa.view.hour:300}")
+    protected Integer correctCpaViewHour;
+
+    @Value("${correct.cpa.view.day:300}")
+    protected Integer correctCpaViewDay;
+
+    @Value("${correct.cpa.min.ctcvr:0.000001}")
+    protected Double minCtcvr;
+
+
+    @Value("${exclude.exp:772}")
+    protected String excludeExp;
+
+    @Value("${target.crowd.exclude.exp:785}")
+    protected String targetCrowdExcludeExp;
+
+    @Value("${calibration.coefficient.exp:786}")
+    protected String calibrationCoefficientExp;
+
+    @Value("${calibration.view:2000}")
+    protected Integer calibrationView;
+
+    @Value("${calibration.alpha:0.5}")
+    protected Double calibrationAlpha;
+
+    @Value("${guarantee.weight:30}")
+    protected Integer guaranteeWeight;
+
+    @Value("${guarantee.switching.time:1754236800000}")
+    protected Long guaranteeSwitchingTime;
+
+    // 保量人群加权系数配置
+    @Value("${guarantee.crowd.weight.coefficient:1.0}")
+    protected Double guaranteeCrowdWeightCoefficient;
+
     @Autowired
     private FeatureService featureService;
     @Autowired
     protected AdRedisHelper adRedisHelper;
+    @Autowired
+    protected AlgorithmRedisHelper algRedisHelper;
+
+    String adPlatformGuaranteeKey = "ad:platform:guarantee:data:{date}:{adverId}";
+
+    String realLayerCtcvrKey = "ad:platform:real:ctcvr:{model}:{layer}:{class}";
+
+    String adCustomerLayerHourKey = "ad:platform:customer:hour:{layer}:{clazz}:{customerId}";
+
+    String adVerLayerHourKey = "ad:platform:adver:hour:{layer}:{clazz}:{adverId}";
+    String adLayerHourKey = "ad:platform:ad:hour:{layer}:{clazz}:{adId}";
+
+    String adLayerDayKey = "ad:platform:ad:day:{layer}:{clazz}:{adId}";
+
+    String cidLayerKey = "ad:engine:cid:layer:info:{cid}:{userLayer}";
+
+    private static final double DEFAULT_CORRECTION = 1.0;
+
+
+    protected static final List<String> hasChannelScenes = new ArrayList<String>() {{
+        add("DaiTou");
+        add("GzhTouLiu");
+        add("daitou");
+        add("dyyjs");
+        add("dyyqw");
+        add("fwhdyy");
+        add("gzhhz");
+        add("gzhhzzx");
+        add("shequn");
+        add("touliu");
+        add("xcxdt");
+    }};
+
 
     protected Feature getFeature(ScoreParam param, RankRecommendRequestParam request) {
         List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
@@ -42,7 +152,13 @@ public abstract class RankStrategyBasic implements RankStrategy {
                 .filter(StringUtils::isNotBlank)
                 .distinct()
                 .collect(Collectors.toList());
-        return featureService.getFeature(cidList, adVerIdList, param);
+
+        List<Long> skuIdList = adIdList.stream()
+                .map(AdPlatformCreativeDTO::getSkuId)
+                .filter(Objects::nonNull)
+                .distinct()
+                .collect(Collectors.toList());
+        return featureService.getFeature(cidList, adVerIdList, skuIdList, param);
     }
 
     protected Set<String> getNoApiAdVerIds() {
@@ -61,6 +177,59 @@ public abstract class RankStrategyBasic implements RankStrategy {
         return creativeScoreCoefficient;
     }
 
+    protected Map<String, String> getReqFeature(ScoreParam scoreParam, RankRecommendRequestParam request) {
+        Map<String, String> map = new HashMap<>();
+        map.put("brand", scoreParam.getRequestContext().getMachineinfoBrand().equalsIgnoreCase("-1") ?
+                "" : scoreParam.getRequestContext().getMachineinfoBrand().toUpperCase());
+        map.put("region", Objects.equals(scoreParam.getRequestContext().getRegion(), "-1") ?
+                "" : scoreParam.getRequestContext().getRegion());
+        map.put("city", Objects.equals(scoreParam.getRequestContext().getCity(), "-1") ?
+                "" : scoreParam.getRequestContext().getCity());
+        map.put("vid", String.valueOf(request.getVideoId()));
+        map.put("apptype", String.valueOf(request.getAppType()));
+        String rootSessionId = request.getRootSessionId();
+        String sessionId = request.getSessionId();
+        String subSessionId = request.getSubSessionId();
+        String rootSourceId = request.getRootSourceId();
+        int isFirstLayer;
+        if (Objects.equals(rootSessionId, subSessionId) || Objects.equals(rootSessionId, sessionId)) {
+            isFirstLayer = 1;
+        } else {
+            isFirstLayer = 2;
+        }
+        map.put("is_first_layer", String.valueOf(isFirstLayer));
+        if (StringUtils.isNotEmpty(rootSourceId)) {
+            String rootSourceScene = rootSourceId.split("_")[0];
+            map.put("root_source_scene", rootSourceScene);
+            if (hasChannelScenes.contains(rootSourceScene) && rootSourceId.split("_").length > 1) {
+                String rootSourceChannel = rootSourceId.split("_")[1];
+                map.put("root_source_channel", rootSourceChannel);
+            }
+        }
+        Map<String, String> userLayer = getUserLayer(request.getMid());
+        String layer = userLayer.getOrDefault("layer", "无曝光");
+        String clazz = userLayer.getOrDefault("class", "近期未出现");
+        if (request.getIsFilterUser()) {
+            layer = layer + "-炸";
+        }
+        String userLayerClass = layer + "-" + clazz;
+        map.put("user_layer_class", userLayerClass);
+        map.put("layer", layer);
+        map.put("clazz", clazz);
+        return map;
+    }
+
+    protected Map<String, String> handleSceneFeature(long ts) {
+        Map<String, String> sceneFeatureMap = new HashMap<>();
+        sceneFeatureMap.put("hour_" + DateUtils.getHourByTimestamp(ts), "0.1");
+        sceneFeatureMap.put("dayofweek_" + DateUtils.getDayOrWeekByTimestamp(ts), "0.1");
+        sceneFeatureMap.put("hour", String.valueOf(DateUtils.getHourByTimestamp(ts)));
+        sceneFeatureMap.put("hour_quarter", String.valueOf(DateUtils.getHourQuarter(ts)));
+        sceneFeatureMap.put("ts", String.valueOf(ts));
+        return sceneFeatureMap;
+    }
+
+
     protected static class Tuple5 {
         public String f1;
         public String f2;
@@ -98,4 +267,667 @@ public abstract class RankStrategyBasic implements RankStrategy {
         }
 
     }
-}
+
+    protected void filterRequestAdList(RankRecommendRequestParam request, ScoreParam scoreParam) {
+        Map<String, String> userLayer = this.getUserLayer(request.getMid());
+        String layer = userLayer.getOrDefault("layer", "无曝光");
+        String clazz = userLayer.getOrDefault("class", "近期未出现");
+        //有转化中医层，中医和兴趣教育同时存在过滤兴趣教育行业
+        if (Objects.equals(layer, "有转化") && Objects.equals(clazz, "中医") && scoreParam.getExpCodeSet().contains(excludeExp)) {
+            List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
+            Set<String> professions = adIdList.stream().map(AdPlatformCreativeDTO::getProfession).collect(Collectors.toSet());
+            //同时存在中医行业和兴趣教育行业
+            if (professions.contains("中医") && professions.contains("兴趣教育")) {
+                List<AdPlatformCreativeDTO> filteredAdList = adIdList.stream()
+                        .filter(e -> !Objects.equals(e.getProfession(), "兴趣教育")).collect(Collectors.toList());
+                request.setAdIdList(filteredAdList);
+            }
+            log.info("excludeExp filtered request={}", JSONObject.toJSONString(request));
+        }
+
+        // 有曝光无转化-其他 过滤德瑞骅客户 客户id 26
+        if (Objects.equals(layer, "有曝光无转化") && Objects.equals(clazz, "其他") && scoreParam.getExpCodeSet().contains(targetCrowdExcludeExp)) {
+            List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
+            List<AdPlatformCreativeDTO> filteredAdList = adIdList.stream().filter(e -> e.getCustomerId() == null || e.getCustomerId() != 26L).collect(Collectors.toList());
+            if (CollectionUtils.isNotEmpty(filteredAdList)) {
+                request.setAdIdList(filteredAdList);
+            }
+            log.info("targetCrowdExcludeExp filtered request={}", JSONObject.toJSONString(request));
+        }
+    }
+
+
+    protected boolean getIsGuaranteedFlow(ScoreParam scoreParam) {
+        if (System.currentTimeMillis() < guaranteeSwitchingTime) {
+            return scoreParam.getExpCodeSet().contains(guaranteeExp);
+        }
+        Random random = new Random();
+        int i = random.nextInt(100);
+        return i < guaranteeWeight;
+    }
+
+    protected Map<String, GuaranteeView> getGuaranteeViewMap(RankRecommendRequestParam request, boolean isGuaranteedFlow) {
+        Map<String, GuaranteeView> map = new HashMap<>();
+        try {
+            if (isGuaranteedFlow) {
+                String thatDayDateString = DateUtils.getThatDayDateString();
+                String redisKey = adPlatformGuaranteeKey.replace("{date}", thatDayDateString);
+                List<String> adVerIds = request.getAdIdList().stream().map(AdPlatformCreativeDTO::getAdVerId).distinct()
+                        .filter(Objects::nonNull).collect(Collectors.toList());
+                List<String> redisKeys = adVerIds.stream().map(e -> redisKey.replace("{adverId}", e)).collect(Collectors.toList());
+                List<String> values = adRedisHelper.mget(redisKeys);
+                if (CollectionUtils.isNotEmpty(values)) {
+                    for (int i = 0; i < redisKeys.size(); i++) {
+                        String value = values.get(i);
+                        if (value != null) {
+                            GuaranteeView guaranteeView = JSONObject.parseObject(value, GuaranteeView.class);
+                            map.put(adVerIds.get(i), guaranteeView);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("getGuaranteeViewMap error", e);
+        }
+
+        return map;
+    }
+
+    protected void setGuaranteeWeight(Map<String, GuaranteeView> map, String adVerId, Map<String, Object> ext, boolean isGuaranteedFlow, Map<String, String> reqFeature) {
+        if (isGuaranteedFlow && MapUtils.isNotEmpty(map)) {
+            GuaranteeView guaranteeView = map.get(adVerId);
+            if (guaranteeView != null) {
+                double guaranteeWeight = calculateGuaranteeWeightWithCrowd(guaranteeView, reqFeature);
+                boolean isGuaranteed = isGuaranteed(guaranteeView);
+                ext.put("guaranteeView", guaranteeView.toString());
+                ext.put("guaranteeWeight", guaranteeWeight);
+                ext.put("isGuaranteed", isGuaranteed);
+                ext.put("isGuaranteedFlow", isGuaranteedFlow);
+            }
+        }
+    }
+
+    /**
+     * 兼容旧版本的setGuaranteeWeight方法（不带reqFeature参数）
+     * 为了保持向后兼容性，保留原有方法签名
+     */
+    protected void setGuaranteeWeight(Map<String, GuaranteeView> map, String adVerId, Map<String, Object> ext, boolean isGuaranteedFlow) {
+        setGuaranteeWeight(map, adVerId, ext, isGuaranteedFlow, new HashMap<>());
+    }
+
+    /**
+     * 根据人群信息计算保量权重系数（新版本）
+     *
+     * @param guaranteeView 保量视图对象
+     * @param reqFeature 请求特征信息，包含用户人群layer信息
+     * @return 保量权重系数
+     */
+    protected double calculateGuaranteeWeightWithCrowd(GuaranteeView guaranteeView, Map<String, String> reqFeature) {
+        // 空值检查：如果保量视图为空，返回默认权重
+        if (guaranteeView == null) {
+            return 1.0;
+        }
+
+        // 判断是否配置了保量人群代码
+        String guaranteeCrowdCode = guaranteeView.getGuaranteeCrowdCode();
+        if (StringUtils.isEmpty(guaranteeCrowdCode)) {
+            // 保量人群代码无值，走原有逻辑
+            return calculateGuaranteedWeight(guaranteeView);
+        }
+        // 判断是否勾选了全部人群
+        if (isFullCrowd(guaranteeCrowdCode)) {
+            // 勾选了全部人群，走原有逻辑
+            return calculateGuaranteedWeight(guaranteeView);
+        }
+
+        // 保量人群代码有值，进行人群匹配判断
+        String userLayer = reqFeature.getOrDefault("layer", "");
+
+        // 获取用户人群对应的CrowdLayerEnum code
+        Integer userCrowdCode = getUserCrowdCode(userLayer);
+        if (userCrowdCode == null) {
+            log.error("RankStrategyBasic calculateGuaranteeWeightWithCrowd 无法获取用户人群代码，userLayer={}", userLayer);
+            return 1.0;
+        }
+
+        // 判断用户人群是否在保量人群范围内
+        boolean isUserInGuaranteeCrowd = isUserInGuaranteeCrowd(guaranteeCrowdCode, userCrowdCode);
+
+        if (isUserInGuaranteeCrowd) {
+            // 到这里，就是用户在保量人群中，且广告主勾选的不是全部人群
+            // 计算保量权重
+            double baseWeight = calculateGuaranteedWeight(guaranteeView);
+            // 额外加权
+            // 对勾选了保量人群，且不是勾选了全部的保量人群的广告进行额外加权
+            double finalWeight = baseWeight * guaranteeCrowdWeightCoefficient;
+            log.debug("RankStrategyBasic 保量人群加权: userLayer={}, guaranteeCrowdCode={}, baseWeight={}, coefficient={}, finalWeight={}",
+                     userLayer, guaranteeCrowdCode, baseWeight, guaranteeCrowdWeightCoefficient, finalWeight);
+            return finalWeight;
+        } else {
+            // 用户不在保量人群范围内，权重设为1
+            log.debug("RankStrategyBasic 用户不在保量人群范围内: userLayer={}, guaranteeCrowdCode={}", userLayer, guaranteeCrowdCode);
+            return 1.0;
+        }
+    }
+
+    /**
+     * 计算保量权重系数（原有逻辑）
+     *
+     * 保量逻辑说明：
+     * 1. 根据广告主的保量配置（保量比例、保量上限）和实际曝光情况计算权重
+     * 2. 权重用于调整广告排序分数，帮助未达到保量目标的广告主获得更多曝光
+     * 3. 已达到保量上限的广告主权重为0，避免过度曝光
+     *
+     * @param guaranteeView 保量视图对象，包含广告主保量相关数据
+     *                     - adrId: 广告主ID
+     *                     - adrAlgoViewNum: 广告主当天算法曝光数
+     *                     - allAlgoViewNum: 全平台当天算法曝光数
+     *                     - guaranteeNum: 保量上限（广告主最大允许曝光数）
+     *                     - guaranteeRate: 保量比例（广告主期望占全平台曝光的百分比）
+     * @return 保量权重系数，范围[0.0, 2.0]，默认1.0
+     */
+    protected double calculateGuaranteedWeight(GuaranteeView guaranteeView) {
+        // 1. 空值检查：如果保量视图为空，返回默认权重
+        if (guaranteeView == null) {
+            return 1.0;
+        }
+
+        double guaranteeWeight;
+
+        // 2. 保量配置检查：如果保量上限或保量比例未配置，返回默认权重
+        if (guaranteeView.getGuaranteeNum() == null || guaranteeView.getGuaranteeNum() == 0
+                || guaranteeView.getGuaranteeRate() == null || guaranteeView.getGuaranteeRate() == 0.0) {
+            guaranteeWeight = 1.0;
+        } else {
+            // 3. 保量达标检查：如果广告主曝光数已达到或超过保量上限，停止加权
+            if (guaranteeView.getAdrAlgoViewNum() != null &&
+                    guaranteeView.getGuaranteeNum() <= guaranteeView.getAdrAlgoViewNum()) {
+                // 已达到保量目标，权重设为0，避免过度曝光
+                guaranteeWeight = 0.0;
+            } else {
+                // 4. 计算保量权重
+                // 防止除零错误：如果曝光数为空或0，设为1
+                int allViewNum = guaranteeView.getAllAlgoViewNum() == null || guaranteeView.getAllAlgoViewNum() == 0 ?
+                        1 : guaranteeView.getAllAlgoViewNum();
+                int adrAlogViewNum = guaranteeView.getAdrAlgoViewNum() == null || guaranteeView.getAdrAlgoViewNum() == 0 ?
+                        1 : guaranteeView.getAdrAlgoViewNum();
+
+                // 核心计算公式：权重 = 保量比例 × 0.01 × 全平台曝光数 / 广告主曝光数
+                // guaranteeView.getGuaranteeRate() 是百分之几，需要乘0.01转换为小数
+                // 逻辑：如果广告主当前曝光占比低于目标保量比例，给予更高权重提升排序
+                guaranteeWeight = guaranteeView.getGuaranteeRate() * 0.01 * allViewNum / adrAlogViewNum;
+
+                // 5. 权重边界限制：避免权重过小或过大影响系统稳定性
+                if (guaranteeWeight < 0.5) {
+                    guaranteeWeight = 0.5;  // 最小权重限制
+                }
+                if (guaranteeWeight > 2) {
+                    guaranteeWeight = 2;    // 最大权重限制
+                }
+            }
+        }
+        return guaranteeWeight;
+    }
+
+    protected boolean isGuaranteed(GuaranteeView guaranteeView) {
+        if (guaranteeView == null) {
+            return false;
+        }
+        return guaranteeView.getGuaranteeNum() != null && guaranteeView.getGuaranteeNum() != 0
+                && guaranteeView.getGuaranteeRate() != null && guaranteeView.getGuaranteeRate() != 0.0;
+    }
+
+    protected double getGuaranteeScoreCoefficient(boolean isGuaranteedFlow, Map<String, Object> ext) {
+        if (isGuaranteedFlow) {
+            if (ext.get("guaranteeWeight") == null) {
+                return 1.0;
+            } else {
+                return Math.pow((double) ext.get("guaranteeWeight"), alpha);
+            }
+        } else {
+            return 1.0;
+        }
+    }
+
+    protected Map<Long, CorrectCpaParam> getCorrectCpaParamMap(RankRecommendRequestParam request, ScoreParam scoreParam, Map<String, String> reqFeature) {
+        String layer = reqFeature.get("layer");
+        String clazz = reqFeature.get("clazz");
+        Map<Long, CorrectCpaParam> resultMap = new HashMap<>();
+        try {
+            if (CollectionUtils.isEmpty(request.getAdIdList())) {
+                return resultMap;
+            }
+            String redisAdLayerHourKey = adLayerHourKey.replace("{layer}", layer).replace("{clazz}", clazz);
+            String redisAdLayerDayKey = adLayerDayKey.replace("{layer}", layer).replace("{clazz}", clazz);
+            Function<Long, String> hourKeyFunc = id -> redisAdLayerHourKey.replace("{adId}", String.valueOf(id));
+            Function<Long, String> dayKeyFunc = id -> redisAdLayerDayKey.replace("{adId}", String.valueOf(id));
+            List<Long> adIds = request.getAdIdList().stream()
+                    .map(AdPlatformCreativeDTO::getAdId)
+                    .distinct()
+                    .collect(Collectors.toList());
+            Map<Long, JSONObject> adLayerMap = getRedisData(adIds, hourKeyFunc, dayKeyFunc);
+            adIds.forEach(adId -> {
+                JSONObject jsonObject = adLayerMap.get(adId);
+                CorrectCpaParam param = new CorrectCpaParam();
+
+                if (jsonObject == null) {
+                    param.setCorrectionFactor(DEFAULT_CORRECTION);
+                } else {
+                    // 设置基础参数
+                    param.setViewsHour(jsonObject.getInteger("viewsHour"));
+                    param.setViewsDay(jsonObject.getInteger("viewsDay"));
+                    param.setRealCtcvrHour(jsonObject.getDouble("realCtcvrHour"));
+                    param.setPCtcvrHour(jsonObject.getDouble("pCtcvrHour"));
+                    param.setRealCtcvrDay(jsonObject.getDouble("realCtcvrDay"));
+                    param.setPCtcvrDay(jsonObject.getDouble("pCtcvrDay"));
+                    // 计算校正因子
+                    double factor = calculateCorrection(jsonObject, scoreParam);
+                    param.setCorrectionFactor(factor > 0 ? factor : DEFAULT_CORRECTION);
+                }
+                resultMap.put(adId, param);
+            });
+        } catch (Exception e) {
+            log.error("getCorrectCpaParamMap error", e);
+        }
+        return resultMap;
+    }
+
+    private double calculateCorrection(JSONObject jsonObject, ScoreParam scoreParam) {
+        // 安全获取值
+        Integer viewsHour = jsonObject.getInteger("viewsHour");
+        Integer viewsDay = jsonObject.getInteger("viewsDay");
+        Double realCtcvrHour = safeDouble(jsonObject.getDouble("realCtcvrHour"), minCtcvr);
+        Double pCtcvrHour = jsonObject.getDouble("pCtcvrHour");
+        Double realCtcvrDay = safeDouble(jsonObject.getDouble("realCtcvrDay"), minCtcvr);
+        Double pCtcvrDay = jsonObject.getDouble("pCtcvrDay");
+
+        if (scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
+            double alpha = isValidViews(viewsHour, correctCpaViewHour) ? correctCpaAlpha2 : 0;
+            double beta = isValidViews(viewsDay, correctCpaViewDay) ? correctCpaBeta2 : 0;
+            return (1 - alpha - beta) +
+                    safeDivision(realCtcvrHour, pCtcvrHour) * alpha +
+                    safeDivision(realCtcvrDay, pCtcvrDay) * beta;
+        } else {
+            double alpha = isValidViews(viewsHour, correctCpaViewHour) ? correctCpaAlpha1 : 0;
+            double beta = isValidViews(viewsDay, correctCpaViewDay) ? correctCpaBeta1 : 0;
+            return Math.pow(safeDivision(realCtcvrHour, pCtcvrHour), alpha) *
+                    Math.pow(safeDivision(realCtcvrDay, pCtcvrDay), beta);
+        }
+    }
+
+    private Double safeDivision(Double realCtcvr, Double pCtcvr) {
+        if (pCtcvr == null || pCtcvr == 0) {
+            return 1.0;
+        }
+        if (realCtcvr == null || realCtcvr == 0) {
+            return 0.0;
+        }
+        return realCtcvr / pCtcvr;
+    }
+
+    private <T> Map<T, JSONObject> getRedisData(
+            Collection<T> ids,
+            Function<T, String> hourKeyBuilder,
+            Function<T, String> dayKeyBuilder) {
+
+        Map<T, JSONObject> resultMap = new HashMap<>();
+        if (CollectionUtils.isEmpty(ids)) {
+            return resultMap;
+        }
+
+        // 构建Keys
+        List<String> hourKeys = ids.stream()
+                .map(hourKeyBuilder)
+                .collect(Collectors.toList());
+        List<String> dayKeys = ids.stream()
+                .map(dayKeyBuilder)
+                .collect(Collectors.toList());
+
+        // 批量获取Redis值
+        List<String> hourValues = adRedisHelper.mget(hourKeys);
+        List<String> dayValues = adRedisHelper.mget(dayKeys);
+
+        // 解析数据
+        Iterator<T> idIter = ids.iterator();
+        for (int i = 0; i < ids.size(); i++) {
+            T id = idIter.next();
+            JSONObject jsonObj = new JSONObject();
+
+            // 解析小时数据
+            parseRedisValue(hourValues.get(i), jsonObj, "Hour", "ctcvr", "pCtcvr", "views");
+            // 解析天数据
+            parseRedisValue(dayValues.get(i), jsonObj, "Day", "ctcvr", "pCtcvr", "views");
+
+            resultMap.put(id, jsonObj);
+        }
+        return resultMap;
+    }
+
+    private void parseRedisValue(String value, JSONObject target,
+                                 String suffix, String... keys) {
+        if (StringUtils.isEmpty(value)) return;
+
+        try {
+            JSONObject source = JSONObject.parseObject(value);
+            for (String key : keys) {
+                Object val = source.get(key);
+                if (val != null) {
+                    String newKey = (key.equals("ctcvr") ? "realCtcvr" : key) + suffix;
+                    target.put(newKey, val);
+                }
+            }
+        } catch (Exception e) {
+            log.warn("Parse redis value failed: {}", value, e);
+        }
+    }
+
+    protected void calculateCtcvrScore(List<AdRankItem> items, RankRecommendRequestParam request, ScoreParam scoreParam, String modelName, Map<String, String> reqFeature) {
+        //判断是否走校准试验
+        if (scoreParam.getExpCodeSet().contains(calibrationCtcvrExp)) {
+            try {
+                calibrationDnnCtcvrScore(items, modelName, reqFeature);
+            } catch (Exception e) {
+                log.error("calibrationDnnCtcvrScore error", e);
+            }
+        }
+
+        if (scoreParam.getExpCodeSet().contains(calibrationCoefficientExp)) {
+            try {
+                calibrationCtcvrScore(items, request, reqFeature);
+            } catch (Exception e) {
+                log.error("calibrationCtcvrScore error", e);
+            }
+        }
+    }
+
+    protected void calibrationCtcvrScore(List<AdRankItem> items, RankRecommendRequestParam request, Map<String, String> reqFeature) {
+        String userLayerClass = reqFeature.get("layer") + "-" + reqFeature.get("clazz");
+        String cidLayerClassKey = cidLayerKey.replace("{userLayer}", userLayerClass);
+
+        // 3. 批量查询Redis
+        List<Long> cidList = items.stream().map(AdRankItem::getAdId).collect(Collectors.toList());
+        List<String> redisKeys = cidList.stream()
+                .map(cid -> cidLayerClassKey.replace("{cid}", cid.toString()))
+                .collect(Collectors.toList());
+
+        List<String> redisValues = algRedisHelper.mget(redisKeys);
+        Map<Long, CorrectCtcvrScoreParam> calibrationMap = parseRedisValues(cidList, redisValues);
+
+        // 4. 应用校准逻辑
+        applyCalibration(items, calibrationMap);
+    }
+
+    // 解析Redis返回值到Map
+    private Map<Long, CorrectCtcvrScoreParam> parseRedisValues(List<Long> cidList, List<String> values) {
+        Map<Long, CorrectCtcvrScoreParam> map = new HashMap<>();
+        for (int i = 0; i < cidList.size(); i++) {
+            String value = values.get(i);
+            if (StringUtils.isEmpty(value)) continue;
+
+            try {
+                JSONObject json = JSONObject.parseObject(value);
+                Integer view = json.getInteger("view");
+                Double ctcvr = json.getDouble("ctcvr");
+                if (view != null && ctcvr != null) {
+                    CorrectCtcvrScoreParam param = new CorrectCtcvrScoreParam();
+                    param.setView(view);
+                    param.setRealCtcvr(ctcvr);
+                    map.put(cidList.get(i), param);
+                }
+            } catch (Exception e) {
+                log.error("Failed to parse calibration data for cid={}", cidList.get(i), e);
+            }
+        }
+        return map;
+    }
+
+    // 应用校准到广告项
+    private void applyCalibration(List<AdRankItem> items, Map<Long, CorrectCtcvrScoreParam> calibrationMap) {
+        for (AdRankItem item : items) {
+            CorrectCtcvrScoreParam param = calibrationMap.get(item.getAdId());
+            if (param == null || param.getView() == null || param.getView() < calibrationView) {
+                continue; // 跳过无效数据
+            }
+
+            double realCtcvr = Optional.ofNullable(param.getRealCtcvr()).orElse(0.0);
+            double calibratedScore = item.getLrScore() * calibrationAlpha + (1 - calibrationAlpha) * realCtcvr;
+            item.getExt().put("correctCtcvrScoreParam", JSONObject.toJSONString(param));
+            item.getScoreMap().put("cidCorrectCtcvrScore", calibratedScore);
+            item.getScoreMap().put("ctcvrScore", calibratedScore);
+            item.setLrScore(calibratedScore);
+        }
+    }
+
+
+    protected AdRankItem creativeCovertRankItem(AdPlatformCreativeDTO dto, RankRecommendRequestParam request, Set<String> noApiAdVerIds) {
+        AdRankItem adRankItem = new AdRankItem();
+        adRankItem.setAdId(dto.getCreativeId());
+        adRankItem.setCreativeCode(dto.getCreativeCode());
+        adRankItem.setAdVerId(dto.getAdVerId());
+        adRankItem.setVideoId(request.getVideoId());
+        adRankItem.setCpa(dto.getCpa());
+        adRankItem.setId(dto.getAdId());
+        adRankItem.setCampaignId(dto.getCampaignId());
+        adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+        adRankItem.setSkuId(dto.getSkuId());
+        adRankItem.getExt().put("recallsources", dto.getRecallSources());
+        adRankItem.setRandom(new Random().nextInt(1000));
+        if (CollectionUtils.isNotEmpty(noApiAdVerIds)) {
+            if (noApiAdVerIds.contains(dto.getAdVerId())) {
+                adRankItem.getExt().put("isApi", "0");
+            } else {
+                adRankItem.getExt().put("isApi", "1");
+            }
+        }
+
+        return adRankItem;
+    }
+
+    protected void putMetaFeature(AdRankItem adRankItem, Feature feature, Map<String, String> reqFeature,
+                                  Map<String, String> sceneFeatureMap, RankRecommendRequestParam request) {
+        Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+        Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
+        Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
+        Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
+        for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
+            if (MapUtils.isNotEmpty(entry.getValue())) {
+                adRankItem.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
+            if (MapUtils.isNotEmpty(entry.getValue())) {
+                adRankItem.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(adRankItem.getAdVerId(), new HashMap<>());
+        for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
+            if (MapUtils.isNotEmpty(entry.getValue())) {
+                adRankItem.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(adRankItem.getAdId()), new HashMap<>());
+        for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
+            if (MapUtils.isNotEmpty(entry.getValue())) {
+                adRankItem.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(adRankItem.getSkuId()), new HashMap<>());
+        for (Map.Entry<String, Map<String, String>> entry : skuFeature.entrySet()) {
+            if (MapUtils.isNotEmpty(entry.getValue())) {
+                adRankItem.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+            }
+        }
+        long cid = adRankItem.getAdId();
+        List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
+        AdPlatformCreativeDTO adPlatformCreativeDTO = adIdList.stream()
+                .filter(dto -> cid == dto.getCreativeId())
+                .findFirst().orElse(null);
+        if (adPlatformCreativeDTO != null) {
+            reqFeature.put("cid", String.valueOf(adPlatformCreativeDTO.getCreativeId()));
+            reqFeature.put("adid", String.valueOf(adPlatformCreativeDTO.getAdId()));
+            reqFeature.put("adverid", String.valueOf(adPlatformCreativeDTO.getAdVerId()));
+            reqFeature.put("profession", adPlatformCreativeDTO.getProfession());
+        }
+        adRankItem.getMetaFeatureMap().put("reqFeature", reqFeature);
+        adRankItem.getMetaFeatureMap().put("sceneFeature", sceneFeatureMap);
+    }
+
+    protected Map<String, String> getUserLayer(String mid) {
+        if (StringUtils.isEmpty(mid)) {
+            return new HashMap<>();
+        }
+        String key = String.format("ad:engine:mid:layer:%s", mid);
+        String value = algRedisHelper.get(key);
+        if (StringUtils.isEmpty(value)) {
+            return new HashMap<>();
+        }
+        try {
+            Map<String, String> map = JSON.parseObject(value, new TypeReference<Map<String, String>>() {
+            });
+            if (map.containsKey("layer") && Objects.equals(map.get("layer"), "已转化")) {
+                map.put("layer", "有转化");
+            }
+            return map;
+        } catch (Exception e) {
+            log.error("getUserLayer error. mid: {}, \n", mid, e);
+            return new HashMap<>();
+        }
+    }
+
+    protected void calibrationDnnCtcvrScore(List<AdRankItem> adRankItems, String modelName, Map<String, String> reqFeature) {
+        if (StringUtils.isEmpty(modelName)) {
+            return;
+        }
+        // 构建Key模板
+        String layerKeyTemplate = realLayerCtcvrKey
+                .replace("{model}", modelName)
+                .replace("{layer}", reqFeature.get("layer"))
+                .replace("{class}", reqFeature.get("clazz"));
+
+
+        String value = adRedisHelper.get(layerKeyTemplate);
+        JSONObject json = JSONObject.parseObject(value);
+        Integer view = json.getInteger("view");
+        Double realCtcvr = json.getDouble("ctcvr");
+        Double pCtcvr = json.getDouble("pCtcvr");
+        CalibrationModelCtcvrData calibrationModelCtcvrData = new CalibrationModelCtcvrData();
+        calibrationModelCtcvrData.setRealCtcvr(realCtcvr);
+        calibrationModelCtcvrData.setView(view);
+        calibrationModelCtcvrData.setPCtcvr(pCtcvr);
+        for (AdRankItem item : adRankItems) {
+            item.getExt().put("calibrationModelCtcvrData", JSONObject.toJSONString(calibrationModelCtcvrData));
+            if (pCtcvr == null || pCtcvr == 0.0 || realCtcvr == null || realCtcvr == 0.0) {
+                continue;
+            }
+            double diff = realCtcvr / pCtcvr;
+            if (Math.abs(diff - 1) < 0.1) {
+                continue;
+            }
+            double correctCtcvrScore = item.getLrScore() * diff;
+            item.getScoreMap().put("layerCorrectCtcvrScore", correctCtcvrScore);
+            item.getScoreMap().put("ctcvrScore", correctCtcvrScore);
+            item.setLrScore(correctCtcvrScore);
+        }
+    }
+
+    // 安全的数值转换
+    private double safeDouble(Double value, Double min) {
+        return (value == null || value == 0) ? min : value;
+    }
+
+    // 视图数校验封装
+    private boolean isValidViews(Integer views, int threshold) {
+        return views != null && views >= threshold;
+    }
+
+    /**
+     * 根据用户人群layer获取对应的CrowdLayerEnum code
+     *
+     * @param userLayer 用户人群layer，如"无曝光"、"有转化"、"有曝光无转化"
+     * @return 对应的CrowdLayerEnum code，如果无法匹配则返回null
+     */
+    private Integer getUserCrowdCode(String userLayer) {
+        if (StringUtils.isEmpty(userLayer)) {
+            return null;
+        }
+
+        // 处理可能的"-炸"后缀
+        String cleanLayer = userLayer.replace("-炸", "");
+
+        // 使用CrowdLayerEnum提供的方法根据描述获取枚举对象
+        CrowdLayerEnum crowdLayerEnum = CrowdLayerEnum.getBytDesc(cleanLayer);
+        if (crowdLayerEnum != null) {
+            return crowdLayerEnum.getCode();
+        }
+
+        log.warn("无法匹配用户人群layer到CrowdLayerEnum: {}", userLayer);
+        return null;
+    }
+
+    /**
+     * 判断用户人群是否在保量人群范围内
+     *
+     * @param guaranteeCrowdCode 保量人群代码字符串，格式如"1,2,3"
+     * @param userCrowdCode 用户人群代码
+     * @return true表示用户在保量人群范围内
+     */
+    private boolean isUserInGuaranteeCrowd(String guaranteeCrowdCode, Integer userCrowdCode) {
+        if (StringUtils.isEmpty(guaranteeCrowdCode) || userCrowdCode == null) {
+            return false;
+        }
+
+        try {
+            // 将保量人群代码字符串按逗号分割
+            String[] crowdCodes = guaranteeCrowdCode.split(",");
+            for (String code : crowdCodes) {
+                if (String.valueOf(userCrowdCode).equals(code.trim())) {
+                    return true;
+                }
+            }
+        } catch (Exception e) {
+            log.error("解析保量人群代码失败: guaranteeCrowdCode={}", guaranteeCrowdCode, e);
+        }
+
+        return false;
+    }
+
+    /**
+     * 判断是否全部保量人群
+     * @param guaranteeCrowdCode 保量人群代码字符串
+     * @return true表示是全部保量人群
+     */
+    private boolean isFullCrowd(String guaranteeCrowdCode) {
+        if (StringUtils.isEmpty(guaranteeCrowdCode)) {
+            return false;
+        }
+
+        try {
+            // 获取CrowdLayerEnum中的所有code值
+            Set<String> allCrowdCodes = new HashSet<>();
+            for (CrowdLayerEnum crowdLayer : CrowdLayerEnum.values()) {
+                allCrowdCodes.add(String.valueOf(crowdLayer.getCode()));
+            }
+
+            // 解析保量人群代码
+            Set<String> guaranteeCodes = new HashSet<>();
+            String[] codes = guaranteeCrowdCode.split(",");
+            for (String code : codes) {
+                guaranteeCodes.add(code.trim());
+            }
+
+            // 判断保量人群代码是否为全量值（包含所有CrowdLayerEnum的code）
+            boolean isFullCrowd = guaranteeCodes.containsAll(allCrowdCodes);
+            return isFullCrowd;
+
+        } catch (Exception e) {
+            log.error("判断保量人群加权失败: guaranteeCrowdCode={}",
+                     guaranteeCrowdCode, e);
+            return false;
+        }
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
+import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
@@ -7,9 +8,12 @@ import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.CorrectCpaParam;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
@@ -62,6 +66,8 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         }
 
         long start = System.currentTimeMillis();
+        //过滤创意
+        filterRequestAdList(request, scoreParam);
         // 特征处理
         // feature1
         Feature feature = this.getFeature(scoreParam, request);
@@ -70,6 +76,9 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
+        Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
+
 
         Map<String, String> userFeatureMap = new HashMap<>();
         Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
@@ -88,7 +97,9 @@ public class RankStrategyBy679 extends RankStrategyBasic {
 
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
-
+        boolean isGuaranteedFlow = getIsGuaranteedFlow(scoreParam);
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, isGuaranteedFlow);
+        Map<Long, CorrectCpaParam> correctCpaMap = getCorrectCpaParamMap(request, scoreParam, reqFeature);
         List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
@@ -105,6 +116,9 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                     adRankItem.setId(dto.getAdId());
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setSkuId(dto.getSkuId());
+                    adRankItem.setCustomerId(dto.getCustomerId());
+                    adRankItem.setProfession(dto.getProfession());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -112,7 +126,9 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                         adRankItem.getExt().put("isApi", "1");
                     }
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-
+                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
+                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow, reqFeature);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
@@ -206,50 +222,38 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         // getScorerPipeline
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20241105).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
         long time5 = System.currentTimeMillis();
+        calculateCtcvrScore(result, request, scoreParam, null, reqFeature);
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
-        for (AdRankItem item : result) {
 
+        boolean isGuaranteeType = false;
+        for (AdRankItem item : result) {
+            double bid = item.getCpa();
+            if (scoreParam.getExpCodeSet().contains(correctCpaExp1) || scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
+                Double correctionFactor = (Double) item.getExt().get("correctionFactor");
+                item.getScoreMap().put("correctionFactor", correctionFactor);
+                bid = bid * correctionFactor;
+            }
+            item.getScoreMap().put("ecpm", item.getLrScore() * bid * 1000);
+            if (isGuaranteedFlow && item.getExt().get("isGuaranteed") != null && (boolean) item.getExt().get("isGuaranteed")) {
+                isGuaranteeType = true;
+            }
             double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
-            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
-
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(isGuaranteedFlow, item.getExt());
+            double score = item.getLrScore() * bid * scoreCoefficient * guaranteeScoreCoefficient;
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
+            item.getScoreMap().put("bid", bid);
             item.getScoreMap().put("cpmCoefficient", cpmCoefficient);
             item.getScoreMap().put("scoreCoefficient", scoreCoefficient);
             item.getFeatureMap().putAll(userFeatureMap);
             item.getFeatureMap().putAll(sceneFeatureMap);
-
             // 没有转化回传的广告主，使用后台配置的CPM
             if (noApiAdVerIds.contains(item.getAdVerId())) {
-                item.setScore(item.getCpm() * cpmCoefficient / 1000);
-            }
-
-            for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(item.getAdVerId(), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
+                score = item.getCpm() * cpmCoefficient / 1000;
             }
+            item.setScore(score);
         }
 
         log.info("cost={}, feature1={}, feature2={}, feature31={}, feature32={}, feature4={}, getScorerPipeline={}, " +
@@ -259,6 +263,11 @@ public class RankStrategyBy679 extends RankStrategyBasic {
 
         result.sort(ComparatorUtil.equalsRandomComparator());
 
+        if (CollectionUtils.isNotEmpty(result)) {
+            AdRankItem top1Item = result.get(0);
+            top1Item.getExt().put("isGuaranteeType", isGuaranteeType);
+            putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+        }
         return result;
     }
 
@@ -298,11 +307,14 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
+                double ecpm = NumUtil.div(income * 1000, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -328,11 +340,14 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
+                double ecpm = NumUtil.div(income * 1000, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -370,6 +385,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
         featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
         featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
 
         return midActionList;
     }
@@ -423,6 +439,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
             featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
             featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
             featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
         }
     }
 
@@ -431,7 +448,8 @@ public class RankStrategyBy679 extends RankStrategyBasic {
             return;
         }
 
-        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
         for (String prefix1 : prefixes1) {
@@ -551,13 +569,6 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         return vidRankMaps;
     }
 
-    public Map<String, String> handleSceneFeature(long ts) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        sceneFeatureMap.put("hour_" + DateUtils.getHourByTimestamp(ts), "0.1");
-        sceneFeatureMap.put("dayofweek_" + DateUtils.getDayOrWeekByTimestamp(ts), "0.1");
-        return sceneFeatureMap;
-    }
-
     private void readBucketFile() {
         if (MapUtils.isNotEmpty(bucketsMap)) {
             return;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -1,10 +1,13 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
+import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.CorrectCpaParam;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
@@ -17,6 +20,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
+import javax.annotation.PostConstruct;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -28,6 +32,8 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static com.tzld.piaoquan.ad.engine.commons.math.Const.*;
+
 @Slf4j
 @Component
 public class RankStrategyBy680 extends RankStrategyBasic {
@@ -39,14 +45,26 @@ public class RankStrategyBy680 extends RankStrategyBasic {
     @Value("${word2vec.exp:694}")
     private String word2vecExp;
 
+    @Value("${swap.exp:772}")
+    protected String excludeExp;
+
     @ApolloJsonValue("${rank.score.weight.680:{}}")
     private Map<String, Double> weightMap;
 
+    @ApolloJsonValue("${rank.score.neg_sample_rate:0.04}")
+    Double negSampleRate;
+
+    @PostConstruct
+    public void afterInit() {
+        this.readBucketFile();
+    }
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
         Map<String, Double> weightParam = ObjUtil.nullOrDefault(weightMap, new HashMap<>());
 
+
         Map<Long, Double> creativeScoreCoefficient = getCreativeScoreCoefficient();
         Set<String> noApiAdVerIds = getNoApiAdVerIds();
 
@@ -58,6 +76,8 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         }
 
         long start = System.currentTimeMillis();
+        //过滤创意
+        filterRequestAdList(request, scoreParam);
         // 特征处理
         // feature1
         Feature feature = this.getFeature(scoreParam, request);
@@ -66,9 +86,13 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
+        Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
+
 
         Map<String, String> userFeatureMap = new HashMap<>();
         Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
+
         List<TupleMapEntry<Tuple5>> midActionList = this.handleC1Feature(c1Feature, userFeatureMap);
 
         Map<String, Double> midTimeDiffMap = this.parseC1FeatureListToTimeDiffMap(midActionList, ts);
@@ -85,6 +109,9 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
+        boolean isGuaranteedFlow = getIsGuaranteedFlow(scoreParam);
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, isGuaranteedFlow);
+        Map<Long, CorrectCpaParam> correctCpaMap = getCorrectCpaParamMap(request, scoreParam, reqFeature);
         List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
@@ -101,6 +128,8 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                     adRankItem.setId(dto.getAdId());
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setSkuId(dto.getSkuId());
+                    adRankItem.setCustomerId(dto.getCustomerId());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -109,7 +138,9 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                     }
 
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-
+                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
+                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow, reqFeature);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
@@ -178,7 +209,6 @@ public class RankStrategyBy680 extends RankStrategyBasic {
 
         long time3 = System.currentTimeMillis();
         // 分桶
-        this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
         CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
         for (AdRankItem adRankItem : adRankItems) {
@@ -201,57 +231,57 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         // getScorerPipeline
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20240909).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
         long time5 = System.currentTimeMillis();
+
+        // calibrate score for negative sampling
+        for (AdRankItem item : result) {
+            double originalScore = item.getLrScore();
+            double calibratedScore = originalScore / (originalScore + (1 - originalScore) / negSampleRate);
+            item.setLrScore(calibratedScore);
+            item.getScoreMap().put("originCtcvrScore", originalScore);
+            item.getScoreMap().put("modelCtcvrScore", calibratedScore);
+            item.getScoreMap().put("ctcvrScore", calibratedScore);
+        }
+
+        calculateCtcvrScore(result, request, scoreParam, null, reqFeature);
         // loop
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
 
+        boolean isGuaranteeType = false;
         for (AdRankItem item : result) {
-
+            double bid = item.getCpa();
+            if (scoreParam.getExpCodeSet().contains(correctCpaExp1) || scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
+                Double correctionFactor = (Double) item.getExt().get("correctionFactor");
+                item.getScoreMap().put("correctionFactor", correctionFactor);
+                bid = bid * correctionFactor;
+            }
+            item.getScoreMap().put("ecpm", item.getLrScore() * bid * 1000);
+            if (isGuaranteedFlow && item.getExt().get("isGuaranteed") != null && (boolean) item.getExt().get("isGuaranteed")) {
+                isGuaranteeType = true;
+            }
             double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
-            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
-
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(isGuaranteedFlow, item.getExt());
+            double score = item.getLrScore() * bid * scoreCoefficient * guaranteeScoreCoefficient;
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
+            item.getScoreMap().put("bid", bid);
             item.getScoreMap().put("cpmCoefficient", cpmCoefficient);
             item.getScoreMap().put("scoreCoefficient", scoreCoefficient);
             item.getFeatureMap().putAll(userFeatureMap);
             item.getFeatureMap().putAll(sceneFeatureMap);
-
             // 没有转化回传的广告主，使用后台配置的CPM
             if (noApiAdVerIds.contains(item.getAdVerId())) {
-                item.setScore(item.getCpm() * cpmCoefficient / 1000);
+                score = item.getCpm() * cpmCoefficient / 1000;
             }
+            item.setScore(score);
         }
-
-
         result.sort(ComparatorUtil.equalsRandomComparator());
 
         if (CollectionUtils.isNotEmpty(result)) {
             AdRankItem top1Item = result.get(0);
-            for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(top1Item.getAdVerId(), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(top1Item.getAdId()), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
+            top1Item.getExt().put("isGuaranteeType", isGuaranteeType);
+            putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+            top1Item.getExt().put("model", "xgb");
         }
         long time6 = System.currentTimeMillis();
         log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +
@@ -306,15 +336,20 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
-                double f2 = NumUtil.div(conver, view);
-                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
-                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                double cpc = NumUtil.div(income, click);
+                double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+                double ctcvr = NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR);
+                double ecpm = ctr * cpc * 1000;
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(ctr));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(ctcvr));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * ctcvr));
             }
         }
 
@@ -337,16 +372,19 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
                 double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
-                double f2 = NumUtil.div(conver, view);
-                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
-                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                double cpc = NumUtil.div(income, click);
+                double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+                double ctcvr = NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR);
+                double ecpm = ctr * cpc * 1000;
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(ctr));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(ctcvr));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * ctcvr));
             }
         }
 
@@ -380,7 +418,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
         featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
         featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
-        // featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
 
         return midActionList;
     }
@@ -430,11 +468,13 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             double click = Double.parseDouble(d1Feature.getOrDefault("ad_click_" + prefix, "0"));
             double conver = Double.parseDouble(d1Feature.getOrDefault("ad_conversion_" + prefix, "0"));
             double income = Double.parseDouble(d1Feature.getOrDefault("ad_income_" + prefix, "0"));
-            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(NumUtil.div(click, view)));
-            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
-            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+            double cpc = NumUtil.div(income, click);
+            double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(ctr));
+            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)));
+            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
             featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
-            // featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(ctr * cpc * 1000));
         }
     }
 
@@ -443,8 +483,8 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             return;
         }
 
-        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
-        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
         for (String prefix1 : prefixes1) {
@@ -565,19 +605,13 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         return vidRankMaps;
     }
 
-    public Map<String, String> handleSceneFeature(long ts) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        sceneFeatureMap.put("hour_" + DateUtils.getHourByTimestamp(ts), "0.1");
-        sceneFeatureMap.put("dayofweek_" + DateUtils.getDayOrWeekByTimestamp(ts), "0.1");
-        return sceneFeatureMap;
-    }
-
     private void readBucketFile() {
         if (MapUtils.isNotEmpty(bucketsMap)) {
             return;
         }
         synchronized (this) {
-            InputStream resourceStream = RankStrategyBy680.class.getClassLoader().getResourceAsStream("20240718_ad_bucket_688.txt");
+            String bucketFile = "20250217_ad_bucket_688.txt";
+            InputStream resourceStream = RankStrategyBy680.class.getClassLoader().getResourceAsStream(bucketFile);
             if (resourceStream != null) {
                 try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
                     Map<String, double[]> bucketsMap = new HashMap<>();
@@ -600,8 +634,9 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                     this.bucketsMap = bucketsMap;
                     this.bucketsLen = bucketsLen;
                 } catch (IOException e) {
-                    log.error("something is wrong in parse bucket file:", e);
+                    log.error("something is wrong in parse bucket file: ", e);
                 }
+                log.info("load bucket file success: {}", bucketFile);
             } else {
                 log.error("no bucket file");
             }
@@ -629,4 +664,4 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         return newFeatureMap;
     }
 
-}
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -1,19 +1,26 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.CorrectCpaParam;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
+import javax.annotation.PostConstruct;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -25,6 +32,8 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static com.tzld.piaoquan.ad.engine.commons.math.Const.*;
+
 @Slf4j
 @Component
 public class RankStrategyBy683 extends RankStrategyBasic {
@@ -33,13 +42,40 @@ public class RankStrategyBy683 extends RankStrategyBasic {
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
+    @Value("${word2vec.exp:694}")
+    private String word2vecExp;
+
+    // FIXME(zhoutian): 可能需要独立配置
+    @ApolloJsonValue("${rank.score.weight.680:{}}")
+    private Map<String, Double> weightMap;
+
+    @ApolloJsonValue("${rank.score.neg_sample_rate:0.01}")
+    Double negSampleRate;
+
+    @PostConstruct
+    public void afterInit() {
+        this.readBucketFile();
+    }
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+
+        Map<String, Double> weightParam = ObjUtil.nullOrDefault(weightMap, new HashMap<>());
+
+
+        Map<Long, Double> creativeScoreCoefficient = getCreativeScoreCoefficient();
         Set<String> noApiAdVerIds = getNoApiAdVerIds();
 
         long ts = System.currentTimeMillis() / 1000;
 
+        String brand = scoreParam.getRequestContext().getMachineinfoBrand();
+        if (StringUtils.isNotEmpty(brand)) {
+            scoreParam.getRequestContext().setMachineinfoBrand(brand + "-n");
+        }
+
         long start = System.currentTimeMillis();
+        //过滤创意
+        filterRequestAdList(request, scoreParam);
         // 特征处理
         // feature1
         Feature feature = this.getFeature(scoreParam, request);
@@ -48,6 +84,9 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
+        Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
+
 
         Map<String, String> userFeatureMap = new HashMap<>();
         Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
@@ -67,7 +106,10 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
-        List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
+        boolean isGuaranteedFlow = getIsGuaranteedFlow(scoreParam);
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, isGuaranteedFlow);
+        Map<Long, CorrectCpaParam> correctCpaMap = getCorrectCpaParamMap(request, scoreParam, reqFeature);
+        List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
         CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
@@ -83,14 +125,19 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     adRankItem.setId(dto.getAdId());
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setSkuId(dto.getSkuId());
+                    adRankItem.setCustomerId(dto.getCustomerId());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
                     } else {
                         adRankItem.getExt().put("isApi", "1");
                     }
-                    adRankItem.getExt().put("recallsources", dto.getRecallSources());
 
+                    adRankItem.getExt().put("recallsources", dto.getRecallSources());
+                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
+                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow, reqFeature);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
@@ -138,30 +185,27 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             String title = b1Feature.getOrDefault("cidtitle", "");
             ThreadPoolFactory.defaultPool().submit(() -> {
                 try {
-                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap(), scoreParam);
                 } finally {
                     cdl2.countDown();
                 }
             });
             ThreadPoolFactory.defaultPool().submit(() -> {
                 try {
-                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap(), scoreParam);
                 } finally {
                     cdl2.countDown();
                 }
             });
         }
-        long time31 = System.currentTimeMillis();
         try {
             cdl2.await(150, TimeUnit.MILLISECONDS);
         } catch (Exception e) {
             log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
         }
 
-        // feature4
         long time3 = System.currentTimeMillis();
         // 分桶
-        this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
         CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
         for (AdRankItem adRankItem : adRankItems) {
@@ -184,53 +228,64 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         // getScorerPipeline
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_683).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
         long time5 = System.currentTimeMillis();
+
+        // calibrate score for negative sampling
+        for (AdRankItem item : result) {
+            double originalScore = item.getLrScore();
+            double calibratedScore = originalScore / (originalScore + (1 - originalScore) / negSampleRate);
+            item.setLrScore(calibratedScore);
+            item.getScoreMap().put("originCtcvrScore", originalScore);
+            item.getScoreMap().put("modelCtcvrScore", calibratedScore);
+            item.getScoreMap().put("ctcvrScore", calibratedScore);
+        }
+
+        calculateCtcvrScore(result, request, scoreParam, null, reqFeature);
         // loop
+        double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
+        boolean isGuaranteeType = false;
         for (AdRankItem item : result) {
-            item.setScore(item.getLrScore() * item.getCpa());
+            if (isGuaranteedFlow && item.getExt().get("isGuaranteed") != null && (boolean) item.getExt().get("isGuaranteed")) {
+                isGuaranteeType = true;
+            }
+            double bid = item.getCpa();
+            if (scoreParam.getExpCodeSet().contains(correctCpaExp1) || scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
+                Double correctionFactor = (Double) item.getExt().get("correctionFactor");
+                item.getScoreMap().put("correctionFactor", correctionFactor);
+                bid = bid * correctionFactor;
+            }
+            item.getScoreMap().put("ecpm", item.getLrScore() * bid * 1000);
+            double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(isGuaranteedFlow, item.getExt());
+            double score = item.getLrScore() * bid * scoreCoefficient * guaranteeScoreCoefficient;
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
+            item.getScoreMap().put("bid", bid);
+            item.getScoreMap().put("cpmCoefficient", cpmCoefficient);
+            item.getScoreMap().put("scoreCoefficient", scoreCoefficient);
             item.getFeatureMap().putAll(userFeatureMap);
             item.getFeatureMap().putAll(sceneFeatureMap);
-
             // 没有转化回传的广告主，使用后台配置的CPM
             if (noApiAdVerIds.contains(item.getAdVerId())) {
-                item.setScore(item.getCpm() / 1000);
-            }
-
-            for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(item.getAdVerId(), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
+                score = item.getCpm() * cpmCoefficient / 1000;
             }
+            item.setScore(score);
         }
 
-        log.info("cost={}, feature1={}, feature2={}, feature31={}, feature32={}, feature4={}, getScorerPipeline={}, " +
-                        "adIdSize={}, adRankItemsSize={}",
-                time5 - start, time1 - start, time2 - time1, time31 - time2, time3 - time31, time4 - time3,
-                time5 - time4, request.getAdIdList().size(), adRankItems.size());
 
         result.sort(ComparatorUtil.equalsRandomComparator());
 
+        if (CollectionUtils.isNotEmpty(result)) {
+            AdRankItem top1Item = result.get(0);
+            top1Item.getExt().put("isGuaranteeType", isGuaranteeType);
+            putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+        }
+        long time6 = System.currentTimeMillis();
+        log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +
+                        "other={}, adIdSize={}, adRankItemsSize={}",
+                time6 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3,
+                time5 - time4, time6 - time5, request.getAdIdList().size(), adRankItems.size());
+
         return result;
     }
 
@@ -278,15 +333,20 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
-                double f2 = NumUtil.div(conver, view);
-                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
-                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                double cpc = NumUtil.div(income, click);
+                double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+                double ctcvr = NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR);
+                double ecpm = ctr * cpc * 1000;
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(ctr));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(ctcvr));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * ctcvr));
             }
         }
 
@@ -309,16 +369,19 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
                 double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
-                double f2 = NumUtil.div(conver, view);
-                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
-                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                double cpc = NumUtil.div(income, click);
+                double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+                double ctcvr = NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR);
+                double ecpm = ctr * cpc * 1000;
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(ctr));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(ctcvr));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * ctcvr));
             }
         }
 
@@ -352,7 +415,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
         featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
         featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
-        // featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
 
         return midActionList;
     }
@@ -387,17 +450,11 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             featureMap.put("actionstatic_ctr", String.valueOf(ctr));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_conver_" + cid)) {
-            double ctcvr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
-            );
+            double ctcvr = NumUtil.div(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0), midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0));
             featureMap.put("actionstatic_ctcvr", String.valueOf(ctcvr));
         }
         if (midActionStatic.containsKey("actionstatic_conver_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            double cvr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)
-            );
+            double cvr = NumUtil.div(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0), midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0));
             featureMap.put("actionstatic_cvr", String.valueOf(cvr));
         }
     }
@@ -408,11 +465,13 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             double click = Double.parseDouble(d1Feature.getOrDefault("ad_click_" + prefix, "0"));
             double conver = Double.parseDouble(d1Feature.getOrDefault("ad_conversion_" + prefix, "0"));
             double income = Double.parseDouble(d1Feature.getOrDefault("ad_income_" + prefix, "0"));
-            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(NumUtil.div(click, view)));
-            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
-            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+            double cpc = NumUtil.div(income, click);
+            double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(ctr));
+            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)));
+            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
             featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
-            // featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(ctr * cpc * 1000));
         }
     }
 
@@ -421,8 +480,8 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             return;
         }
 
-        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
-        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
         for (String prefix1 : prefixes1) {
@@ -438,24 +497,28 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         }
     }
 
-    private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap) {
+    private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap,
+                                      ScoreParam scoreParam) {
         if (MapUtils.isEmpty(d3Feature) || !d3Feature.containsKey("title") || StringUtils.isEmpty(cTitle)) {
             return;
         }
         String vTitle = d3Feature.get("title");
-        double score = Similarity.conceptSimilarity(cTitle, vTitle);
+        double score;
+        if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
+            score = SimilarityUtils.word2VecSimilarity(cTitle, vTitle);
+        } else {
+            score = Similarity.conceptSimilarity(cTitle, vTitle);
+        }
         featureMap.put("ctitle_vtitle_similarity", String.valueOf(score));
     }
 
-    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title, Map<String, String> featureMap) {
+    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title,
+                                      Map<String, String> featureMap, ScoreParam scoreParam) {
         if (StringUtils.isEmpty(title)) {
             return;
         }
 
-        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(
-                new Tuple2<>(e1Feature, "e1"),
-                new Tuple2<>(e2Feature, "e2")
-        );
+        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(new Tuple2<>(e1Feature, "e1"), new Tuple2<>(e2Feature, "e2"));
 
         List<String> tagsFieldList = Arrays.asList("tags_3d", "tags_7d", "tags_14d");
         for (Tuple2<Map<String, String>, String> tuple2 : tuple2List) {
@@ -468,7 +531,13 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             for (String tagsField : tagsFieldList) {
                 if (StringUtils.isNotEmpty(feature.get(tagsField))) {
                     String tags = feature.get(tagsField);
-                    Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    // Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    Double[] doubles;
+                    if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
+                        doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
+                    } else {
+                        doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    }
                     featureMap.put(prefix + "_" + tagsField + "_matchnum", String.valueOf(doubles[0]));
                     featureMap.put(prefix + "_" + tagsField + "_maxscore", String.valueOf(doubles[1]));
                     featureMap.put(prefix + "_" + tagsField + "_avgscore", String.valueOf(doubles[2]));
@@ -527,27 +596,19 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
             String key = entry.getKey();
             String value = entry.getValue();
-            Map<String, Double> valueMap = Arrays.stream(value.split(","))
-                    .map(r -> r.split(":"))
-                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
+            Map<String, Double> valueMap = Arrays.stream(value.split(",")).map(r -> r.split(":")).collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
             vidRankMaps.put(key, valueMap);
         }
         return vidRankMaps;
     }
 
-    public Map<String, String> handleSceneFeature(long ts) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        sceneFeatureMap.put("hour_" + DateUtils.getHourByTimestamp(ts), "0.1");
-        sceneFeatureMap.put("dayofweek_" + DateUtils.getDayOrWeekByTimestamp(ts), "0.1");
-        return sceneFeatureMap;
-    }
-
     private void readBucketFile() {
         if (MapUtils.isNotEmpty(bucketsMap)) {
             return;
         }
         synchronized (this) {
-            InputStream resourceStream = RankStrategyBy683.class.getClassLoader().getResourceAsStream("20240718_ad_bucket_688.txt");
+            String bucketFile = "20250217_ad_bucket_688.txt";
+            InputStream resourceStream = RankStrategyBy683.class.getClassLoader().getResourceAsStream(bucketFile);
             if (resourceStream != null) {
                 try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
                     Map<String, double[]> bucketsMap = new HashMap<>();
@@ -562,9 +623,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                                 String key = rList[0];
                                 double value1 = Double.parseDouble(rList[1]);
                                 bucketsLen.put(key, value1);
-                                double[] value2 = Arrays.stream(rList[2].split(","))
-                                        .mapToDouble(Double::valueOf)
-                                        .toArray();
+                                double[] value2 = Arrays.stream(rList[2].split(",")).mapToDouble(Double::valueOf).toArray();
                                 bucketsMap.put(key, value2);
                             }
                         }
@@ -572,8 +631,9 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     this.bucketsMap = bucketsMap;
                     this.bucketsLen = bucketsLen;
                 } catch (IOException e) {
-                    log.error("something is wrong in parse bucket file:", e);
+                    log.error("something is wrong in parse bucket file: ", e);
                 }
+                log.info("load bucket file success: {}", bucketFile);
             } else {
                 log.error("no bucket file");
             }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy687.java

@@ -1,636 +1,311 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
-import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
-import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
-import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
-import com.tzld.piaoquan.ad.engine.commons.util.*;
-import com.tzld.piaoquan.ad.engine.service.feature.Feature;
+import com.alibaba.fastjson.JSON;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.util.WeightRandom;
+import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
-import org.xm.Similarity;
 
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 @Slf4j
 @Service
 public class RankStrategyBy687 extends RankStrategyBasic {
 
-    private Map<String, double[]> bucketsMap = new HashMap<>();
+    @Autowired
+    private RankStrategyBy688 rankStrategyBy688;
 
-    private Map<String, Double> bucketsLen = new HashMap<>();
+    @ApolloJsonValue("${cid.open.hot.rank.user.layer:[]}")
+    private Set<String> openHotRankUserLayers;
 
-    @Value("${similarity.concurrent: false}")
-    private boolean similarityConcurrent;
-
-    @Override
-    public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
-        Set<String> noApiAdVerIds = getNoApiAdVerIds();
-
-        long ts = System.currentTimeMillis() / 1000;
-
-        String brand = scoreParam.getRequestContext().getMachineinfoBrand();
-        if (StringUtils.isNotEmpty(brand)) {
-            scoreParam.getRequestContext().setMachineinfoBrand(brand + "-n");
-        }
-
-        long start = System.currentTimeMillis();
-        // 特征处理
-        Feature feature = this.getFeature(scoreParam, request);
-
-        Map<String, Map<String, String>> userFeature = feature.getUserFeature();
-        Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
-        Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
-        Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
-
-        Map<String, String> userFeatureMap = new HashMap<>();
-        Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
-        List<TupleMapEntry<Tuple5>> midActionList = this.handleC1Feature(c1Feature, userFeatureMap);
-
-        Map<String, Double> midTimeDiffMap = this.parseC1FeatureListToTimeDiffMap(midActionList, ts);
-        Map<String, Double> actionStaticMap = this.parseC1FeatureListToActionStaticMap(midActionList);
-
-        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
-        Map<String, String> d3Feature = videoFeature.getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
-
-        Map<String, Map<String, Double>> vidRankMaps = this.parseD2FeatureMap(d2Feature);
-
-        Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
-        Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
-
-        Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
-        long time1 = System.currentTimeMillis();
-
-        List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
-        Random random = new Random();
-        if (similarityConcurrent) {
-            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-                AdRankItem adRankItem = new AdRankItem();
-                adRankItem.setAdId(dto.getCreativeId());
-                adRankItem.setCreativeCode(dto.getCreativeCode());
-                adRankItem.setAdVerId(dto.getAdVerId());
-                adRankItem.setVideoId(request.getVideoId());
-                adRankItem.setCpa(dto.getCpa());
-                adRankItem.setId(dto.getAdId());
-                adRankItem.setCampaignId(dto.getCampaignId());
-                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                adRankItem.setRandom(random.nextInt(1000));
-                if (noApiAdVerIds.contains(dto.getAdVerId())) {
-                    adRankItem.getExt().put("isApi", "0");
-                } else {
-                    adRankItem.getExt().put("isApi", "1");
-                }
-                adRankItem.getExt().put("recallsources", dto.getRecallSources());
+    @Value("${hot.rank.customer.view.threshold:5000}")
+    private Integer hotRankCustomerViewThreshold;
 
+    @Value("${hot.rank.creative.view.threshold:5000}")
+    private Integer hotRankCreativeViewThreshold;
 
-                String cidStr = dto.getCreativeId().toString();
-                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+    @Value("${hot.rank.calc.weight.coefficient:2.5}")
+    private Double hotRankCalcWeightCoefficient;
 
-                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+    @Value("${hot.rank.max.weight:1000}")
+    private Double hotRankMaxWeight;
 
-                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+    private static final String CUSTOMER_FIELD_NAME = "customer_name";
 
-                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-
-                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-
-                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-
-                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-
-                this.handleD1Feature(d1Feature, cidFeatureMap);
-
-                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+    @Override
+    public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+        // 没有mid或者未配置分层，走基线排序
+        if (StringUtils.isEmpty(request.getMid()) || CollectionUtils.isEmpty(openHotRankUserLayers)) {
+            return rankStrategyBy688.adItemRank(request, scoreParam);
+        }
 
+        // 当前分层未开启此策略
+        String userLayer = this.getUserLayer(request);
+        if (!openHotRankUserLayers.contains(userLayer)) {
+            return rankStrategyBy688.adItemRank(request, scoreParam);
+        }
 
-                // adRankItem.setFeatureMap(cidFeatureMap);
+        Set<String> noApiAdVerIds = getNoApiAdVerIds();
 
-                adRankItems.add(adRankItem);
+        List<AdPlatformCreativeDTO> recallCreativeList = request.getAdIdList();
 
-            }
+        // 获取所有创意对应的客户列表
+        Feature feature = this.getFeature(scoreParam, request);
 
+        List<AdRankItem> rankItems = new ArrayList<>(recallCreativeList.size());
+        for (AdPlatformCreativeDTO dto : recallCreativeList) {
+            AdRankItem adRankItem = this.creativeCovertRankItem(dto, request, noApiAdVerIds);
+            this.fullCustomerInfo(feature.getCidFeature(), adRankItem);
+            adRankItem.getExt().put("isHotRank", true);
+            rankItems.add(adRankItem);
+        }
 
-            CountDownLatch cdl = new CountDownLatch(adRankItems.size());
-            for (AdRankItem item : adRankItems) {
-                ThreadPoolFactory.defaultPool().submit(() -> {
-                    try {
-                        String cidStr = String.valueOf(item.getAdId());
-                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-                        String title = b1Feature.getOrDefault("cidtitle", "");
-                        long time21 = System.currentTimeMillis();
-                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
-                        long time22 = System.currentTimeMillis();
-                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
-                        long time23 = System.currentTimeMillis();
-                        log.info("cost={} handleE1AndE2Feature={} handleD3AndB1Feature={}", time23 - time21, time22 - time21, time23 - time22);
-                    } finally {
-                        cdl.countDown();
-                    }
-                });
-            }
-            try {
-                cdl.await(200, TimeUnit.MILLISECONDS);
-            } catch (Exception e) {
-                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        // 选择一个客户
+        List<String> customers = rankItems.stream().map(i -> i.getExt().getOrDefault(CUSTOMER_FIELD_NAME, ""))
+                .map(Objects::toString)
+                .filter(StringUtils::isNotEmpty)
+                .distinct()
+                .collect(Collectors.toList());
+        String keyFormat = "ad:engine:customer:layer:info:%s:" + userLayer;
+        List<HotRankFeatureInfo> customerFeature = this.multiGetFeature(customers, keyFormat);
+        String customer = this.choose(customerFeature, hotRankCustomerViewThreshold);
+
+        // 从当前客户的所有创意中，选择一个创意
+        List<String> customerCreativeIds = rankItems.stream().filter(c -> StringUtils.equals(c.getExt().getOrDefault(CUSTOMER_FIELD_NAME, "").toString(), customer))
+                .map(AdRankItem::getAdId)
+                .map(Objects::toString)
+                .collect(Collectors.toList());
+        String cidKeyFormat = "ad:engine:cid:layer:info:%s:" + userLayer;
+        List<HotRankFeatureInfo> creativeFeature = this.multiGetFeature(customerCreativeIds, cidKeyFormat);
+        String chooseCreativeIdStr = this.choose(creativeFeature, hotRankCreativeViewThreshold);
+        Long chooseCreativeId = Long.parseLong(chooseCreativeIdStr);
+
+        Map<String, HotRankFeatureInfo> allCustomerFeatureMap = customerFeature.stream()
+                .collect(Collectors.toMap(HotRankFeatureInfo::getLabel, Function.identity(), (o1, o2) -> o1));
+
+        Map<Long, HotRankFeatureInfo> allCreativeFeatureInfoMap = creativeFeature.stream()
+                .collect(Collectors.toMap(i -> Long.parseLong(i.getLabel()), Function.identity(), (o1, o2) -> o1));
+
+        Map<String, String> sceneFeature = this.handleSceneFeature(System.currentTimeMillis() / 1000);
+        Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
+
+        List<AdRankItem> result = new ArrayList<>(rankItems.size());
+        for (AdRankItem rankItem : rankItems) {
+            // 补充特征信息
+            this.fullMetaFeature(rankItem, userLayer, allCustomerFeatureMap, allCreativeFeatureInfoMap, feature);
+            this.putMetaFeature(rankItem, feature, reqFeature, sceneFeature, request);
+            if (Objects.equals(rankItem.getAdId(), chooseCreativeId)) {
+                result.add(0, rankItem);
+            } else {
+                result.add(rankItem);
             }
-        } else {
-            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-                long time20 = System.currentTimeMillis();
-                AdRankItem adRankItem = new AdRankItem();
-                adRankItem.setAdId(dto.getCreativeId());
-                adRankItem.setCreativeCode(dto.getCreativeCode());
-                adRankItem.setAdVerId(dto.getAdVerId());
-                adRankItem.setVideoId(request.getVideoId());
-                adRankItem.setCpa(dto.getCpa());
-                adRankItem.setId(dto.getAdId());
-                adRankItem.setCampaignId(dto.getCampaignId());
-                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                adRankItem.setRandom(random.nextInt(1000));
-
-                String cidStr = dto.getCreativeId().toString();
-                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
-                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-
-                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-
-                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-
-                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-
-                this.handleD1Feature(d1Feature, cidFeatureMap);
+        }
 
-                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+        return result;
+    }
 
-                String title = b1Feature.getOrDefault("cidtitle", "");
-                long time21 = System.currentTimeMillis();
-                this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
-                long time22 = System.currentTimeMillis();
-                this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
-                long time23 = System.currentTimeMillis();
-                log.info("cost={} other={} handleE1AndE2Feature={} handleD3AndB1Feature={}", time23 - time20,
-                        time21 - time20, time22 - time21, time23 - time22);
+    private String getUserLayer(RankRecommendRequestParam request) {
 
-                // adRankItem.setFeatureMap(cidFeatureMap);
 
-                adRankItems.add(adRankItem);
+        Map<String, String> userLayer = this.getUserLayer(request.getMid());
 
-            }
+        String layer = userLayer.getOrDefault("layer", "无曝光");
+        String clazz = userLayer.getOrDefault("class", "近期未出现");
+        if (request.getIsFilterUser()) {
+            return String.join("-", layer, "炸", clazz);
+        } else {
+            return String.join("-", layer, clazz);
         }
+    }
 
-        //
+    private List<HotRankFeatureInfo> multiGetFeature(List<String> customers, String keyFormat) {
+        List<HotRankFeatureInfo> hotRankFeatureInfos = new ArrayList<>(customers.size());
 
-        long time2 = System.currentTimeMillis();
-        // 分桶
-        this.readBucketFile();
-        userFeatureMap = this.featureBucket(userFeatureMap);
-        for (AdRankItem adRankItem : adRankItems) {
-            Map<String, String> featureMap = adRankItem.getFeatureMap();
-            adRankItem.setFeatureMap(this.featureBucket(featureMap));
+        List<String> keys = new ArrayList<>(customers.size());
+        for (String customer : customers) {
+            String key = String.format(keyFormat, customer);
+            keys.add(key);
         }
 
+        List<String> values = algRedisHelper.mget(keys);
 
-        // 打分排序
-        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_683)
-                .scoring(sceneFeatureMap, userFeatureMap, adRankItems);
-
-        long time3 = System.currentTimeMillis();
-        for (AdRankItem item : result) {
-            item.setScore(item.getLrScore() * item.getCpa());
-            item.getScoreMap().put("cpa", item.getCpa());
-            item.getScoreMap().put("cpm", item.getCpm());
-            item.getFeatureMap().putAll(userFeatureMap);
-            item.getFeatureMap().putAll(sceneFeatureMap);
-
-            // 没有转化回传的广告主，使用后台配置的CPM
-            if (noApiAdVerIds.contains(item.getAdVerId())) {
-                item.setScore(item.getCpm() / 1000);
-            }
-
-            for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(item.getAdVerId(), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
+        for (int i = 0; i < customers.size(); i++) {
+            String value = values.get(i);
+            if (StringUtils.isEmpty(value)) {
+                value = "{}";
             }
+            HotRankFeatureInfo hotRankFeatureInfo = JSON.parseObject(value, HotRankFeatureInfo.class);
+            hotRankFeatureInfo.setLabel(customers.get(i));
+            hotRankFeatureInfos.add(hotRankFeatureInfo);
         }
 
-        long time4 = System.currentTimeMillis();
-        log.info("cost={}, feature1={}, feature2={}, getScorerPipeline={}, loop={}",
-                time4 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3);
-
-        result.sort(ComparatorUtil.equalsRandomComparator());
-
-        return result;
+        return hotRankFeatureInfos;
     }
 
-    private void handleB1Feature(Map<String, String> b1Feature, Map<String, String> cidFeatureMap, String cid) {
-        cidFeatureMap.put("cid_" + cid, "0.1");
-        // if (StringUtils.isNotBlank(b1Feature.get("adid"))) {
-        //     String adId = b1Feature.get("adid");
-        //     cidFeatureMap.put("adid_" + adId, idDefaultValue);
-        // }
-        if (StringUtils.isNotBlank(b1Feature.get("adverid"))) {
-            String adVerId = b1Feature.get("adverid");
-            cidFeatureMap.put("adverid_" + adVerId, "0.1");
-        }
-        // if (StringUtils.isNotBlank(b1Feature.get("targeting_conversion"))) {
-        //     String targetingConversion = b1Feature.get("targeting_conversion");
-        //     cidFeatureMap.put("targeting_conversion_" + targetingConversion, idDefaultValue);
-        // }
-        if (StringUtils.isNotBlank(b1Feature.get("cpa"))) {
-            String cpa = b1Feature.get("cpa");
-            cidFeatureMap.put("cpa", cpa);
-        }
+    private void fullCustomerInfo(Map<String, Map<String, Map<String, String>>> cidFeature, AdRankItem rankItem) {
+        Map<String, Map<String, String>> feature = cidFeature.getOrDefault(String.valueOf(rankItem.getAdId()), new HashMap<>());
+        Map<String, String> basicInfo = feature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+        String customer = basicInfo.getOrDefault(CUSTOMER_FIELD_NAME, "");
+        rankItem.getExt().put(CUSTOMER_FIELD_NAME, customer);
     }
 
-    private void handleB2ToB5AndB8ToB9Feature(Map<String, Map<String, String>> c1Feature, Map<String, Map<String, String>> adVerFeature, Map<String, String> cidFeatureMap) {
-        Map<String, String> b2Feature = adVerFeature.getOrDefault("alg_cid_feature_adver_action", new HashMap<>());
-        Map<String, String> b3Feature = c1Feature.getOrDefault("alg_cid_feature_cid_action", new HashMap<>());
-        Map<String, String> b4Feature = c1Feature.getOrDefault("alg_cid_feature_region_action", new HashMap<>());
-        Map<String, String> b5Feature = c1Feature.getOrDefault("alg_cid_feature_app_action", new HashMap<>());
-        Map<String, String> b8Feature = c1Feature.getOrDefault("alg_cid_feature_brand_action", new HashMap<>());
-        Map<String, String> b9Feature = c1Feature.getOrDefault("alg_cid_feature_weChatVersion_action", new HashMap<>());
-
-        List<String> timeList = Arrays.asList("1h", "2h", "3h", "6h", "12h", "1d", "3d", "7d", "yesterday", "today");
-        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
-                new Tuple2<>(b2Feature, "b2"),
-                new Tuple2<>(b3Feature, "b3"),
-                new Tuple2<>(b4Feature, "b4"),
-                new Tuple2<>(b5Feature, "b5"),
-                new Tuple2<>(b8Feature, "b8"),
-                new Tuple2<>(b9Feature, "b9")
-        );
-        for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
-            Map<String, String> feature = tuple2.f1;
-            String prefix = tuple2.f2;
-            for (String time : timeList) {
-                double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
-                double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
-                double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
-                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
-                double f2 = NumUtil.div(conver, view);
-                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
-                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
-
-                cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
+    private String choose(List<HotRankFeatureInfo> hotRankFeatureInfos, int viewThreshold) {
+        // 按CPM排序
+        hotRankFeatureInfos.sort(Comparator.comparing(HotRankFeatureInfo::getCpm).reversed());
+
+        // 按照曝光拆分
+        List<HotRankFeatureInfo> highView = new ArrayList<>();
+        List<HotRankFeatureInfo> tailView = new ArrayList<>();
+        int highIdx = 1, tailIdx = 1;
+        for (HotRankFeatureInfo hotRankFeatureInfo : hotRankFeatureInfos) {
+            if (hotRankFeatureInfo.getView() >= viewThreshold) {
+                hotRankFeatureInfo.setRank(highIdx++);
+                highView.add(hotRankFeatureInfo);
+            } else {
+                hotRankFeatureInfo.setRank(tailIdx++);
+                tailView.add(hotRankFeatureInfo);
             }
         }
 
-    }
+        // 设置高曝光列表的初始权重
+        this.calcFirstItemWeight(highView, tailView, true);
+        // 计算高曝光列表中其他的权重
+        this.calcListOtherWeight(highView);
 
-    private void handleB6ToB7Feature(Map<String, Map<String, String>> c1Feature, Map<String, String> cidFeatureMap) {
-        Map<String, String> b6Feature = c1Feature.getOrDefault("alg_cid_feature_week_action", new HashMap<>());
-        Map<String, String> b7Feature = c1Feature.getOrDefault("alg_cid_feature_hour_action", new HashMap<>());
-
-        List<String> timeList = Arrays.asList("7d", "14d");
-        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
-                new Tuple2<>(b6Feature, "b6"),
-                new Tuple2<>(b7Feature, "b7")
-        );
-        for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
-            Map<String, String> feature = tuple2.f1;
-            String prefix = tuple2.f2;
-            for (String time : timeList) {
-                double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
-                double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
-                double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
-                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
-                double f2 = NumUtil.div(conver, view);
-                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
-                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
-
-                cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
-            }
-        }
-
-    }
+        // 设置低曝光列表的初始权重
+        this.calcFirstItemWeight(highView, tailView, false);
+        // 计算低曝光列表中其他的权重
+        this.calcListOtherWeight(tailView);
 
-    private List<TupleMapEntry<Tuple5>> handleC1Feature(Map<String, String> c1Feature, Map<String, String> featureMap) {
-
-        // 用户特征
-        List<TupleMapEntry<Tuple5>> midActionList = new ArrayList<>();
-        if (c1Feature.containsKey("action")) {
-            String action = c1Feature.get("action");
-            midActionList = Arrays.stream(action.split(","))
-                    .map(r -> {
-                        String[] rList = r.split(":");
-                        Tuple5 tuple5 = new Tuple5(rList[1], rList[2], rList[3], rList[4], rList[5]);
-                        return new TupleMapEntry<>(rList[0], tuple5);
-                    })
-                    // TODO 倒排
-                    .sorted((a, b) -> Integer.compare(Integer.parseInt(b.value.f1), Integer.parseInt(a.value.f1)))
-                    .collect(Collectors.toList());
+        List<WeightRandom.ItemWithWeight<String>> itemWithWeights = new ArrayList<>(hotRankFeatureInfos.size());
+        for (HotRankFeatureInfo hotRankFeatureInfo : hotRankFeatureInfos) {
+            WeightRandom.ItemWithWeight<String> itemWeight = new WeightRandom.ItemWithWeight<>(hotRankFeatureInfo.getLabel(), hotRankFeatureInfo.getWeight());
+            itemWithWeights.add(itemWeight);
         }
 
-        double viewAll = midActionList.size();
-        double clickAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f2)).sum();
-        double converAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f3)).sum();
-        double incomeAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f4)).sum();
-        featureMap.put("viewAll", String.valueOf(viewAll));
-        featureMap.put("clickAll", String.valueOf(clickAll));
-        featureMap.put("converAll", String.valueOf(converAll));
-        featureMap.put("incomeAll", String.valueOf(incomeAll));
-        featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
-        featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
-        featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
-        // featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
-
-        return midActionList;
+        return new WeightRandom<>(itemWithWeights).choose();
     }
 
-    private void handleC1UIFeature(Map<String, Double> midTimeDiffMap, Map<String, Double> midActionStatic, Map<String, String> featureMap, String cid) {
-        if (midTimeDiffMap.containsKey("timediff_view_" + cid)) {
-            featureMap.put("timediff_view", String.valueOf(midTimeDiffMap.getOrDefault("timediff_view_" + cid, 0.0)));
-        }
-        if (midTimeDiffMap.containsKey("timediff_click_" + cid)) {
-            featureMap.put("timediff_click", String.valueOf(midTimeDiffMap.getOrDefault("timediff_click_" + cid, 0.0)));
-        }
-        if (midTimeDiffMap.containsKey("timediff_conver_" + cid)) {
-            featureMap.put("timediff_conver", String.valueOf(midTimeDiffMap.getOrDefault("timediff_conver_" + cid, 0.0)));
-        }
-        if (midActionStatic.containsKey("actionstatic_view_" + cid)) {
-            featureMap.put("actionstatic_view", String.valueOf(midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)));
-        }
-        if (midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            featureMap.put("actionstatic_click", String.valueOf(midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)));
-        }
-        if (midActionStatic.containsKey("actionstatic_conver_" + cid)) {
-            featureMap.put("actionstatic_conver", String.valueOf(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0)));
-        }
-        if (midActionStatic.containsKey("actionstatic_income_" + cid)) {
-            featureMap.put("actionstatic_income", String.valueOf(midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0)));
-        }
-        if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            double ctr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
-            );
-            featureMap.put("actionstatic_ctr", String.valueOf(ctr));
-        }
-        if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_conver_" + cid)) {
-            double ctcvr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
-            );
-            featureMap.put("actionstatic_ctcvr", String.valueOf(ctcvr));
-        }
-        if (midActionStatic.containsKey("actionstatic_conver_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            double cvr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)
-            );
-            featureMap.put("actionstatic_cvr", String.valueOf(cvr));
-        }
-    }
-
-    private void handleD1Feature(Map<String, String> d1Feature, Map<String, String> featureMap) {
-        for (String prefix : Arrays.asList("3h", "6h", "12h", "1d", "3d", "7d")) {
-            double view = Double.parseDouble(d1Feature.getOrDefault("ad_view_" + prefix, "0"));
-            double click = Double.parseDouble(d1Feature.getOrDefault("ad_click_" + prefix, "0"));
-            double conver = Double.parseDouble(d1Feature.getOrDefault("ad_conversion_" + prefix, "0"));
-            double income = Double.parseDouble(d1Feature.getOrDefault("ad_income_" + prefix, "0"));
-            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(NumUtil.div(click, view)));
-            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
-            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
-            featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
-            // featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+    private void calcFirstItemWeight(List<HotRankFeatureInfo> high, List<HotRankFeatureInfo> tail, boolean isHigh) {
+        if (CollectionUtils.isEmpty(high) && CollectionUtils.isEmpty(tail)) {
+            return;
         }
-    }
 
-    private void handleD2Feature(Map<String, Map<String, Double>> vidRankMaps, Map<String, String> featureMap, String cid) {
-        if (MapUtils.isEmpty(vidRankMaps)) {
+        // 设置高曝光的初始权重
+        if (isHigh && CollectionUtils.isNotEmpty(high)) {
+            high.get(0).setWeight(Math.max(1, hotRankMaxWeight));
             return;
         }
 
-        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
-        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
-        List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
-
-        for (String prefix1 : prefixes1) {
-            for (String prefix2 : prefixes2) {
-                String combinedKey = prefix1 + "_" + prefix2;
-                if (vidRankMaps.containsKey(combinedKey)) {
-                    Double rank = vidRankMaps.get(combinedKey).getOrDefault(cid, 0.0);
-                    if (rank >= 1.0) {
-                        featureMap.put("vid_rank_" + combinedKey, String.valueOf(NumUtil.div(1, rank)));
-                    }
+        // 设置低曝光的初始权重，从高曝光列表中找到第一个CPM小于等于低曝光最高CPM的元素权重
+        if (!isHigh && CollectionUtils.isNotEmpty(tail)) {
+            HotRankFeatureInfo firstTail = tail.get(0);
+            double firstCpm = firstTail.getCpm();
+            for (HotRankFeatureInfo info : high) {
+                if (firstCpm >= info.getCpm()) {
+                    // 权重最小为1
+                    double weight = Math.max(1, info.getWeight() / hotRankCalcWeightCoefficient);
+                    firstTail.setWeight(weight);
+                    return;
                 }
             }
+            firstTail.setWeight(1);
         }
     }
 
-    private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap) {
-        if (MapUtils.isEmpty(d3Feature) || !d3Feature.containsKey("title") || StringUtils.isEmpty(cTitle)) {
-            return;
-        }
-        String vTitle = d3Feature.get("title");
-        double score = Similarity.conceptSimilarity(cTitle, vTitle);
-        featureMap.put("ctitle_vtitle_similarity", String.valueOf(score));
-    }
-
-    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title, Map<String, String> featureMap) {
-        if (StringUtils.isEmpty(title)) {
-            return;
-        }
-
-        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(
-                new Tuple2<>(e1Feature, "e1"),
-                new Tuple2<>(e2Feature, "e2")
-        );
-
-        List<String> tagsFieldList = Arrays.asList("tags_3d", "tags_7d", "tags_14d");
-        for (Tuple2<Map<String, String>, String> tuple2 : tuple2List) {
-            Map<String, String> feature = tuple2.f1;
-            String prefix = tuple2.f2;
-            if (MapUtils.isEmpty(feature)) {
+    /**
+     * 计算列表中第二个元素往后的权重
+     */
+    private void calcListOtherWeight(List<HotRankFeatureInfo> rankItem) {
+        for (int i = 1; i < rankItem.size(); i++) {
+            HotRankFeatureInfo curr = rankItem.get(i);
+            if (curr.getCpm() == 0) {
+                curr.setWeight(1);
                 continue;
             }
-
-            for (String tagsField : tagsFieldList) {
-                if (StringUtils.isNotEmpty(feature.get(tagsField))) {
-                    String tags = feature.get(tagsField);
-                    Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
-                    featureMap.put(prefix + "_" + tagsField + "_matchnum", String.valueOf(doubles[0]));
-                    featureMap.put(prefix + "_" + tagsField + "_maxscore", String.valueOf(doubles[1]));
-                    featureMap.put(prefix + "_" + tagsField + "_avgscore", String.valueOf(doubles[2]));
-                }
+            HotRankFeatureInfo prev = rankItem.get(i - 1);
+            double currWeight1 = prev.getWeight() * (1 - (1 - curr.getCpm() / prev.getCpm()) * hotRankCalcWeightCoefficient);
+            double currWeight2 = prev.getWeight() / hotRankCalcWeightCoefficient;
+            double currWeight = Math.min(currWeight1, currWeight2);
+            if (currWeight <= 1) {
+                currWeight = Math.max(currWeight1, currWeight2);
             }
+            curr.setWeight(Math.max(1, currWeight));
         }
     }
 
-    private Map<String, Double> parseC1FeatureListToTimeDiffMap(List<TupleMapEntry<Tuple5>> midActionList, long ts) {
-        Map<String, Double> midTimeDiffMap = new HashMap<>();
-        for (TupleMapEntry<Tuple5> entry : midActionList) {
-            String cid = entry.key;
-            double tsHistory = Double.parseDouble(entry.value.f1);
-            double click = Double.parseDouble(entry.value.f2);
-            double conver = Double.parseDouble(entry.value.f3);
-            double d = (ts - tsHistory) / 3600 / 24;
-            if (!midTimeDiffMap.containsKey("timediff_view_" + cid)) {
-                midTimeDiffMap.put("timediff_view_" + cid, NumUtil.div(1, d));
-            }
-            if (!midTimeDiffMap.containsKey("timediff_click_" + cid) && click > 0) {
-                midTimeDiffMap.put("timediff_click_" + cid, NumUtil.div(1, d));
-            }
-            if (!midTimeDiffMap.containsKey("timediff_conver_" + cid) && conver > 0) {
-                midTimeDiffMap.put("timediff_conver_" + cid, NumUtil.div(1, d));
-            }
-        }
-        return midTimeDiffMap;
-    }
+    private void fullMetaFeature(AdRankItem rankItem, String userLayer, Map<String, HotRankFeatureInfo> allCustomerFeatureMap, Map<Long, HotRankFeatureInfo> allCreativeFeatureMap, Feature feature) {
+        String customer = rankItem.getExt().getOrDefault(CUSTOMER_FIELD_NAME, "").toString();
 
-    private Map<String, Double> parseC1FeatureListToActionStaticMap(List<TupleMapEntry<Tuple5>> midActionList) {
-        Map<String, Double> midActionStaticsMap = new HashMap<>();
-        for (TupleMapEntry<Tuple5> entry : midActionList) {
-            String cid = entry.key;
-            double click = Double.parseDouble(entry.value.f2);
-            double conver = Double.parseDouble(entry.value.f3);
-            double income = Double.parseDouble(entry.value.f4);
+        Map<String, Double> scoreMap = new HashMap<>();
+        scoreMap.put("hotRankCustomerViewThreshold", hotRankCustomerViewThreshold.doubleValue());
+        scoreMap.put("hotRankCreativeViewThreshold", hotRankCreativeViewThreshold.doubleValue());
+        scoreMap.put("hotRankCalcWeightCoefficient", hotRankCalcWeightCoefficient);
+        scoreMap.put("hotRankMaxWeight", hotRankMaxWeight);
 
-            Double viewSum = midActionStaticsMap.getOrDefault("actionstatic_view_" + cid, 0.0);
-            midActionStaticsMap.put("actionstatic_view_" + cid, 1 + viewSum);
+        if (StringUtils.isNotBlank(customer) && allCustomerFeatureMap.containsKey(customer)) {
+            HotRankFeatureInfo customerFeatureInfo = allCustomerFeatureMap.get(customer);
+            rankItem.getMetaFeatureMap().put("customerFeature", customerFeatureInfo.covertToFeatureMap());
 
-            Double clickSum = midActionStaticsMap.getOrDefault("actionstatic_click_" + cid, 0.0);
-            midActionStaticsMap.put("actionstatic_click_" + cid, clickSum + click);
+            scoreMap.put("customer_weight", customerFeatureInfo.getWeight());
+            scoreMap.put("customer_cpm", customerFeatureInfo.getCpm());
+            scoreMap.put("customer_rank", (double) customerFeatureInfo.getRank());
+            scoreMap.put("customer_view", (double) customerFeatureInfo.getView());
+        }
 
-            Double converSum = midActionStaticsMap.getOrDefault("actionstatic_conver_" + cid, 0.0);
-            midActionStaticsMap.put("actionstatic_conver_" + cid, converSum + conver);
+        if (allCreativeFeatureMap.containsKey(rankItem.getAdId())) {
+            HotRankFeatureInfo creativeFeatureInfo = allCreativeFeatureMap.get(rankItem.getAdId());
+            rankItem.getMetaFeatureMap().put("creativeFeature", creativeFeatureInfo.covertToFeatureMap());
 
-            Double incomSum = midActionStaticsMap.getOrDefault("actionstatic_income_" + cid, 0.0);
-            midActionStaticsMap.put("actionstatic_income_" + cid, incomSum + income);
+            scoreMap.put("weight", creativeFeatureInfo.getWeight());
+            scoreMap.put("cpm", creativeFeatureInfo.getCpm());
+            scoreMap.put("rank", (double) creativeFeatureInfo.getRank());
+            scoreMap.put("view", (double) creativeFeatureInfo.getView());
         }
 
-        return midActionStaticsMap;
-    }
+        Map<String, String> midFeature = new HashMap<>(4);
+        midFeature.put("userLayer", userLayer);
+        rankItem.getMetaFeatureMap().put("midFeature", midFeature);
 
-    private Map<String, Map<String, Double>> parseD2FeatureMap(Map<String, String> d2Feature) {
-        Map<String, Map<String, Double>> vidRankMaps = new HashMap<>();
-        for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
-            String key = entry.getKey();
-            String value = entry.getValue();
-            Map<String, Double> valueMap = Arrays.stream(value.split(","))
-                    .map(r -> r.split(":"))
-                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
-            vidRankMaps.put(key, valueMap);
-        }
-        return vidRankMaps;
+        rankItem.getScoreMap().putAll(scoreMap);
     }
 
-    public Map<String, String> handleSceneFeature(long ts) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        sceneFeatureMap.put("hour_" + DateUtils.getHourByTimestamp(ts), "0.1");
-        sceneFeatureMap.put("dayofweek_" + DateUtils.getDayOrWeekByTimestamp(ts), "0.1");
-        return sceneFeatureMap;
-    }
+    @Data
+    @NoArgsConstructor
+    @AllArgsConstructor
+    private static class HotRankFeatureInfo {
+        private String label;
 
-    private void readBucketFile() {
-        if (MapUtils.isNotEmpty(bucketsMap)) {
-            return;
-        }
-        synchronized (this) {
-            InputStream resourceStream = RankStrategyBy687.class.getClassLoader().getResourceAsStream("20240718_ad_bucket_688.txt");
-            if (resourceStream != null) {
-                try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
-                    Map<String, double[]> bucketsMap = new HashMap<>();
-                    Map<String, Double> bucketsLen = new HashMap<>();
-                    String line;
-                    while ((line = reader.readLine()) != null) {
-                        // 替换空格和换行符，过滤空行
-                        line = line.replace(" ", "").replaceAll("\n", "");
-                        if (!line.isEmpty()) {
-                            String[] rList = line.split("\t");
-                            if (rList.length == 3) {
-                                String key = rList[0];
-                                double value1 = Double.parseDouble(rList[1]);
-                                bucketsLen.put(key, value1);
-                                double[] value2 = Arrays.stream(rList[2].split(","))
-                                        .mapToDouble(Double::valueOf)
-                                        .toArray();
-                                bucketsMap.put(key, value2);
-                            }
-                        }
-                    }
-                    this.bucketsMap = bucketsMap;
-                    this.bucketsLen = bucketsLen;
-                } catch (IOException e) {
-                    log.error("something is wrong in parse bucket file:", e);
-                }
-            } else {
-                log.error("no bucket file");
-            }
-        }
-    }
+        private double cpm = 0;
 
-    private Map<String, String> featureBucket(Map<String, String> featureMap) {
-        Map<String, String> newFeatureMap = new ConcurrentHashMap<>(featureMap.size());
-        for (Map.Entry<String, String> entry : featureMap.entrySet()) {
-            String name = entry.getKey();
-            double score = Double.parseDouble(entry.getValue());
-            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
-            if (score > 1E-8) {
-                if (this.bucketsMap.containsKey(name) && this.bucketsLen.containsKey(name)) {
-                    double[] buckets = this.bucketsMap.get(name);
-                    double bucketNum = this.bucketsLen.get(name);
-                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
-                    newFeatureMap.put(name, String.valueOf(scoreNew));
-                } else {
-                    newFeatureMap.put(name, String.valueOf(score));
-                }
-            }
-        }
+        private int view = 0;
 
-        return newFeatureMap;
+        private double weight = 1;
+
+        private int rank;
+
+        public Map<String, String> covertToFeatureMap() {
+            Map<String, String> featureMap = new HashMap<>(4);
+            featureMap.put("label", label);
+            featureMap.put("rank", String.valueOf(this.getRank()));
+            featureMap.put("cpm", String.valueOf(this.getCpm()));
+            featureMap.put("view", String.valueOf(this.getView()));
+            featureMap.put("weight", String.valueOf(this.getWeight()));
+            return featureMap;
+        }
     }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -1,20 +1,27 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
+import com.tzld.piaoquan.ad.engine.commons.helper.DnnCidDataHelper;
+import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.CorrectCpaParam;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
-import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
-import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
+import javax.annotation.PostConstruct;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -26,11 +33,12 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static com.tzld.piaoquan.ad.engine.commons.math.Const.*;
+
 @Slf4j
 @Component
 public class RankStrategyBy688 extends RankStrategyBasic {
 
-
     private Map<String, double[]> bucketsMap = new HashMap<>();
 
     private Map<String, Double> bucketsLen = new HashMap<>();
@@ -38,9 +46,37 @@ public class RankStrategyBy688 extends RankStrategyBasic {
     @Value("${word2vec.exp:694}")
     private String word2vecExp;
 
+
+    // FIXME(zhoutian): 可能需要独立配置
+    @ApolloJsonValue("${rank.score.weight.680:{}}")
+    private Map<String, Double> weightMap;
+
+    /**
+     * 人群分层&创意的权重
+     * 格式：{layer_creativeId: weight}
+     */
+    @ApolloJsonValue("${rank.score.weight.layer.and.creative:{}}")
+    private Map<String, Double> layerAndCreativeWeightMap;
+
+    @ApolloJsonValue("${rank.score.neg_sample_rate:0.01}")
+    Double negSampleRate;
+
+    Set<String> sparseFeatureSet;
+
+
+    @PostConstruct
+    public void afterInit() {
+        this.readBucketFile();
+        this.initSparseFeatureNames();
+    }
+
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+        Map<String, Double> weightParam = ObjUtil.nullOrDefault(weightMap, new HashMap<>());
+
 
+        Map<Long, Double> creativeScoreCoefficient = getCreativeScoreCoefficient();
         Set<String> noApiAdVerIds = getNoApiAdVerIds();
 
         long ts = System.currentTimeMillis() / 1000;
@@ -51,14 +87,19 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         }
 
         long start = System.currentTimeMillis();
+        //过滤创意
+        filterRequestAdList(request, scoreParam);
         // 特征处理
         // feature1
         Feature feature = this.getFeature(scoreParam, request);
 
+
         Map<String, Map<String, String>> userFeature = feature.getUserFeature();
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
+        Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
 
         Map<String, String> userFeatureMap = new HashMap<>();
         Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
@@ -75,9 +116,43 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
         Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
 
+        Map<String, String> g1Feature = userFeature.getOrDefault("mid_return_video_cate", new HashMap<>());
+        Map<String, String> g2Feature = userFeature.getOrDefault("mid_share_video_cate", new HashMap<>());
+
+
+        userFeatureMap.put("brand", reqFeature.getOrDefault("brand", ""));
+        userFeatureMap.put("region", reqFeature.getOrDefault("region", ""));
+        userFeatureMap.put("city", reqFeature.getOrDefault("city", ""));
+        userFeatureMap.put("vid", reqFeature.getOrDefault("vid", ""));
+        userFeatureMap.put("apptype", reqFeature.getOrDefault("apptype", ""));
+        userFeatureMap.put("is_first_layer", reqFeature.getOrDefault("is_first_layer", ""));
+        userFeatureMap.put("root_source_scene", reqFeature.getOrDefault("root_source_scene", ""));
+        userFeatureMap.put("root_source_channel", reqFeature.getOrDefault("root_source_channel", ""));
+
+
+        userFeatureMap.put("cate1", d3Feature.get("merge_first_level_cate"));
+        userFeatureMap.put("cate2", d3Feature.get("merge_second_level_cate"));
+        userFeatureMap.put("user_vid_return_tags_2h", e1Feature.getOrDefault("tags_2h", null));
+        userFeatureMap.put("user_vid_return_tags_1d", e1Feature.getOrDefault("tags_1d", null));
+        userFeatureMap.put("user_vid_return_tags_3d", e1Feature.getOrDefault("tags_3d", null));
+        userFeatureMap.put("user_vid_return_tags_7d", e1Feature.getOrDefault("tags_7d", null));
+        userFeatureMap.put("user_vid_return_tags_14d", e1Feature.getOrDefault("tags_14d", null));
+        userFeatureMap.put("title_split", d3Feature.getOrDefault("title_split", null));
+        userFeatureMap.put("user_vid_share_tags_1d", e2Feature.getOrDefault("tags_1d", null));
+        userFeatureMap.put("user_vid_share_tags_14d", e2Feature.getOrDefault("tags_14d", null));
+        userFeatureMap.put("user_vid_return_cate1_14d", g1Feature.getOrDefault("cate1_14d", null));
+        userFeatureMap.put("user_vid_return_cate2_14d", g1Feature.getOrDefault("cate2_14d", null));
+        userFeatureMap.put("user_vid_share_cate1_14d", g2Feature.getOrDefault("cate1_14d", null));
+        userFeatureMap.put("user_vid_share_cate2_14d", g2Feature.getOrDefault("cate2_14d", null));
+
+        userFeatureMap.put("user_layer_class", reqFeature.getOrDefault("user_layer_class", null));
+
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
+        boolean isGuaranteedFlow = getIsGuaranteedFlow(scoreParam);
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, isGuaranteedFlow);
+        Map<Long, CorrectCpaParam> correctCpaMap = getCorrectCpaParamMap(request, scoreParam, reqFeature);
         List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
@@ -94,6 +169,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     adRankItem.setId(dto.getAdId());
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setSkuId(dto.getSkuId());
+                    adRankItem.setCustomerId(dto.getCustomerId());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -101,14 +178,16 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                         adRankItem.getExt().put("isApi", "1");
                     }
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-
+                    adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
+                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow, reqFeature);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
                     Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
 
                     Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
+                    Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(dto.getSkuId()), new HashMap<>());
                     Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
 
                     this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
@@ -117,6 +196,17 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
                     this.handleD1Feature(d1Feature, cidFeatureMap);
                     this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                    this.handleH1AndH2Feature(skuFeature, adVerFeature, cidFeatureMap);
+                    cidFeatureMap.put("cid", dto.getCreativeId() != null ? String.valueOf(dto.getCreativeId()) : "");
+                    cidFeatureMap.put("adid", dto.getAdId() != null ? String.valueOf(dto.getAdId()) : "");
+                    cidFeatureMap.put("adverid", dto.getAdVerId() != null ? dto.getAdVerId() : "");
+                    cidFeatureMap.put("profession", dto.getProfession() != null ? dto.getProfession() : "");
+                    //DNN模型没训练过的cid才不传入广告相关的稀疏特征
+                    if (CollectionUtils.isNotEmpty(DnnCidDataHelper.getCidSet()) && !DnnCidDataHelper.getCidSet().contains(adRankItem.getAdId())) {
+                        cidFeatureMap.put("cid", "");
+                        cidFeatureMap.put("adid", "");
+                        cidFeatureMap.put("adverid", "");
+                    }
                     return adRankItem;
                 } finally {
                     cdl1.countDown();
@@ -162,17 +252,14 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                 }
             });
         }
-        long time31 = System.currentTimeMillis();
         try {
             cdl2.await(150, TimeUnit.MILLISECONDS);
         } catch (Exception e) {
             log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
         }
 
-        // feature4
         long time3 = System.currentTimeMillis();
         // 分桶
-        this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
         CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
         for (AdRankItem adRankItem : adRankItems) {
@@ -193,58 +280,117 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         long time4 = System.currentTimeMillis();
         // 打分排序
         // getScorerPipeline
-        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20240909).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
+        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.PAI_SCORE_CONF_20250214).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
         long time5 = System.currentTimeMillis();
+        // calibrate score for negative sampling or cold start
+        for (AdRankItem item : result) {
+            double originalScore = item.getLrScore();
+            double calibratedScore = originalScore / (originalScore + (1 - originalScore) / negSampleRate);
+            // 该创意尚未在模型中训练，打分不可靠
+            if (CollectionUtils.isNotEmpty(DnnCidDataHelper.getCidSet()) && !DnnCidDataHelper.getCidSet().contains(item.getAdId())) {
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
+                Map<String, String> b3Feature = cidFeature.getOrDefault("alg_cid_feature_cid_action", new HashMap<>());
+                double view = Double.parseDouble(b3Feature.getOrDefault("ad_view_14d", "0"));
+                double conver = Double.parseDouble(b3Feature.getOrDefault("ad_conversion_14d", "0"));
+                double smoothCxr = NumUtil.divSmoothV1(conver, view, 1.64);
+                //模型打分和统计计算取打分更低的
+                calibratedScore = Math.min(smoothCxr, calibratedScore);
+            }
+            item.setLrScore(calibratedScore);
+            item.getScoreMap().put("originCtcvrScore", originalScore);
+            item.getScoreMap().put("modelCtcvrScore", calibratedScore);
+            item.getScoreMap().put("ctcvrScore", calibratedScore);
+        }
+
+        calculateCtcvrScore(result, request, scoreParam, "dnn", reqFeature);
         // loop
+        double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
+        boolean isGuaranteeType = false;
+        // 查询人群分层信息
+        String peopleLayer = Optional.of(reqFeature)
+                .map(f -> f.get("layer"))
+                .map(s -> s.replace("-炸", ""))
+                .orElse(null);
         for (AdRankItem item : result) {
-            item.setScore(item.getLrScore() * item.getCpa());
+            double bid = item.getCpa();
+            if (scoreParam.getExpCodeSet().contains(correctCpaExp1) || scoreParam.getExpCodeSet().contains(correctCpaExp2)) {
+                Double correctionFactor = (Double) item.getExt().get("correctionFactor");
+                item.getScoreMap().put("correctionFactor", correctionFactor);
+                bid = bid * correctionFactor;
+            }
+            item.getScoreMap().put("ecpm", item.getLrScore() * bid * 1000);
+            if (isGuaranteedFlow && item.getExt().get("isGuaranteed") != null && (boolean) item.getExt().get("isGuaranteed")) {
+                isGuaranteeType = true;
+            }
+
+            String layerAndCreativeWeightMapKey = getLayerAndCreativeWeightMapKey(peopleLayer, String.valueOf(item.getAdId()));
+            // 人群分层&创意的权重
+            double layerAndCreativeWeight = getLayerAndCreativeWeight(layerAndCreativeWeightMapKey);
+            double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(isGuaranteedFlow, item.getExt());
+            double score = item.getLrScore() * bid * scoreCoefficient * guaranteeScoreCoefficient * layerAndCreativeWeight;
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
+            item.getScoreMap().put("bid", bid);
+            item.getScoreMap().put("cpmCoefficient", cpmCoefficient);
+            item.getScoreMap().put("scoreCoefficient", scoreCoefficient);
             item.getFeatureMap().putAll(userFeatureMap);
             item.getFeatureMap().putAll(sceneFeatureMap);
 
             // 没有转化回传的广告主，使用后台配置的CPM
             if (noApiAdVerIds.contains(item.getAdVerId())) {
-                item.setScore(item.getCpm() / 1000);
+                score = item.getCpm() * cpmCoefficient / 1000;
             }
+            item.setScore(score);
+        }
 
-            for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
 
-            for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
-
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(item.getAdVerId(), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
+        result.sort(ComparatorUtil.equalsRandomComparator());
 
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
-            for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
-                if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
-                }
-            }
+        if (CollectionUtils.isNotEmpty(result)) {
+            AdRankItem top1Item = result.get(0);
+            top1Item.getExt().put("isGuaranteeType", isGuaranteeType);
+            putMetaFeature(top1Item, feature, reqFeature, sceneFeatureMap, request);
+            top1Item.getExt().put("model", "dnn");
         }
+        long time6 = System.currentTimeMillis();
+        log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +
+                        "other={}, adIdSize={}, adRankItemsSize={}",
+                time6 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3,
+                time5 - time4, time6 - time5, request.getAdIdList().size(), adRankItems.size());
 
-        log.info("cost={}, feature1={}, feature2={}, feature31={}, feature32={}, feature4={}, getScorerPipeline={}, " +
-                        "adIdSize={}, adRankItemsSize={}",
-                time5 - start, time1 - start, time2 - time1, time31 - time2, time3 - time31, time4 - time3,
-                time5 - time4, request.getAdIdList().size(), adRankItems.size());
+        return result;
+    }
 
-        result.sort(ComparatorUtil.equalsRandomComparator());
+    /**
+     * 获取人群分层和创意的权重
+     *
+     * @param key
+     * @return
+     */
+    private Double getLayerAndCreativeWeight(String key) {
+        if (StringUtils.isBlank(key)) {
+            return 1d;
+        }
+        return layerAndCreativeWeightMap.getOrDefault(key, 1d);
+    }
 
-        return result;
+    /**
+     * 获取人群分层和创意的权重key
+     *
+     * @param layer
+     * @param creativeId
+     * @return
+     */
+    private String getLayerAndCreativeWeightMapKey(String layer, String creativeId) {
+        if (StringUtils.isBlank(layer) || StringUtils.isBlank(creativeId)) {
+            return null;
+        }
+        return layer + "_" + creativeId;
     }
 
+
     private void handleB1Feature(Map<String, String> b1Feature, Map<String, String> cidFeatureMap, String cid) {
         cidFeatureMap.put("cid_" + cid, "0.1");
         // if (StringUtils.isNotBlank(b1Feature.get("adid"))) {
@@ -289,15 +435,20 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
-                double f2 = NumUtil.div(conver, view);
-                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
-                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                double cpc = NumUtil.div(income, click);
+                double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+                double ctcvr = NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR);
+                double ecpm = ctr * cpc * 1000;
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(ctr));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(ctcvr));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * ctcvr));
             }
         }
 
@@ -320,16 +471,19 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
                 double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
-                double f2 = NumUtil.div(conver, view);
-                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
-                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                double cpc = NumUtil.div(income, click);
+                double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+                double ctcvr = NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR);
+                double ecpm = ctr * cpc * 1000;
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(ctr));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(ctcvr));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(ecpm));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
-                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * ctcvr));
             }
         }
 
@@ -337,6 +491,15 @@ public class RankStrategyBy688 extends RankStrategyBasic {
 
     private List<TupleMapEntry<Tuple5>> handleC1Feature(Map<String, String> c1Feature, Map<String, String> featureMap) {
 
+        //用户近1年内是否有转化
+        if (c1Feature.containsKey("user_has_conver_1y") && c1Feature.get("user_has_conver_1y") != null) {
+            featureMap.put("user_has_conver_1y", c1Feature.get("user_has_conver_1y"));
+        }
+        //用户历史转化过品类
+        if (c1Feature.containsKey("user_conver_ad_class") && c1Feature.get("user_conver_ad_class") != null) {
+            featureMap.put("user_conver_ad_class", c1Feature.get("user_conver_ad_class"));
+        }
+
         // 用户特征
         List<TupleMapEntry<Tuple5>> midActionList = new ArrayList<>();
         if (c1Feature.containsKey("action")) {
@@ -363,8 +526,24 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
         featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
         featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
-        // featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
-
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
+        if (CollectionUtils.isNotEmpty(midActionList)) {
+            List<String> cidClickList = new ArrayList<>();
+            List<String> cidConverList = new ArrayList<>();
+            for (TupleMapEntry<Tuple5> tupleMapEntry : midActionList) {
+                String cid = tupleMapEntry.key;
+                String click = tupleMapEntry.value.f2;
+                String conver = tupleMapEntry.value.f3;
+                if (Objects.equals(click, "1")) {
+                    cidClickList.add(cid);
+                }
+                if (Objects.equals(conver, "1")) {
+                    cidConverList.add(cid);
+                }
+            }
+            featureMap.put("user_cid_click_list", String.join(",", cidClickList));
+            featureMap.put("user_cid_conver_list", String.join(",", cidConverList));
+        }
         return midActionList;
     }
 
@@ -413,11 +592,13 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             double click = Double.parseDouble(d1Feature.getOrDefault("ad_click_" + prefix, "0"));
             double conver = Double.parseDouble(d1Feature.getOrDefault("ad_conversion_" + prefix, "0"));
             double income = Double.parseDouble(d1Feature.getOrDefault("ad_income_" + prefix, "0"));
-            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(NumUtil.div(click, view)));
-            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
-            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+            double cpc = NumUtil.div(income, click);
+            double ctr = NumUtil.divSmoothV2(click, view, CTR_SMOOTH_BETA_FACTOR);
+            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(ctr));
+            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.divSmoothV2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)));
+            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.divSmoothV2(conver, click, CVR_SMOOTH_BETA_FACTOR)));
             featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
-            // featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(ctr * cpc * 1000));
         }
     }
 
@@ -426,8 +607,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             return;
         }
 
-        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
-        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        // List<String> prefixes1 = Arrays.asList("ctr", "ctcvr");
         List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
 
         for (String prefix1 : prefixes1) {
@@ -443,6 +624,33 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         }
     }
 
+    private void handleH1AndH2Feature(Map<String, Map<String, String>> skuFeature,
+                                      Map<String, Map<String, String>> adVerFeature,
+                                      Map<String, String> cidFeatureMap) {
+        Map<String, String> h1Feature = adVerFeature.getOrDefault("alg_mid_feature_adver_action", new HashMap<>());
+        Map<String, String> h2Feature = skuFeature.getOrDefault("alg_mid_feature_sku_action", new HashMap<>());
+        List<String> timeList = Arrays.asList("3d", "7d", "30d");
+        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
+                new Tuple2<>(h1Feature, "adverid"),
+                new Tuple2<>(h2Feature, "skuid")
+        );
+        for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
+            Map<String, String> feature = tuple2.f1;
+            String prefix = tuple2.f2;
+            for (String time : timeList) {
+                String timeValue = feature.getOrDefault(time, "");
+                if (StringUtils.isNotEmpty(timeValue)) {
+                    String[] split = timeValue.split(",");
+                    cidFeatureMap.put("user" + "_" + prefix + "_" + "view" + "_" + time, split[0]);
+                    cidFeatureMap.put("user" + "_" + prefix + "_" + "click" + "_" + time, split[1]);
+                    cidFeatureMap.put("user" + "_" + prefix + "_" + "conver" + "_" + time, split[2]);
+                }
+            }
+        }
+
+
+    }
+
     private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap,
                                       ScoreParam scoreParam) {
         if (MapUtils.isEmpty(d3Feature) || !d3Feature.containsKey("title") || StringUtils.isEmpty(cTitle)) {
@@ -477,7 +685,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             for (String tagsField : tagsFieldList) {
                 if (StringUtils.isNotEmpty(feature.get(tagsField))) {
                     String tags = feature.get(tagsField);
-                    //Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    // Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
                     Double[] doubles;
                     if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
                         doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
@@ -548,19 +756,13 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         return vidRankMaps;
     }
 
-    public Map<String, String> handleSceneFeature(long ts) {
-        Map<String, String> sceneFeatureMap = new HashMap<>();
-        sceneFeatureMap.put("hour_" + DateUtils.getHourByTimestamp(ts), "0.1");
-        sceneFeatureMap.put("dayofweek_" + DateUtils.getDayOrWeekByTimestamp(ts), "0.1");
-        return sceneFeatureMap;
-    }
-
     private void readBucketFile() {
         if (MapUtils.isNotEmpty(bucketsMap)) {
             return;
         }
         synchronized (this) {
-            InputStream resourceStream = RankStrategyBy688.class.getClassLoader().getResourceAsStream("20240718_ad_bucket_688.txt");
+            String bucketFile = "20250217_ad_bucket_688.txt";
+            InputStream resourceStream = this.getClass().getClassLoader().getResourceAsStream(bucketFile);
             if (resourceStream != null) {
                 try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
                     Map<String, double[]> bucketsMap = new HashMap<>();
@@ -583,18 +785,81 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     this.bucketsMap = bucketsMap;
                     this.bucketsLen = bucketsLen;
                 } catch (IOException e) {
-                    log.error("something is wrong in parse bucket file:", e);
+                    log.error("something is wrong in parse bucket file: ", e);
                 }
+                log.info("load bucket file success: {}", bucketFile);
             } else {
                 log.error("no bucket file");
             }
         }
     }
 
+    private void initSparseFeatureNames() {
+        this.sparseFeatureSet = new HashSet<String>() {{
+            add("brand");
+            add("region");
+            add("city");
+            add("vid");
+            add("cate1");
+            add("cate2");
+            add("cid");
+            add("adid");
+            add("adverid");
+            add("user_cid_click_list");
+            add("user_cid_conver_list");
+            add("user_vid_return_tags_2h");
+            add("user_vid_return_tags_1d");
+            add("user_vid_return_tags_3d");
+            add("user_vid_return_tags_7d");
+            add("user_vid_return_tags_14d");
+            add("apptype");
+            add("hour");
+            add("hour_quarter");
+            add("root_source_scene");
+            add("root_source_channel");
+            add("is_first_layer");
+            add("title_split");
+            add("profession");
+            add("user_vid_share_tags_1d");
+            add("user_vid_share_tags_14d");
+            add("user_vid_return_cate1_14d");
+            add("user_vid_return_cate2_14d");
+            add("user_vid_share_cate1_14d");
+            add("user_vid_share_cate2_14d");
+            add("user_has_conver_1y");
+            add("user_conver_ad_class");
+            add("user_adverid_view_3d");
+            add("user_adverid_click_3d");
+            add("user_adverid_conver_3d");
+            add("user_adverid_view_7d");
+            add("user_adverid_click_7d");
+            add("user_adverid_conver_7d");
+            add("user_adverid_view_30d");
+            add("user_adverid_click_30d");
+            add("user_adverid_conver_30d");
+            add("user_skuid_view_3d");
+            add("user_skuid_click_3d");
+            add("user_skuid_conver_3d");
+            add("user_skuid_view_7d");
+            add("user_skuid_click_7d");
+            add("user_skuid_conver_7d");
+            add("user_skuid_view_30d");
+            add("user_skuid_click_30d");
+            add("user_skuid_conver_30d");
+            add("user_layer_class");
+        }};
+    }
+
     private Map<String, String> featureBucket(Map<String, String> featureMap) {
         Map<String, String> newFeatureMap = new ConcurrentHashMap<>(featureMap.size());
         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
             String name = entry.getKey();
+            if (this.sparseFeatureSet.contains(name)) {
+                if (entry.getValue() != null) {
+                    newFeatureMap.put(name, entry.getValue());
+                }
+                continue;
+            }
             double score = Double.parseDouble(entry.getValue());
             // 注意：0值、不在分桶文件中的特征，会被过滤掉。
             if (score > 1E-8) {
@@ -608,8 +873,6 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                 }
             }
         }
-
         return newFeatureMap;
     }
-
 }

pom.xml

@@ -322,6 +322,16 @@
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-starter-openfeign</artifactId>
         </dependency>
+
+        <!-- https://mvnrepository.com/artifact/com.aliyun.openservices.eas/eas-sdk -->
+        <dependency>
+            <groupId>com.aliyun.openservices.eas</groupId>
+            <artifactId>eas-sdk</artifactId>
+            <version>2.0.23</version>
+        </dependency>
+
+
+
         <!--easyexcel-->
     </dependencies>