Merge branch 'feature_20240621_zhaohaipeng_fm' of algorithm/ad-engine into master

.gitignore

@@ -33,4 +33,6 @@ build/
 .vscode/
 apollo-cache-dir
 sentinel
-weblog
+weblog
+logs
+LOG_PATH_IS_UNDEFINED

ad-engine-commons/pom.xml

@@ -1,7 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project
+        xmlns="http://maven.apache.org/POM/4.0.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>ad-engine</artifactId>
         <groupId>com.tzld.piaoquan</groupId>
@@ -25,7 +26,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.1.30</version>
+            <version>1.1.21</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
@@ -57,17 +58,37 @@
         </dependency>
 
 
-
-        <!--        <dependency>-->
-<!--            <groupId>net.devh</groupId>-->
-<!--            <artifactId>grpc-client-spring-boot-starter</artifactId>-->
-<!--            <version>2.9.0.RELEASE</version>-->
-<!--        </dependency>-->
-<!--        <dependency>-->
-<!--            <groupId>org.projectlombok</groupId>-->
-<!--            <artifactId>lombok</artifactId>-->
-<!--            <version>1.18.22</version>-->
-<!--        </dependency>-->
+        <dependency>
+            <groupId>org.xm</groupId>
+            <artifactId>similarity</artifactId>
+            <version>1.1</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-classic</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-access</artifactId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.collections</groupId>
+                    <artifactId>google-collections</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
 </project>

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/helper/NewExpHelper.java

@@ -0,0 +1,106 @@
+package com.tzld.piaoquan.ad.engine.commons.helper;
+
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import lombok.Data;
+import org.apache.commons.collections4.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+@Component
+public class NewExpHelper {
+    private final static Logger log = LoggerFactory.getLogger(NewExpHelper.class);
+    public static Map<String, Set<String>> appExpIdCache = new HashMap<>();
+    public static Map<String, Exp> expIdAndRangeCache = new HashMap<>();
+
+    public static String flagId;
+
+    @Value("${ad.new.exp.flag.id:0}")
+    private void setFlagId(String flagId) {
+        NewExpHelper.flagId = flagId;
+    }
+
+    @Value("${new.exp,config.v2:[]}")
+    private void setNewExpInfo(String str) {
+        List<ExpConfig> expConfigs = JSONUtils.fromJson(str, new TypeToken<List<ExpConfig>>() {
+        }, Collections.emptyList());
+        for (ExpConfig expConfig : expConfigs) {
+            String appId = expConfig.getAppType();
+            for (Layer layer : expConfig.layers) {
+                if (layer.layerId.equals("ad-server")) {
+                    Set<String> expIdSet = new HashSet<>();
+                    for (Exp exp : layer.getExps()) {
+                        expIdSet.add(exp.getExpId());
+                        expIdAndRangeCache.put(exp.getExpId(), exp);
+                    }
+                    appExpIdCache.put(appId, expIdSet);
+                }
+            }
+        }
+    }
+
+    public static boolean checkInNewExpGroup(String appId, int groupNumber, String expId) {
+        try {
+            if (appExpIdCache.get(appId) == null || !appExpIdCache.get(appId).contains(expId)) {
+                return false;
+            }
+            if (expIdAndRangeCache.get(expId).isAllEnter() && groupNumber < 0) {
+                return true;
+            }
+            return expIdAndRangeCache.get(expId).getRange()[0] <= groupNumber && expIdAndRangeCache.get(expId).getRange()[1] >= groupNumber;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    public static boolean checkInNewExpGroup(String appId, int groupNumber, Collection<String> expIds) {
+        if (CollectionUtils.isEmpty(expIds)) {
+            return false;
+        }
+        for (String expId : expIds) {
+            try {
+                if (appExpIdCache.get(appId) == null || !appExpIdCache.get(appId).contains(expId)) {
+                    continue;
+                }
+                if (expIdAndRangeCache.get(expId).isAllEnter() && groupNumber < 0) {
+                    return true;
+                }
+                if (expIdAndRangeCache.get(expId).getRange()[0] <= groupNumber && expIdAndRangeCache.get(expId).getRange()[1] >= groupNumber) {
+                    return true;
+                }
+            } catch (Exception e) {
+                log.warn("svc=checkInNewExpGroup appId={} groupNumber={} expId={}", appId, groupNumber, expId);
+            }
+        }
+        return false;
+    }
+
+    @Data
+    public static class ExpConfig {
+        private String appType;
+        private List<Layer> layers;
+
+    }
+
+    @Data
+    public static class Layer {
+        private String layerId;
+        private int bucketNum;
+        private List<Exp> exps;
+        private Map<String, String> groupRule;
+
+    }
+
+    @Data
+    public static class Exp {
+        private String expId;
+        private int[] range;
+        private boolean allEnter = false;
+        private Map<String, Object> param;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/AbstractScorer.java

@@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 
 public abstract class AbstractScorer {
@@ -61,7 +62,12 @@ public abstract class AbstractScorer {
     }
 
     public abstract List<AdRankItem> scoring(final ScoreParam param,
-                                           final UserAdFeature userAdFeature,
-                                           final List<AdRankItem> rankItems);
+                                             final UserAdFeature userAdFeature,
+                                             final List<AdRankItem> rankItems);
 
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/BaseFMModelScorer.java

@@ -0,0 +1,20 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.model.FMModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseFMModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseFMModelScorer.class);
+
+    public BaseFMModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(FMModel.class);
+    }
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/BaseLRV2ModelScorer.java

@@ -0,0 +1,21 @@
+package com.tzld.piaoquan.ad.engine.commons.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.model.LRV2Model;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class BaseLRV2ModelScorer extends AbstractScorer {
+
+    private static Logger LOGGER = LoggerFactory.getLogger(BaseLRV2ModelScorer.class);
+
+    public BaseLRV2ModelScorer(ScorerConfigInfo scorerConfigInfo) {
+        super(scorerConfigInfo);
+    }
+
+    @Override
+    public void loadModel() {
+        doLoadModel(LRV2Model.class);
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScoreParam.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.ad.engine.commons.score;
 
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import lombok.Data;
+
 import java.util.*;
 
 /**
@@ -17,9 +18,9 @@ public class ScoreParam {
     private String city;
     private String province;
     private Integer newExpGroup;
+    private String adAbGroup;
     private String pqtId;
-    private Map<String,Object> extraParam=new HashMap<>();
-
+    private Map<String, Object> extraParam = new HashMap<>();
 
 }
 

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerPipeline.java

@@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.*;
 
 
@@ -106,4 +107,80 @@ public class ScorerPipeline {
 
         return items;
     }
+
+
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    final List<AdRankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(scorers)) {
+            log.error("scorers is empty");
+            return rankItems;
+        }
+        List<AdRankItem> items = rankItems;
+
+        for (final AbstractScorer scorer : scorers) {
+            if (!scorer.isEnable()) {
+                continue;
+            }
+
+            final int beforeSize = rankItems.size();
+            final long startTime = System.currentTimeMillis();
+
+            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
+            String[] scorerNames = fullScorerName.split("\\.");
+            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
+
+            final List<AdRankItem> scoreRankerItems = items;
+            Callable<List<AdRankItem>> callable = () -> scorer.scoring(sceneFeatureMap, userFeatureMap, scoreRankerItems);
+
+            // execute score use thread to protected score worst time
+            List<AdRankItem> scoredItems = new ArrayList<AdRankItem>();
+            try {
+                List<Future<List<AdRankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
+                for (Future<List<AdRankItem>> future : futures) {
+                    try {
+                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                            scoredItems.addAll(future.get());
+                        } else {
+                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
+                                    new Object[]{scorerName, scoreRankerItems.size()});
+                        }
+                    } catch (Exception e) {
+                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
+                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+                    }
+                }
+            } catch (Exception e) {
+                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+            }
+
+            //  变更item
+            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
+                items = scoreRankerItems;
+            } else {
+                items = new ArrayList<>(items);
+            }
+
+            int position = 0;
+            for (AdRankItem item : items) {
+                item.getRankerIndex().put(scorerName, position++);
+                item.getRankerScore().put(scorerName, item.getScore());
+            }
+
+            //
+            long spentTime = System.currentTimeMillis() - startTime;
+            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
+                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
+        }
+
+        int position = 0;
+        for (AdRankItem item : items) {
+            item.getRankerIndex().put("finalScore", position++);
+            item.getRankerScore().put("finalScore", item.getScore());
+        }
+
+        return items;
+    }
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerUtils.java

@@ -29,14 +29,18 @@ public final class ScorerUtils {
     public static String CVR_ADJUSTING = "feeds_score_config_cvr_adjusting.conf";
     public static String VIDEO_CREATIVE_THOMPSON = "video_ad_thompson.conf";
 
+    public static String LR_ROV_SCORE_20240626 = "ad_score_config_20240626.conf";
+
     public static void warmUp() {
         log.info("scorer warm up ");
         ScorerUtils.init(BASE_CONF);
         ScorerUtils.init(THOMPSON_CONF);
-        ScorerUtils.init(BREAK_CONFIG);
-        ScorerUtils.init(SHARE0_CONFIG);
+        // ScorerUtils.init(BREAK_CONFIG);
+        // ScorerUtils.init(SHARE0_CONFIG);
         ScorerUtils.init(CVR_ADJUSTING);
-        ScorerUtils.init(VIDEO_CREATIVE_THOMPSON);
+        // ScorerUtils.init(VIDEO_CREATIVE_THOMPSON);
+
+        ScorerUtils.init(LR_ROV_SCORE_20240626);
     }
 
     private ScorerUtils() {

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/FMModel.java

@@ -0,0 +1,144 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class FMModel extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(FMModel.class);
+    private Map<String, List<Float>> model;
+
+    public void putFeature(Map<String, List<Float>> model, String[] items) {
+
+        String featureKey = items[0];
+        List<Float> weights = new ArrayList<>();
+        for (int i = 1; i < items.length; i++) {
+            weights.add(Float.valueOf(items[i]));
+        }
+
+        model.put(featureKey, weights);
+    }
+
+    public float getWeight(Map<String, List<Float>> model, String featureKey, int index) {
+        if (!model.containsKey(featureKey)) {
+            return 0.0f;
+        }
+
+        return model.get(featureKey).get(index);
+
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.model == null)
+            return 0;
+        return model.size();
+    }
+
+    public void cleanModel() {
+        this.model = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+        float sum = 0.0f;
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+            // 计算 sum w*x
+            float sum0 = 0.0f;
+            for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                float x = NumberUtils.toFloat(e.getValue(), 0.0f);
+                float w = getWeight(this.model, e.getKey(), 0);
+                sum0 += w * x;
+            }
+            sum += sum0;
+
+            // 计算 sum v*v*x*X
+            float sum1 = 0.0f;
+            for (int i = 1; i < 9; i++) {
+                float sum10 = 0.0f;
+                float sum11 = 0.0f;
+                for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                    float x = NumberUtils.toFloat(e.getValue(), 0.0f);
+                    float v = getWeight(this.model, e.getKey(), i);
+                    float d = v * x;
+                    sum10 += d;
+                    sum11 += d * d;
+                }
+                sum1 += sum10 * sum10 - sum11;
+            }
+            sum1 = 0.5f * sum1;
+            float biasW = model.get("bias").get(0);
+            sum = biasW + sum0 + sum1;
+        }
+
+        return (float) (1.0f / (1 + Math.exp(-sum)));
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        Map<String, List<Float>> model = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        LOGGER.info("[MODELLOAD] before model load, key size: {}, current time: {}", model.size(), curTime);
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 9) {
+                if (items[0].equals("bias")) {
+                    putFeature(model, items);
+                }
+                continue;
+            }
+
+            putFeature(model, items);
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.model = model;
+
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", model.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 9) {
+                continue;
+            }
+            putFeature(model, items);
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", model.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/LRV2Model.java

@@ -0,0 +1,111 @@
+package com.tzld.piaoquan.ad.engine.commons.score.model;
+
+
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class LRV2Model extends Model {
+    protected static final int MODEL_FIRST_LOAD_COUNT = 1 << 25; // 32M
+    private static final Logger LOGGER = LoggerFactory.getLogger(LRV2Model.class);
+    private Map<String, Float> lrModel;
+
+    public void putFeature(Map<String, Float> model, String featureKey, float weight) {
+        model.put(featureKey, weight);
+    }
+
+    public float getWeight(Map<String, Float> model, String featureKey) {
+        return model.getOrDefault(featureKey, 0.0f);
+    }
+
+    @Override
+    public int getModelSize() {
+        if (this.lrModel == null)
+            return 0;
+        return lrModel.size();
+    }
+
+    public void cleanModel() {
+        this.lrModel = null;
+    }
+
+    public Float score(Map<String, String> featureMap) {
+        float sum = 0.0f;
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+            for (Map.Entry<String, String> e : featureMap.entrySet()) {
+                float w = getWeight(this.lrModel, e.getKey());
+                sum += w * NumberUtils.toFloat(e.getValue(), 0.0f);
+            }
+
+            float biasW = lrModel.get("bias");
+            sum += biasW;
+        }
+
+
+        return (float) (1.0f / (1 + Math.exp(-sum)));
+    }
+
+    /**
+     * 目前模型比较大,分两个阶段load模型
+     * (1). load 8M 模型, 并更新;
+     * (2). load 剩余的模型
+     * 中间提供一段时间有损的打分服务
+     *
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public boolean loadFromStream(InputStreamReader in) throws IOException {
+
+        Map<String, Float> model = new HashMap<>();
+        BufferedReader input = new BufferedReader(in);
+        String line = null;
+        int cnt = 0;
+
+        Integer curTime = new Long(System.currentTimeMillis() / 1000).intValue();
+        //first stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+
+            putFeature(model, items[0], Float.valueOf(items[1].trim()).floatValue());
+            if (cnt++ < 10) {
+                LOGGER.debug("fea: " + items[0] + ", weight: " + items[1]);
+            }
+            if (cnt > MODEL_FIRST_LOAD_COUNT) {
+                break;
+            }
+        }
+        //model update
+        this.lrModel = model;
+
+        LOGGER.info("[MODELLOAD] after first stage model load, key size: {}, current time: {}", lrModel.size(), curTime);
+        //final stage
+        while ((line = input.readLine()) != null) {
+            String[] items = line.split("\t");
+            if (items.length < 2) {
+                continue;
+            }
+            putFeature(model, items[0], Float.valueOf(items[1]).floatValue());
+        }
+        LOGGER.info("[MODELLOAD] after model load, key size: {}, current time: {}", lrModel.size(), curTime);
+
+        LOGGER.info("[MODELLOAD] model load over and size " + cnt);
+        input.close();
+        in.close();
+        return true;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/AbUtil.java

@@ -0,0 +1,39 @@
+package com.tzld.piaoquan.ad.engine.commons.util;
+
+import com.tzld.piaoquan.ad.engine.commons.helper.NewExpHelper;
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class AbUtil {
+
+    public static Set<String> unfoldAllExpCode(List<Map<String, String>> adAbMap) {
+        if (CollectionUtils.isEmpty(adAbMap)) {
+            return new HashSet<>();
+        }
+        return adAbMap.stream()
+                .map(m -> m.get("abExpCode"))
+                .filter(Objects::nonNull)
+                .map(Object::toString)
+                .collect(Collectors.toSet());
+    }
+
+    /**
+     * 判断用户是否在某个实验中，包含在新实验系统和旧实验系统
+     */
+    public static boolean isInAbExp(Collection<String> expCodeSet, Object appType, int newExpGroup, String expCode) {
+        if (CollectionUtils.isEmpty(expCodeSet)) {
+            return false;
+        }
+        if (expCodeSet.contains(expCode)) {
+            return true;
+        }
+
+        if (expCodeSet.contains(NewExpHelper.flagId)) {
+            return NewExpHelper.checkInNewExpGroup(appType.toString(), newExpGroup, expCode);
+        }
+        return false;
+   }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/ExtractorUtils.java

@@ -0,0 +1,205 @@
+package com.tzld.piaoquan.ad.engine.commons.util;
+
+import org.xm.Similarity;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ExtractorUtils {
+
+    public static double sigmoid(double x) {
+        return 1.0 / (1.0 + Math.exp(-x));
+    }
+
+    public static int findInsertPosition(double[] sortedArray, double target) {
+        int low = 0;
+        int high = sortedArray.length - 1;
+
+        while (low <= high) {
+            int mid = low + (high - low) / 2;
+            double midValue = sortedArray[mid];
+
+            if (midValue < target) {
+                low = mid + 1;
+            } else if (midValue > target) {
+                high = mid - 1;
+            } else {
+                // 找到相等的值，尝试在右侧寻找插入点
+                while (mid < sortedArray.length - 1 && sortedArray[mid + 1] == target) {
+                    mid++;
+                }
+                return mid + 1; // 返回当前mid的下一个位置作为插入点
+            }
+        }
+
+        return low; // 返回low作为插入点
+    }
+
+    public static Double[] funcC34567ForTags(String tags, String title) {
+        String[] tagsList = tags.split(",");
+        int d1 = 0;
+        List<String> d2 = new ArrayList<>();
+        double d3 = 0.0;
+        double d4 = 0.0;
+
+        for (String tag : tagsList) {
+            if (title.contains(tag)) {
+                d1++;
+                d2.add(tag);
+            }
+            double score = Similarity.conceptSimilarity(tag, title);
+            if (score > d3) {
+                d3 = score;
+            }
+            d4 += score;
+        }
+
+        d4 = (tagsList.length > 0) ? d4 / tagsList.length : d4;
+
+        // 使用数组来返回多个值
+        return new Double[]{(double) d1, d3, d4};
+    }
+
+    public static Double calDiv(double a, double b) {
+        if (a == 0 || b == 0) {
+            return 0D;
+        }
+        return a / b;
+    }
+
+    public static Double calLog(double a) {
+        if (a <= 0) {
+            return 0D;
+        }
+        return Math.log(a + 1.0);
+    }
+
+    public static Double division(String s1, String s2, Map<String, String> maps) {
+        double rate = 0.0;
+        if (maps.containsKey(s1) && maps.containsKey(s2)) {
+            Double d1 = Double.valueOf(maps.get(s1));
+            if (isDoubleEqualToZero(d1)) {
+                return rate;
+            }
+            Double d2 = Double.valueOf(maps.get(s2));
+            rate = d2 / d1;
+        }
+        return rate;
+    }
+
+    public static Double divisionDouble(Double d1, Double d2) {
+        double rate = 0.0;
+        if (isDoubleEqualToZero(d1)) {
+            return rate;
+        }
+        rate = d2 / d1;
+        return rate;
+    }
+
+    public static boolean isDoubleEqualToZero(double value) {
+        final double epsilon = 1e-10; // 定义一个很小的误差范围
+        // 判断value是否在误差范围内
+        return Math.abs(value) < epsilon;
+    }
+
+
+    public static double calculateVariance(List<Double> numbers) {
+        double average = numbers.stream()
+                .mapToDouble(Double::doubleValue)
+                .average()
+                .orElse(0.0);
+
+        double squaredDiffSum = numbers.stream()
+                .mapToDouble(Double::doubleValue)
+                .map(x -> Math.pow(x - average, 2))
+                .average()
+                .orElse(0.0);
+
+        return squaredDiffSum;
+    }
+
+    public static double calculateAverage(List<Double> numbers) {
+        if (numbers == null || numbers.isEmpty()) {
+            return 0.0;
+        }
+        return numbers.stream()
+                .mapToDouble(Number::doubleValue)
+                .average()
+                .orElse(0.0);
+    }
+
+    public static List<Double> calculateDifferences(List<Double> numbers) {
+        List<Double> differences = new ArrayList<>();
+
+        for (int i = 0; i < numbers.size() - 1; i++) {
+            Double diff = 0.0;
+            if (!isDoubleEqualToZero(numbers.get(i))) {
+                diff = (numbers.get(i + 1) - numbers.get(i)) / numbers.get(i);
+            }
+            differences.add(diff);
+        }
+
+        return differences;
+    }
+
+    public static List<String> generateHourStrings(String timeString, int N) {
+        LocalDateTime dateTime = LocalDateTime.parse(timeString, DateTimeFormatter.ofPattern("yyyyMMddHH"));
+        List<String> hourStrings = new ArrayList<>();
+        for (int i = 0; i < N; i++) {
+            hourStrings.add(dateTime.minusHours(i).format(DateTimeFormatter.ofPattern("yyyyMMddHH")));
+        }
+
+        return hourStrings;
+    }
+
+    public static String subtractHours(String inputDateTime, int hoursToSubtract) {
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHH");
+        LocalDateTime dateTime = LocalDateTime.parse(inputDateTime, formatter);
+        LocalDateTime subtractedDateTime = dateTime.minusHours(hoursToSubtract);
+        return subtractedDateTime.format(formatter);
+    }
+
+    // 针对0-1的数字，进行分桶。
+    public static Integer ceilLogRate(Double key) {
+        double bucket = Math.ceil(
+                Math.pow(key, 0.2) * 100
+        );
+        if (bucket > 300) {
+            bucket = 300;
+        }
+        if (bucket < 0) {
+            bucket = 0;
+        }
+        return (int) bucket;
+    }
+
+    // 针对大于1的数字，进行分桶。
+    public static int bucketCnt(Double key) {
+        long bucket = Math.round(Math.log((key * 10 + 1.0)) * 10);
+        if (bucket > 300) {
+            bucket = 300;
+        }
+        if (bucket < 0) {
+            bucket = 0;
+        }
+        return (int) bucket;
+    }
+
+    public static void main(String[] args) {
+//        System.out.println(ceilLogRate(0.0002));
+//        System.out.println(ceilLogRate(0.01));
+//        System.out.println(ceilLogRate(0.2));
+//        System.out.println(ceilLogRate(4.));
+//        System.out.println(bucketCnt(1.));
+//        System.out.println(bucketCnt(20.));
+//        System.out.println(bucketCnt(500.));
+//        System.out.println(bucketCnt(50000.));
+
+        System.out.println(generateHourStrings("2024011603", 5));
+
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/NumUtil.java

@@ -0,0 +1,12 @@
+package com.tzld.piaoquan.ad.engine.commons.util;
+
+public class NumUtil {
+
+    public static double div(double d1, double d2) {
+        if (d1 == 0 || d2 == 0) {
+            return 0d;
+        }
+        return d1 / d2;
+    }
+
+}

ad-engine-server/pom.xml

@@ -31,6 +31,7 @@
                     <include>**/*.xml</include>
                     <include>**/*.conf</include>
                     <include>**/*.properties</include>
+                    <include>**/*.txt</include>
                 </includes>
                 <filtering>true</filtering>
             </resource>

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/AdRecommendController.java

@@ -15,6 +15,7 @@ import org.springframework.web.bind.annotation.RestController;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 
 @RestController
 @RequestMapping("/recommend")
@@ -26,48 +27,54 @@ public class AdRecommendController {
     RankService rankService;
 
     @RequestMapping("/top1/basic")
-    public Map<String,Object> adRecommendTop1Basic(@RequestBody RankRecommendRequestParam request){
+    public Map<String, Object> adRecommendTop1Basic(@RequestBody RankRecommendRequestParam request) {
         AdRankItem rankResult = rankService.adItemRank(request);
-        HashMap map =new HashMap();
-        map.put("code","0");
-        map.put("msg","success");
-        HashMap contentMap=new HashMap<>();
-        contentMap.put("adId", rankResult.getAdId());
-        contentMap.put("adScore", rankResult.getScore());
-        map.put("content",contentMap);
+        Map<String, Object> map = new HashMap<>();
+
+        if (Objects.isNull(rankResult)) {
+            map.put("code", "1");
+            map.put("msg", "score error");
+        } else {
+            map.put("code", "0");
+            map.put("msg", "success");
+            Map<String, Object> contentMap = new HashMap<>();
+            contentMap.put("adId", rankResult.getAdId());
+            contentMap.put("adScore", rankResult.getScore());
+            map.put("content", contentMap);
+        }
         return map;
     }
 
     @RequestMapping("/top1/video/ad/thompson")
-    public Map<String,Object> adRecommendTop1VideoAdThompson(@RequestBody RankRecommendRequestParam request){
+    public Map<String, Object> adRecommendTop1VideoAdThompson(@RequestBody RankRecommendRequestParam request) {
         AdRankItem rankResult = rankService.adItemRankWithVideoAdThompson(request);
-        HashMap map =new HashMap();
-        map.put("code","0");
-        map.put("msg","success");
-        HashMap contentMap=new HashMap<>();
+        HashMap map = new HashMap();
+        map.put("code", "0");
+        map.put("msg", "success");
+        HashMap contentMap = new HashMap<>();
         contentMap.put("adId", rankResult.getAdId());
         contentMap.put("adScore", rankResult.getScore());
-        map.put("content",contentMap);
+        map.put("content", contentMap);
         return map;
     }
 
     @RequestMapping("/top1/bid/basic")
-    public Map<String,Object> adRecommendTop1BidBasic(@RequestBody BidRankRecommendRequestParam request){
+    public Map<String, Object> adRecommendTop1BidBasic(@RequestBody BidRankRecommendRequestParam request) {
         AdPlatformCreativeDTO rankResult = rankService.adBidRank(request);
-        HashMap map =new HashMap();
-        map.put("code","0");
-        map.put("msg","success");
-        map.put("content",rankResult);
+        HashMap map = new HashMap();
+        map.put("code", "0");
+        map.put("msg", "success");
+        map.put("content", rankResult);
         return map;
     }
 
     @RequestMapping("/top1/bid/new/pid")
-    public Map<String,Object> adRecommendTop1BidNewPid(@RequestBody BidRankRecommendRequestParam request){
+    public Map<String, Object> adRecommendTop1BidNewPid(@RequestBody BidRankRecommendRequestParam request) {
         AdPlatformCreativeDTO rankResult = rankService.adBidRankNewPid(request);
-        HashMap map =new HashMap();
-        map.put("code","0");
-        map.put("msg","success");
-        map.put("content",rankResult);
+        HashMap map = new HashMap();
+        map.put("code", "0");
+        map.put("msg", "success");
+        map.put("content", rankResult);
         return map;
     }
 

ad-engine-server/src/main/resources/ad_score_config_20240626.conf

@@ -0,0 +1,8 @@
+scorer-config = {
+  lr-rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogRovLRScorer"
+    scorer-priority = 99
+    model-path = "zhangbo/model_bkb0.txt"
+  }
+
+}

ad-engine-server/src/main/resources/application.yml

@@ -76,7 +76,8 @@ mybatis:
     log-impl: org.apache.ibatis.logging.slf4j.Slf4jImpl
 
 datalog: /datalog
-#datalog: /Users/gufengshou/Documents/yiqi_project
+
+#datalog: /Users/zhao/Desktop/Code/Java/ad-engine
 
 logging:
   file:

ad-engine-server/src/main/resources/logback-spring.xml

@@ -1,300 +1,300 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- 日志级别从低到高分为TRACE < DEBUG < INFO < WARN < ERROR < FATAL，如果设置为WARN，则低于WARN的信息都不会输出 -->
-<!-- scan:当此属性设置为true时，配置文件如果发生改变，将会被重新加载，默认值为true -->
-<!-- scanPeriod:设置监测配置文件是否有修改的时间间隔，如果没有给出时间单位，默认单位是毫秒。当scan为true时，此属性生效。默认的时间间隔为1分钟。 -->
-<!-- debug:当此属性设置为true时，将打印出logback内部日志信息，实时查看logback运行状态。默认值为false。 -->
-<configuration scan="true" scanPeriod="10 seconds">
-    <!--<include resource="org/springframework/boot/logging/logback/base.xml"/>-->
+<!--<?xml version="1.0" encoding="UTF-8"?>-->
+<!--&lt;!&ndash; 日志级别从低到高分为TRACE < DEBUG < INFO < WARN < ERROR < FATAL，如果设置为WARN，则低于WARN的信息都不会输出 &ndash;&gt;-->
+<!--&lt;!&ndash; scan:当此属性设置为true时，配置文件如果发生改变，将会被重新加载，默认值为true &ndash;&gt;-->
+<!--&lt;!&ndash; scanPeriod:设置监测配置文件是否有修改的时间间隔，如果没有给出时间单位，默认单位是毫秒。当scan为true时，此属性生效。默认的时间间隔为1分钟。 &ndash;&gt;-->
+<!--&lt;!&ndash; debug:当此属性设置为true时，将打印出logback内部日志信息，实时查看logback运行状态。默认值为false。 &ndash;&gt;-->
+<!--<configuration scan="true" scanPeriod="10 seconds">-->
+<!--    &lt;!&ndash;<include resource="org/springframework/boot/logging/logback/base.xml"/>&ndash;&gt;-->
 
-    <contextName>logback</contextName>
-    <!-- name的值是变量的名称，value的值时变量定义的值。通过定义的值会被插入到logger上下文中。定义变量后，可以使“${}”来使用变量。 -->
-    <!-- <property name="LOG_PATH"  value="${logging.file.path}" />-->
+<!--    <contextName>logback</contextName>-->
+<!--    &lt;!&ndash; name的值是变量的名称，value的值时变量定义的值。通过定义的值会被插入到logger上下文中。定义变量后，可以使“${}”来使用变量。 &ndash;&gt;-->
+<!--    &lt;!&ndash; <property name="LOG_PATH"  value="${logging.file.path}" />&ndash;&gt;-->
 
-    <springProperty name="LOG_PATH" source="logging.file.path"/>
-    <springProperty name="ALIYUN_LOG_ENDPOINT" source="aliyun.log.endpoint"/>
-    <springProperty name="ALIYUN_LOG_ACCESSKEYID" source="aliyun.log.accessKeyId"/>
-    <springProperty name="ALIYUN_LOG_ACCESSKEYSECRET" source="aliyun.log.accessKeySecret"/>
-    <springProperty name="ALIYUN_LOG_PROJECT" source="aliyun.log.project"/>
-    <springProperty name="ALIYUN_LOG_LOGSTORE_INFO" source="aliyun.log.logstore.info"/>
-    <springProperty name="ALIYUN_LOG_LOGSTORE_ERROR" source="aliyun.log.logstore.error"/>
+<!--    <springProperty name="LOG_PATH" source="logging.file.path"/>-->
+<!--    <springProperty name="ALIYUN_LOG_ENDPOINT" source="aliyun.log.endpoint"/>-->
+<!--    <springProperty name="ALIYUN_LOG_ACCESSKEYID" source="aliyun.log.accessKeyId"/>-->
+<!--    <springProperty name="ALIYUN_LOG_ACCESSKEYSECRET" source="aliyun.log.accessKeySecret"/>-->
+<!--    <springProperty name="ALIYUN_LOG_PROJECT" source="aliyun.log.project"/>-->
+<!--    <springProperty name="ALIYUN_LOG_LOGSTORE_INFO" source="aliyun.log.logstore.info"/>-->
+<!--    <springProperty name="ALIYUN_LOG_LOGSTORE_ERROR" source="aliyun.log.logstore.error"/>-->
 
-    <!-- 彩色日志 -->
-    <!-- 彩色日志依赖的渲染类 -->
-    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
-    <conversionRule conversionWord="wex"
-                    converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
-    <conversionRule conversionWord="wEx"
-                    converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
-    <!-- 彩色日志格式 -->
-    <property name="CONSOLE_LOG_PATTERN"
-              value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr([%X{pqtId}]){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
-    <!--<property name="CONSOLE_LOG_PATTERN"-->
-    <!--          value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr([%X{√pqtId}]){magenta} %clr(-&#45;&#45;){faint} %clr([%15.15t]){faint} %clr(at %class.%method){cyan} \\(%file:%line\\) %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>-->
+<!--    &lt;!&ndash; 彩色日志 &ndash;&gt;-->
+<!--    &lt;!&ndash; 彩色日志依赖的渲染类 &ndash;&gt;-->
+<!--    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>-->
+<!--    <conversionRule conversionWord="wex"-->
+<!--                    converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>-->
+<!--    <conversionRule conversionWord="wEx"-->
+<!--                    converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>-->
+<!--    &lt;!&ndash; 彩色日志格式 &ndash;&gt;-->
+<!--    <property name="CONSOLE_LOG_PATTERN"-->
+<!--              value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr([%X{pqtId}]){magenta} %clr(-&#45;&#45;){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>-->
+<!--    &lt;!&ndash;<property name="CONSOLE_LOG_PATTERN"&ndash;&gt;-->
+<!--    &lt;!&ndash;          value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr([%X{√pqtId}]){magenta} %clr(-&#45;&#45;){faint} %clr([%15.15t]){faint} %clr(at %class.%method){cyan} \\(%file:%line\\) %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>&ndash;&gt;-->
 
 
-    <!--输出到控制台-->
-    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
-        <!--此日志appender是为开发使用，只配置最底级别，控制台输出的日志级别是大于或等于此级别的日志信息-->
-        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-            <level>debug</level>
-        </filter>
-        <encoder>
-            <Pattern>${CONSOLE_LOG_PATTERN}</Pattern>
-            <!-- 设置字符集 -->
-            <charset>UTF-8</charset>
-        </encoder>
-    </appender>
-    <appender name="ASYNC_CONSOLE" class="ch.qos.logback.classic.AsyncAppender">
-        <appender-ref ref="CONSOLE" />
-        <queueSize>10000</queueSize> <!-- 队列大小，可以根据需要调整 -->
-    </appender>
-    <!--输出到文件-->
-    <!-- 时间滚动输出 level为 DEBUG 日志 -->
-    <appender name="DEBUG_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <!-- 正在记录的日志文件的路径及文件名 -->
-        <file>${LOG_PATH}/debug.log</file>
-        <!--日志文件输出格式-->
-        <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
-            <charset>UTF-8</charset> <!-- 设置字符集 -->
-        </encoder>
-        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-            <!-- 日志归档 -->
-            <fileNamePattern>${LOG_PATH}/debug/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
-            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
-                <maxFileSize>100MB</maxFileSize>
-            </timeBasedFileNamingAndTriggeringPolicy>
-            <!--日志文件保留天数-->
-            <maxHistory>15</maxHistory>
-        </rollingPolicy>
-        <!-- 此日志文件只记录debug级别的 -->
-        <filter class="ch.qos.logback.classic.filter.LevelFilter">
-            <level>debug</level>
-            <onMatch>ACCEPT</onMatch>
-            <onMismatch>DENY</onMismatch>
-        </filter>
-    </appender>
-    <appender name="ASYNC_DEBUG_FILE" class="ch.qos.logback.classic.AsyncAppender">
-        <appender-ref ref="DEBUG_FILE" />
-        <queueSize>1000</queueSize> <!-- 队列大小，可以根据需要调整 -->
-    </appender>
-    <!-- 时间滚动输出 level为 INFO 日志 -->
-    <appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <!-- 正在记录的日志文件的路径及文件名 -->
-        <file>${LOG_PATH}/info.log</file>
-        <!--日志文件输出格式-->
-        <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
-            <charset>UTF-8</charset>
-        </encoder>
-        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-            <!-- 每天日志归档路径以及格式 -->
-            <fileNamePattern>${LOG_PATH}/info/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
-            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
-                <maxFileSize>100MB</maxFileSize>
-            </timeBasedFileNamingAndTriggeringPolicy>
-            <!--日志文件保留天数-->
-            <maxHistory>15</maxHistory>
-        </rollingPolicy>
-        <!-- 此日志文件只记录info级别的 -->
-        <filter class="ch.qos.logback.classic.filter.LevelFilter">
-            <level>info</level>
-            <onMatch>ACCEPT</onMatch>
-            <onMismatch>DENY</onMismatch>
-        </filter>
-    </appender>
-    <appender name="ASYNC_INFO_FILE" class="ch.qos.logback.classic.AsyncAppender">
-        <appender-ref ref="INFO_FILE" />
-        <queueSize>10000</queueSize> <!-- 队列大小，可以根据需要调整 -->
-    </appender>
+<!--    &lt;!&ndash;输出到控制台&ndash;&gt;-->
+<!--    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">-->
+<!--        &lt;!&ndash;此日志appender是为开发使用，只配置最底级别，控制台输出的日志级别是大于或等于此级别的日志信息&ndash;&gt;-->
+<!--        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">-->
+<!--            <level>debug</level>-->
+<!--        </filter>-->
+<!--        <encoder>-->
+<!--            <Pattern>${CONSOLE_LOG_PATTERN}</Pattern>-->
+<!--            &lt;!&ndash; 设置字符集 &ndash;&gt;-->
+<!--            <charset>UTF-8</charset>-->
+<!--        </encoder>-->
+<!--    </appender>-->
+<!--    <appender name="ASYNC_CONSOLE" class="ch.qos.logback.classic.AsyncAppender">-->
+<!--        <appender-ref ref="CONSOLE" />-->
+<!--        <queueSize>10000</queueSize> &lt;!&ndash; 队列大小，可以根据需要调整 &ndash;&gt;-->
+<!--    </appender>-->
+<!--    &lt;!&ndash;输出到文件&ndash;&gt;-->
+<!--    &lt;!&ndash; 时间滚动输出 level为 DEBUG 日志 &ndash;&gt;-->
+<!--    <appender name="DEBUG_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">-->
+<!--        &lt;!&ndash; 正在记录的日志文件的路径及文件名 &ndash;&gt;-->
+<!--        <file>${LOG_PATH}/debug.log</file>-->
+<!--        &lt;!&ndash;日志文件输出格式&ndash;&gt;-->
+<!--        <encoder>-->
+<!--            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>-->
+<!--            <charset>UTF-8</charset> &lt;!&ndash; 设置字符集 &ndash;&gt;-->
+<!--        </encoder>-->
+<!--        &lt;!&ndash; 日志记录器的滚动策略，按日期，按大小记录 &ndash;&gt;-->
+<!--        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">-->
+<!--            &lt;!&ndash; 日志归档 &ndash;&gt;-->
+<!--            <fileNamePattern>${LOG_PATH}/debug/%d{yyyy-MM-dd}.%i.log</fileNamePattern>-->
+<!--            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">-->
+<!--                <maxFileSize>100MB</maxFileSize>-->
+<!--            </timeBasedFileNamingAndTriggeringPolicy>-->
+<!--            &lt;!&ndash;日志文件保留天数&ndash;&gt;-->
+<!--            <maxHistory>15</maxHistory>-->
+<!--        </rollingPolicy>-->
+<!--        &lt;!&ndash; 此日志文件只记录debug级别的 &ndash;&gt;-->
+<!--        <filter class="ch.qos.logback.classic.filter.LevelFilter">-->
+<!--            <level>debug</level>-->
+<!--            <onMatch>ACCEPT</onMatch>-->
+<!--            <onMismatch>DENY</onMismatch>-->
+<!--        </filter>-->
+<!--    </appender>-->
+<!--    <appender name="ASYNC_DEBUG_FILE" class="ch.qos.logback.classic.AsyncAppender">-->
+<!--        <appender-ref ref="DEBUG_FILE" />-->
+<!--        <queueSize>1000</queueSize> &lt;!&ndash; 队列大小，可以根据需要调整 &ndash;&gt;-->
+<!--    </appender>-->
+<!--    &lt;!&ndash; 时间滚动输出 level为 INFO 日志 &ndash;&gt;-->
+<!--    <appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">-->
+<!--        &lt;!&ndash; 正在记录的日志文件的路径及文件名 &ndash;&gt;-->
+<!--        <file>${LOG_PATH}/info.log</file>-->
+<!--        &lt;!&ndash;日志文件输出格式&ndash;&gt;-->
+<!--        <encoder>-->
+<!--            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>-->
+<!--            <charset>UTF-8</charset>-->
+<!--        </encoder>-->
+<!--        &lt;!&ndash; 日志记录器的滚动策略，按日期，按大小记录 &ndash;&gt;-->
+<!--        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">-->
+<!--            &lt;!&ndash; 每天日志归档路径以及格式 &ndash;&gt;-->
+<!--            <fileNamePattern>${LOG_PATH}/info/%d{yyyy-MM-dd}.%i.log</fileNamePattern>-->
+<!--            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">-->
+<!--                <maxFileSize>100MB</maxFileSize>-->
+<!--            </timeBasedFileNamingAndTriggeringPolicy>-->
+<!--            &lt;!&ndash;日志文件保留天数&ndash;&gt;-->
+<!--            <maxHistory>15</maxHistory>-->
+<!--        </rollingPolicy>-->
+<!--        &lt;!&ndash; 此日志文件只记录info级别的 &ndash;&gt;-->
+<!--        <filter class="ch.qos.logback.classic.filter.LevelFilter">-->
+<!--            <level>info</level>-->
+<!--            <onMatch>ACCEPT</onMatch>-->
+<!--            <onMismatch>DENY</onMismatch>-->
+<!--        </filter>-->
+<!--    </appender>-->
+<!--    <appender name="ASYNC_INFO_FILE" class="ch.qos.logback.classic.AsyncAppender">-->
+<!--        <appender-ref ref="INFO_FILE" />-->
+<!--        <queueSize>10000</queueSize> &lt;!&ndash; 队列大小，可以根据需要调整 &ndash;&gt;-->
+<!--    </appender>-->
 
-    <!-- 时间滚动输出 level为 WARN 日志 -->
-    <appender name="WARN_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <!-- 正在记录的日志文件的路径及文件名 -->
-        <file>${LOG_PATH}/warn.log</file>
-        <!--日志文件输出格式-->
-        <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
-            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
-        </encoder>
-        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-            <fileNamePattern>${LOG_PATH}/warn/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
-            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
-                <maxFileSize>100MB</maxFileSize>
-            </timeBasedFileNamingAndTriggeringPolicy>
-            <!--日志文件保留天数-->
-            <maxHistory>15</maxHistory>
-        </rollingPolicy>
-        <!-- 此日志文件只记录warn级别的 -->
-        <filter class="ch.qos.logback.classic.filter.LevelFilter">
-            <level>warn</level>
-            <onMatch>ACCEPT</onMatch>
-            <onMismatch>DENY</onMismatch>
-        </filter>
-    </appender>
-    <appender name="ASYNC_WARN_FILE" class="ch.qos.logback.classic.AsyncAppender">
-        <appender-ref ref="WARN_FILE" />
-        <queueSize>3000</queueSize> <!-- 队列大小，可以根据需要调整 -->
-    </appender>
+<!--    &lt;!&ndash; 时间滚动输出 level为 WARN 日志 &ndash;&gt;-->
+<!--    <appender name="WARN_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">-->
+<!--        &lt;!&ndash; 正在记录的日志文件的路径及文件名 &ndash;&gt;-->
+<!--        <file>${LOG_PATH}/warn.log</file>-->
+<!--        &lt;!&ndash;日志文件输出格式&ndash;&gt;-->
+<!--        <encoder>-->
+<!--            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>-->
+<!--            <charset>UTF-8</charset> &lt;!&ndash; 此处设置字符集 &ndash;&gt;-->
+<!--        </encoder>-->
+<!--        &lt;!&ndash; 日志记录器的滚动策略，按日期，按大小记录 &ndash;&gt;-->
+<!--        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">-->
+<!--            <fileNamePattern>${LOG_PATH}/warn/%d{yyyy-MM-dd}.%i.log</fileNamePattern>-->
+<!--            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">-->
+<!--                <maxFileSize>100MB</maxFileSize>-->
+<!--            </timeBasedFileNamingAndTriggeringPolicy>-->
+<!--            &lt;!&ndash;日志文件保留天数&ndash;&gt;-->
+<!--            <maxHistory>15</maxHistory>-->
+<!--        </rollingPolicy>-->
+<!--        &lt;!&ndash; 此日志文件只记录warn级别的 &ndash;&gt;-->
+<!--        <filter class="ch.qos.logback.classic.filter.LevelFilter">-->
+<!--            <level>warn</level>-->
+<!--            <onMatch>ACCEPT</onMatch>-->
+<!--            <onMismatch>DENY</onMismatch>-->
+<!--        </filter>-->
+<!--    </appender>-->
+<!--    <appender name="ASYNC_WARN_FILE" class="ch.qos.logback.classic.AsyncAppender">-->
+<!--        <appender-ref ref="WARN_FILE" />-->
+<!--        <queueSize>3000</queueSize> &lt;!&ndash; 队列大小，可以根据需要调整 &ndash;&gt;-->
+<!--    </appender>-->
 
-    <!-- 时间滚动输出 level为 ERROR 日志 -->
-    <appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <!-- 正在记录的日志文件的路径及文件名 -->
-        <file>${LOG_PATH}/error.log</file>
-        <!--日志文件输出格式-->
-        <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
-            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
-        </encoder>
-        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-            <fileNamePattern>${LOG_PATH}/error/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
-            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
-                <maxFileSize>100MB</maxFileSize>
-            </timeBasedFileNamingAndTriggeringPolicy>
-            <!--日志文件保留天数-->
-            <maxHistory>15</maxHistory>
-        </rollingPolicy>
-        <!-- 此日志文件只记录ERROR级别的 -->
-        <filter class="ch.qos.logback.classic.filter.LevelFilter">
-            <level>ERROR</level>
-            <onMatch>ACCEPT</onMatch>
-            <onMismatch>DENY</onMismatch>
-        </filter>
-    </appender>
-    <appender name="ASYNC_ERROR_FILE" class="ch.qos.logback.classic.AsyncAppender">
-        <appender-ref ref="ERROR_FILE" />
-        <queueSize>3000</queueSize> <!-- 队列大小，可以根据需要调整 -->
-    </appender>
+<!--    &lt;!&ndash; 时间滚动输出 level为 ERROR 日志 &ndash;&gt;-->
+<!--    <appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">-->
+<!--        &lt;!&ndash; 正在记录的日志文件的路径及文件名 &ndash;&gt;-->
+<!--        <file>${LOG_PATH}/error.log</file>-->
+<!--        &lt;!&ndash;日志文件输出格式&ndash;&gt;-->
+<!--        <encoder>-->
+<!--            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>-->
+<!--            <charset>UTF-8</charset> &lt;!&ndash; 此处设置字符集 &ndash;&gt;-->
+<!--        </encoder>-->
+<!--        &lt;!&ndash; 日志记录器的滚动策略，按日期，按大小记录 &ndash;&gt;-->
+<!--        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">-->
+<!--            <fileNamePattern>${LOG_PATH}/error/%d{yyyy-MM-dd}.%i.log</fileNamePattern>-->
+<!--            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">-->
+<!--                <maxFileSize>100MB</maxFileSize>-->
+<!--            </timeBasedFileNamingAndTriggeringPolicy>-->
+<!--            &lt;!&ndash;日志文件保留天数&ndash;&gt;-->
+<!--            <maxHistory>15</maxHistory>-->
+<!--        </rollingPolicy>-->
+<!--        &lt;!&ndash; 此日志文件只记录ERROR级别的 &ndash;&gt;-->
+<!--        <filter class="ch.qos.logback.classic.filter.LevelFilter">-->
+<!--            <level>ERROR</level>-->
+<!--            <onMatch>ACCEPT</onMatch>-->
+<!--            <onMismatch>DENY</onMismatch>-->
+<!--        </filter>-->
+<!--    </appender>-->
+<!--    <appender name="ASYNC_ERROR_FILE" class="ch.qos.logback.classic.AsyncAppender">-->
+<!--        <appender-ref ref="ERROR_FILE" />-->
+<!--        <queueSize>3000</queueSize> &lt;!&ndash; 队列大小，可以根据需要调整 &ndash;&gt;-->
+<!--    </appender>-->
 
-    <appender name="ALIYUN_LOG_INFO" class="com.aliyun.openservices.log.logback.LoghubAppender">
-        <endpoint>${ALIYUN_LOG_ENDPOINT}</endpoint>
-        <accessKeyId>${ALIYUN_LOG_ACCESSKEYID}</accessKeyId>
-        <accessKeySecret>${ALIYUN_LOG_ACCESSKEYSECRET}</accessKeySecret>
-        <project>${ALIYUN_LOG_PROJECT}</project>
-        <logStore>${ALIYUN_LOG_LOGSTORE_INFO}</logStore>
+<!--    <appender name="ALIYUN_LOG_INFO" class="com.aliyun.openservices.log.logback.LoghubAppender">-->
+<!--        <endpoint>${ALIYUN_LOG_ENDPOINT}</endpoint>-->
+<!--        <accessKeyId>${ALIYUN_LOG_ACCESSKEYID}</accessKeyId>-->
+<!--        <accessKeySecret>${ALIYUN_LOG_ACCESSKEYSECRET}</accessKeySecret>-->
+<!--        <project>${ALIYUN_LOG_PROJECT}</project>-->
+<!--        <logStore>${ALIYUN_LOG_LOGSTORE_INFO}</logStore>-->
 
-        <totalSizeInBytes>104857600</totalSizeInBytes>
-        <maxBlockMs>0</maxBlockMs>
-        <ioThreadCount>8</ioThreadCount>
-        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>
-        <batchCountThreshold>4096</batchCountThreshold>
-        <lingerMs>2000</lingerMs>
-        <retries>10</retries>
-        <baseRetryBackoffMs>100</baseRetryBackoffMs>
-        <maxRetryBackoffMs>50000</maxRetryBackoffMs>
+<!--        <totalSizeInBytes>104857600</totalSizeInBytes>-->
+<!--        <maxBlockMs>0</maxBlockMs>-->
+<!--        <ioThreadCount>8</ioThreadCount>-->
+<!--        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>-->
+<!--        <batchCountThreshold>4096</batchCountThreshold>-->
+<!--        <lingerMs>2000</lingerMs>-->
+<!--        <retries>10</retries>-->
+<!--        <baseRetryBackoffMs>100</baseRetryBackoffMs>-->
+<!--        <maxRetryBackoffMs>50000</maxRetryBackoffMs>-->
 
-        <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
-            <charset>UTF-8</charset>
-        </encoder>
+<!--        <encoder>-->
+<!--            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>-->
+<!--            <charset>UTF-8</charset>-->
+<!--        </encoder>-->
 
-        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>
-        <timeZone>Asia/Shanghai</timeZone>
+<!--        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>-->
+<!--        <timeZone>Asia/Shanghai</timeZone>-->
 
-        <filter class="ch.qos.logback.classic.filter.LevelFilter">
-            <level>INFO</level>
-        </filter>
+<!--        <filter class="ch.qos.logback.classic.filter.LevelFilter">-->
+<!--            <level>INFO</level>-->
+<!--        </filter>-->
 
-        <mdcFields>pqtId</mdcFields>
-    </appender>
+<!--        <mdcFields>pqtId</mdcFields>-->
+<!--    </appender>-->
 
-    <appender name="ALIYUN_LOG_ERROR" class="com.aliyun.openservices.log.logback.LoghubAppender">
-        <endpoint>${ALIYUN_LOG_ENDPOINT}</endpoint>
-        <accessKeyId>${ALIYUN_LOG_ACCESSKEYID}</accessKeyId>
-        <accessKeySecret>${ALIYUN_LOG_ACCESSKEYSECRET}</accessKeySecret>
-        <project>${ALIYUN_LOG_PROJECT}</project>
-        <logStore>${ALIYUN_LOG_LOGSTORE_ERROR}</logStore>
+<!--    <appender name="ALIYUN_LOG_ERROR" class="com.aliyun.openservices.log.logback.LoghubAppender">-->
+<!--        <endpoint>${ALIYUN_LOG_ENDPOINT}</endpoint>-->
+<!--        <accessKeyId>${ALIYUN_LOG_ACCESSKEYID}</accessKeyId>-->
+<!--        <accessKeySecret>${ALIYUN_LOG_ACCESSKEYSECRET}</accessKeySecret>-->
+<!--        <project>${ALIYUN_LOG_PROJECT}</project>-->
+<!--        <logStore>${ALIYUN_LOG_LOGSTORE_ERROR}</logStore>-->
 
-        <totalSizeInBytes>104857600</totalSizeInBytes>
-        <maxBlockMs>0</maxBlockMs>
-        <ioThreadCount>8</ioThreadCount>
-        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>
-        <batchCountThreshold>4096</batchCountThreshold>
-        <lingerMs>2000</lingerMs>
-        <retries>10</retries>
-        <baseRetryBackoffMs>100</baseRetryBackoffMs>
-        <maxRetryBackoffMs>50000</maxRetryBackoffMs>
+<!--        <totalSizeInBytes>104857600</totalSizeInBytes>-->
+<!--        <maxBlockMs>0</maxBlockMs>-->
+<!--        <ioThreadCount>8</ioThreadCount>-->
+<!--        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>-->
+<!--        <batchCountThreshold>4096</batchCountThreshold>-->
+<!--        <lingerMs>2000</lingerMs>-->
+<!--        <retries>10</retries>-->
+<!--        <baseRetryBackoffMs>100</baseRetryBackoffMs>-->
+<!--        <maxRetryBackoffMs>50000</maxRetryBackoffMs>-->
 
-        <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
-            <charset>UTF-8</charset>
-        </encoder>
+<!--        <encoder>-->
+<!--            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>-->
+<!--            <charset>UTF-8</charset>-->
+<!--        </encoder>-->
 
-        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>
-        <timeZone>Asia/Shanghai</timeZone>
+<!--        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>-->
+<!--        <timeZone>Asia/Shanghai</timeZone>-->
 
-        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-            <level>ERROR</level>
-        </filter>
+<!--        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">-->
+<!--            <level>ERROR</level>-->
+<!--        </filter>-->
 
-        <mdcFields>pqtId</mdcFields>
-    </appender>
+<!--        <mdcFields>pqtId</mdcFields>-->
+<!--    </appender>-->
 
-    <!--
-        <logger>用来设置某一个包或者具体的某一个类的日志打印级别、以及指定<appender>。
-        <logger>仅有一个name属性，一个可选的level和一个可选的addtivity属性。
-        name:用来指定受此logger约束的某一个包或者具体的某一个类。
-        level:用来设置打印级别，大小写无关：TRACE, DEBUG, INFO, WARN, ERROR, ALL 和 OFF，
-              还有一个特俗值INHERITED或者同义词NULL，代表强制执行上级的级别。
-              如果未设置此属性，那么当前logger将会继承上级的级别。
-        addtivity:是否向上级logger传递打印信息。默认是true。
-    -->
-    <!--<logger name="org.springframework.web" level="info"/>-->
-    <!--<logger name="org.springframework.scheduling.annotation.ScheduledAnnotationBeanPostProcessor" level="INFO"/>-->
+<!--    &lt;!&ndash;-->
+<!--        <logger>用来设置某一个包或者具体的某一个类的日志打印级别、以及指定<appender>。-->
+<!--        <logger>仅有一个name属性，一个可选的level和一个可选的addtivity属性。-->
+<!--        name:用来指定受此logger约束的某一个包或者具体的某一个类。-->
+<!--        level:用来设置打印级别，大小写无关：TRACE, DEBUG, INFO, WARN, ERROR, ALL 和 OFF，-->
+<!--              还有一个特俗值INHERITED或者同义词NULL，代表强制执行上级的级别。-->
+<!--              如果未设置此属性，那么当前logger将会继承上级的级别。-->
+<!--        addtivity:是否向上级logger传递打印信息。默认是true。-->
+<!--    &ndash;&gt;-->
+<!--    &lt;!&ndash;<logger name="org.springframework.web" level="info"/>&ndash;&gt;-->
+<!--    &lt;!&ndash;<logger name="org.springframework.scheduling.annotation.ScheduledAnnotationBeanPostProcessor" level="INFO"/>&ndash;&gt;-->
 
-    <!--
-        使用mybatis的时候，sql语句是debug下才会打印，而这里我们只配置了info，所以想要查看sql语句的话，有以下两种操作：
-        第一种把<root level="info">改成<root level="DEBUG">这样就会打印sql，不过这样日志那边会出现很多其他消息
-        第二种就是单独给dao下目录配置 <logger> 指定 level 属性为 debug，这样配置sql语句会打印，其他还是正常info级别：
-     -->
+<!--    &lt;!&ndash;-->
+<!--        使用mybatis的时候，sql语句是debug下才会打印，而这里我们只配置了info，所以想要查看sql语句的话，有以下两种操作：-->
+<!--        第一种把<root level="info">改成<root level="DEBUG">这样就会打印sql，不过这样日志那边会出现很多其他消息-->
+<!--        第二种就是单独给dao下目录配置 <logger> 指定 level 属性为 debug，这样配置sql语句会打印，其他还是正常info级别：-->
+<!--     &ndash;&gt;-->
 
-    <!--<springProfile name="dev">-->
-    <!--    <logger name="com.tzld.piaoquan.ad" level="debug"/>-->
-    <!--    <logger name="com.ctrip.framework.apollo.internals.RemoteConfigLongPollService" level="error"/>-->
-    <!--</springProfile>-->
-    <!--<springProfile name="test">-->
-    <!--    <logger name="com.tzld.piaoquan.ad" level="info"/>-->
-    <!--</springProfile>-->
-    <!--<springProfile name="pre">-->
-    <!--    <logger name="com.tzld.piaoquan.ad" level="info"/>-->
-    <!--</springProfile>-->
-    <!--<springProfile name="stress">-->
-    <!--    <logger name="com.tzld.piaoquan.ad" level="info"/>-->
-    <!--</springProfile>-->
-    <!--<springProfile name="prod">-->
-    <!--    <logger name="com.tzld.piaoquan.ad" level="info"/>-->
-    <!--</springProfile>-->
-    <!--<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="warn"/>-->
+<!--    &lt;!&ndash;<springProfile name="dev">&ndash;&gt;-->
+<!--    &lt;!&ndash;    <logger name="com.tzld.piaoquan.ad" level="debug"/>&ndash;&gt;-->
+<!--    &lt;!&ndash;    <logger name="com.ctrip.framework.apollo.internals.RemoteConfigLongPollService" level="error"/>&ndash;&gt;-->
+<!--    &lt;!&ndash;</springProfile>&ndash;&gt;-->
+<!--    &lt;!&ndash;<springProfile name="test">&ndash;&gt;-->
+<!--    &lt;!&ndash;    <logger name="com.tzld.piaoquan.ad" level="info"/>&ndash;&gt;-->
+<!--    &lt;!&ndash;</springProfile>&ndash;&gt;-->
+<!--    &lt;!&ndash;<springProfile name="pre">&ndash;&gt;-->
+<!--    &lt;!&ndash;    <logger name="com.tzld.piaoquan.ad" level="info"/>&ndash;&gt;-->
+<!--    &lt;!&ndash;</springProfile>&ndash;&gt;-->
+<!--    &lt;!&ndash;<springProfile name="stress">&ndash;&gt;-->
+<!--    &lt;!&ndash;    <logger name="com.tzld.piaoquan.ad" level="info"/>&ndash;&gt;-->
+<!--    &lt;!&ndash;</springProfile>&ndash;&gt;-->
+<!--    &lt;!&ndash;<springProfile name="prod">&ndash;&gt;-->
+<!--    &lt;!&ndash;    <logger name="com.tzld.piaoquan.ad" level="info"/>&ndash;&gt;-->
+<!--    &lt;!&ndash;</springProfile>&ndash;&gt;-->
+<!--    &lt;!&ndash;<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="warn"/>&ndash;&gt;-->
 
-    <!-- 可用来获取 StatusManager 中的状态 -->
-    <!--<statusListener class="ch.qos.logback.core.status.OnConsoleStatusListener"/>-->
-    <!-- 解决 aliyun loghub debug模式下循环发送的问题 -->
-    <logger name="org.apache.http.impl.conn.Wire" level="WARN"/>
-    <!--aliyun loghub 为了防止进程退出时，内存中的数据丢失，请加上此选项-->
-    <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+<!--    &lt;!&ndash; 可用来获取 StatusManager 中的状态 &ndash;&gt;-->
+<!--    &lt;!&ndash;<statusListener class="ch.qos.logback.core.status.OnConsoleStatusListener"/>&ndash;&gt;-->
+<!--    &lt;!&ndash; 解决 aliyun loghub debug模式下循环发送的问题 &ndash;&gt;-->
+<!--    <logger name="org.apache.http.impl.conn.Wire" level="WARN"/>-->
+<!--    &lt;!&ndash;aliyun loghub 为了防止进程退出时，内存中的数据丢失，请加上此选项&ndash;&gt;-->
+<!--    <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>-->
 
-    <!--
-        root节点是必选节点，用来指定最基础的日志输出级别，只有一个level属性
-        level:用来设置打印级别，大小写无关：TRACE, DEBUG, INFO, WARN, ERROR, ALL 和 OFF，
-        不能设置为INHERITED或者同义词NULL。默认是DEBUG
-        可以包含零个或多个元素，标识这个appender将会添加到这个logger。
-    -->
-    <root level="info">
-<!--        <appender-ref ref="CONSOLE"/>-->
-<!--        <appender-ref ref="DEBUG_FILE"/>-->
-<!--        <appender-ref ref="INFO_FILE"/>-->
-<!--        <appender-ref ref="WARN_FILE"/>-->
-<!--        <appender-ref ref="ERROR_FILE"/>-->
-            <appender-ref ref="ASYNC_CONSOLE"/>
-            <appender-ref ref="ASYNC_DEBUG_FILE"/>
-            <appender-ref ref="ASYNC_INFO_FILE"/>
-            <appender-ref ref="ASYNC_WARN_FILE"/>
-            <appender-ref ref="ASYNC_ERROR_FILE"/>
-        <appender-ref ref="ALIYUN_LOG_INFO"/>
-        <appender-ref ref="ALIYUN_LOG_ERROR"/>
-    </root>
-</configuration>
+<!--    &lt;!&ndash;-->
+<!--        root节点是必选节点，用来指定最基础的日志输出级别，只有一个level属性-->
+<!--        level:用来设置打印级别，大小写无关：TRACE, DEBUG, INFO, WARN, ERROR, ALL 和 OFF，-->
+<!--        不能设置为INHERITED或者同义词NULL。默认是DEBUG-->
+<!--        可以包含零个或多个元素，标识这个appender将会添加到这个logger。-->
+<!--    &ndash;&gt;-->
+<!--    <root level="info">-->
+<!--&lt;!&ndash;        <appender-ref ref="CONSOLE"/>&ndash;&gt;-->
+<!--&lt;!&ndash;        <appender-ref ref="DEBUG_FILE"/>&ndash;&gt;-->
+<!--&lt;!&ndash;        <appender-ref ref="INFO_FILE"/>&ndash;&gt;-->
+<!--&lt;!&ndash;        <appender-ref ref="WARN_FILE"/>&ndash;&gt;-->
+<!--&lt;!&ndash;        <appender-ref ref="ERROR_FILE"/>&ndash;&gt;-->
+<!--            <appender-ref ref="ASYNC_CONSOLE"/>-->
+<!--            <appender-ref ref="ASYNC_DEBUG_FILE"/>-->
+<!--            <appender-ref ref="ASYNC_INFO_FILE"/>-->
+<!--            <appender-ref ref="ASYNC_WARN_FILE"/>-->
+<!--            <appender-ref ref="ASYNC_ERROR_FILE"/>-->
+<!--        <appender-ref ref="ALIYUN_LOG_INFO"/>-->
+<!--        <appender-ref ref="ALIYUN_LOG_ERROR"/>-->
+<!--    </root>-->
+<!--</configuration>-->

ad-engine-server/src/main/resources/logback.xml

@@ -0,0 +1,300 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 日志级别从低到高分为TRACE < DEBUG < INFO < WARN < ERROR < FATAL，如果设置为WARN，则低于WARN的信息都不会输出 -->
+<!-- scan:当此属性设置为true时，配置文件如果发生改变，将会被重新加载，默认值为true -->
+<!-- scanPeriod:设置监测配置文件是否有修改的时间间隔，如果没有给出时间单位，默认单位是毫秒。当scan为true时，此属性生效。默认的时间间隔为1分钟。 -->
+<!-- debug:当此属性设置为true时，将打印出logback内部日志信息，实时查看logback运行状态。默认值为false。 -->
+<configuration scan="true" scanPeriod="10 seconds">
+    <!--<include resource="org/springframework/boot/logging/logback/base.xml"/>-->
+
+    <contextName>logback</contextName>
+    <!-- name的值是变量的名称，value的值时变量定义的值。通过定义的值会被插入到logger上下文中。定义变量后，可以使“${}”来使用变量。 -->
+    <!-- <property name="LOG_PATH"  value="${logging.file.path}" />-->
+
+    <springProperty name="LOG_PATH" source="logging.file.path"/>
+    <springProperty name="ALIYUN_LOG_ENDPOINT" source="aliyun.log.endpoint"/>
+    <springProperty name="ALIYUN_LOG_ACCESSKEYID" source="aliyun.log.accessKeyId"/>
+    <springProperty name="ALIYUN_LOG_ACCESSKEYSECRET" source="aliyun.log.accessKeySecret"/>
+    <springProperty name="ALIYUN_LOG_PROJECT" source="aliyun.log.project"/>
+    <springProperty name="ALIYUN_LOG_LOGSTORE_INFO" source="aliyun.log.logstore.info"/>
+    <springProperty name="ALIYUN_LOG_LOGSTORE_ERROR" source="aliyun.log.logstore.error"/>
+
+    <!-- 彩色日志 -->
+    <!-- 彩色日志依赖的渲染类 -->
+    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
+    <conversionRule conversionWord="wex"
+                    converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
+    <conversionRule conversionWord="wEx"
+                    converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
+    <!-- 彩色日志格式 -->
+    <property name="CONSOLE_LOG_PATTERN"
+              value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr([%X{pqtId}]){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
+    <!--<property name="CONSOLE_LOG_PATTERN"-->
+    <!--          value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr([%X{√pqtId}]){magenta} %clr(-&#45;&#45;){faint} %clr([%15.15t]){faint} %clr(at %class.%method){cyan} \\(%file:%line\\) %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>-->
+
+
+    <!--输出到控制台-->
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <!--此日志appender是为开发使用，只配置最底级别，控制台输出的日志级别是大于或等于此级别的日志信息-->
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>debug</level>
+        </filter>
+        <encoder>
+            <Pattern>${CONSOLE_LOG_PATTERN}</Pattern>
+            <!-- 设置字符集 -->
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="ASYNC_CONSOLE" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="CONSOLE" />
+        <queueSize>10000</queueSize> <!-- 队列大小，可以根据需要调整 -->
+    </appender>
+    <!--输出到文件-->
+    <!-- 时间滚动输出 level为 DEBUG 日志 -->
+    <appender name="DEBUG_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${LOG_PATH}/debug.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- 日志归档 -->
+            <fileNamePattern>${LOG_PATH}/debug/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录debug级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>debug</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+    <appender name="ASYNC_DEBUG_FILE" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="DEBUG_FILE" />
+        <queueSize>1000</queueSize> <!-- 队列大小，可以根据需要调整 -->
+    </appender>
+    <!-- 时间滚动输出 level为 INFO 日志 -->
+    <appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${LOG_PATH}/info.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- 每天日志归档路径以及格式 -->
+            <fileNamePattern>${LOG_PATH}/info/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录info级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>info</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+    <appender name="ASYNC_INFO_FILE" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="INFO_FILE" />
+        <queueSize>10000</queueSize> <!-- 队列大小，可以根据需要调整 -->
+    </appender>
+
+    <!-- 时间滚动输出 level为 WARN 日志 -->
+    <appender name="WARN_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${LOG_PATH}/warn.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_PATH}/warn/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录warn级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>warn</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+    <appender name="ASYNC_WARN_FILE" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="WARN_FILE" />
+        <queueSize>3000</queueSize> <!-- 队列大小，可以根据需要调整 -->
+    </appender>
+
+    <!-- 时间滚动输出 level为 ERROR 日志 -->
+    <appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- 正在记录的日志文件的路径及文件名 -->
+        <file>${LOG_PATH}/error.log</file>
+        <!--日志文件输出格式-->
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
+        </encoder>
+        <!-- 日志记录器的滚动策略，按日期，按大小记录 -->
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_PATH}/error/%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <!--日志文件保留天数-->
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <!-- 此日志文件只记录ERROR级别的 -->
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>ERROR</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+    </appender>
+    <appender name="ASYNC_ERROR_FILE" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="ERROR_FILE" />
+        <queueSize>3000</queueSize> <!-- 队列大小，可以根据需要调整 -->
+    </appender>
+
+    <appender name="ALIYUN_LOG_INFO" class="com.aliyun.openservices.log.logback.LoghubAppender">
+        <endpoint>${ALIYUN_LOG_ENDPOINT}</endpoint>
+        <accessKeyId>${ALIYUN_LOG_ACCESSKEYID}</accessKeyId>
+        <accessKeySecret>${ALIYUN_LOG_ACCESSKEYSECRET}</accessKeySecret>
+        <project>${ALIYUN_LOG_PROJECT}</project>
+        <logStore>${ALIYUN_LOG_LOGSTORE_INFO}</logStore>
+
+        <totalSizeInBytes>104857600</totalSizeInBytes>
+        <maxBlockMs>0</maxBlockMs>
+        <ioThreadCount>8</ioThreadCount>
+        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>
+        <batchCountThreshold>4096</batchCountThreshold>
+        <lingerMs>2000</lingerMs>
+        <retries>10</retries>
+        <baseRetryBackoffMs>100</baseRetryBackoffMs>
+        <maxRetryBackoffMs>50000</maxRetryBackoffMs>
+
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+
+        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>
+        <timeZone>Asia/Shanghai</timeZone>
+
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>INFO</level>
+        </filter>
+
+        <mdcFields>pqtId</mdcFields>
+    </appender>
+
+    <appender name="ALIYUN_LOG_ERROR" class="com.aliyun.openservices.log.logback.LoghubAppender">
+        <endpoint>${ALIYUN_LOG_ENDPOINT}</endpoint>
+        <accessKeyId>${ALIYUN_LOG_ACCESSKEYID}</accessKeyId>
+        <accessKeySecret>${ALIYUN_LOG_ACCESSKEYSECRET}</accessKeySecret>
+        <project>${ALIYUN_LOG_PROJECT}</project>
+        <logStore>${ALIYUN_LOG_LOGSTORE_ERROR}</logStore>
+
+        <totalSizeInBytes>104857600</totalSizeInBytes>
+        <maxBlockMs>0</maxBlockMs>
+        <ioThreadCount>8</ioThreadCount>
+        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>
+        <batchCountThreshold>4096</batchCountThreshold>
+        <lingerMs>2000</lingerMs>
+        <retries>10</retries>
+        <baseRetryBackoffMs>100</baseRetryBackoffMs>
+        <maxRetryBackoffMs>50000</maxRetryBackoffMs>
+
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level [%X{pqtId}] %logger{50} [%L] - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+
+        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>
+        <timeZone>Asia/Shanghai</timeZone>
+
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>ERROR</level>
+        </filter>
+
+        <mdcFields>pqtId</mdcFields>
+    </appender>
+
+    <!--
+        <logger>用来设置某一个包或者具体的某一个类的日志打印级别、以及指定<appender>。
+        <logger>仅有一个name属性，一个可选的level和一个可选的addtivity属性。
+        name:用来指定受此logger约束的某一个包或者具体的某一个类。
+        level:用来设置打印级别，大小写无关：TRACE, DEBUG, INFO, WARN, ERROR, ALL 和 OFF，
+              还有一个特俗值INHERITED或者同义词NULL，代表强制执行上级的级别。
+              如果未设置此属性，那么当前logger将会继承上级的级别。
+        addtivity:是否向上级logger传递打印信息。默认是true。
+    -->
+    <!--<logger name="org.springframework.web" level="info"/>-->
+    <!--<logger name="org.springframework.scheduling.annotation.ScheduledAnnotationBeanPostProcessor" level="INFO"/>-->
+
+    <!--
+        使用mybatis的时候，sql语句是debug下才会打印，而这里我们只配置了info，所以想要查看sql语句的话，有以下两种操作：
+        第一种把<root level="info">改成<root level="DEBUG">这样就会打印sql，不过这样日志那边会出现很多其他消息
+        第二种就是单独给dao下目录配置 <logger> 指定 level 属性为 debug，这样配置sql语句会打印，其他还是正常info级别：
+     -->
+
+    <!--<springProfile name="dev">-->
+    <!--    <logger name="com.tzld.piaoquan.ad" level="debug"/>-->
+    <!--    <logger name="com.ctrip.framework.apollo.internals.RemoteConfigLongPollService" level="error"/>-->
+    <!--</springProfile>-->
+    <!--<springProfile name="test">-->
+    <!--    <logger name="com.tzld.piaoquan.ad" level="info"/>-->
+    <!--</springProfile>-->
+    <!--<springProfile name="pre">-->
+    <!--    <logger name="com.tzld.piaoquan.ad" level="info"/>-->
+    <!--</springProfile>-->
+    <!--<springProfile name="stress">-->
+    <!--    <logger name="com.tzld.piaoquan.ad" level="info"/>-->
+    <!--</springProfile>-->
+    <!--<springProfile name="prod">-->
+    <!--    <logger name="com.tzld.piaoquan.ad" level="info"/>-->
+    <!--</springProfile>-->
+    <!--<logger name="com.netflix.discovery.shared.resolver.aws.ConfigClusterResolver" level="warn"/>-->
+
+    <!-- 可用来获取 StatusManager 中的状态 -->
+    <!--<statusListener class="ch.qos.logback.core.status.OnConsoleStatusListener"/>-->
+    <!-- 解决 aliyun loghub debug模式下循环发送的问题 -->
+    <logger name="org.apache.http.impl.conn.Wire" level="WARN"/>
+    <!--aliyun loghub 为了防止进程退出时，内存中的数据丢失，请加上此选项-->
+    <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
+
+    <!--
+        root节点是必选节点，用来指定最基础的日志输出级别，只有一个level属性
+        level:用来设置打印级别，大小写无关：TRACE, DEBUG, INFO, WARN, ERROR, ALL 和 OFF，
+        不能设置为INHERITED或者同义词NULL。默认是DEBUG
+        可以包含零个或多个元素，标识这个appender将会添加到这个logger。
+    -->
+    <root level="info">
+<!--        <appender-ref ref="CONSOLE"/>-->
+<!--        <appender-ref ref="DEBUG_FILE"/>-->
+<!--        <appender-ref ref="INFO_FILE"/>-->
+<!--        <appender-ref ref="WARN_FILE"/>-->
+<!--        <appender-ref ref="ERROR_FILE"/>-->
+            <appender-ref ref="ASYNC_CONSOLE"/>
+            <appender-ref ref="ASYNC_DEBUG_FILE"/>
+            <appender-ref ref="ASYNC_INFO_FILE"/>
+            <appender-ref ref="ASYNC_WARN_FILE"/>
+            <appender-ref ref="ASYNC_ERROR_FILE"/>
+        <appender-ref ref="ALIYUN_LOG_INFO"/>
+        <appender-ref ref="ALIYUN_LOG_ERROR"/>
+    </root>
+</configuration>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/Feature.java

@@ -0,0 +1,22 @@
+package com.tzld.piaoquan.ad.engine.service.feature;
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+public class Feature {
+    // k1:创意、k2:表、k3:特征、v:特征值
+    private Map<String, Map<String, Map<String, String>>> cidFeature = new HashMap<>();
+
+    // k1:视频、k2:表、v:特征值
+    private Map<String, Map<String, String>> videoFeature = new HashMap<>();
+
+    // k1: 广告主ID、k2:表、k3:特征、v:特征值
+    private Map<String, Map<String, Map<String, String>>> adVerFeature = new HashMap<>();
+
+    // k1:mid、k2:表、v:特征值
+    private Map<String, Map<String, String>> userFeature = new HashMap<>();
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/FeatureService.java

@@ -0,0 +1,223 @@
+package com.tzld.piaoquan.ad.engine.service.feature;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import com.tzld.piaoquan.ad.engine.service.remote.FeatureV2RemoteService;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
+import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+@Slf4j
+@Component
+public class FeatureService {
+
+    @Autowired
+    private FeatureV2RemoteService remoteService;
+
+    private static final String cidUkFormat = "c:%s:%s";
+    private static final String cidUkDoubleFieldFormat = "c:%s:%s:%s";
+    private static final String avUkFormat = "av:%s:%s";
+    private static final String vidUkFormat = "v:%s:%s";
+    private static final String midUkFormat = "m:%s:%s";
+
+    public Feature getFeature(Collection<String> cidList, Collection<String> adVerIdList, ScoreParam param) {
+        AdRequestContext context = param.getRequestContext();
+
+        List<FeatureKeyProto> protos = new ArrayList<>();
+        for (String cidStr : cidList) {
+
+            // cid
+            protos.add(genWithCid("alg_cid_feature_basic_info", cidStr));
+            protos.add(genWithCid("alg_cid_feature_cid_action", cidStr));
+
+            // cid + region
+            protos.add(genWithCidAndRegion("alg_cid_feature_region_action", cidStr, context.getRegion()));
+
+            // cid + apptype
+            protos.add(genWithCidAndAppType("alg_cid_feature_app_action", cidStr, context.getApptype()));
+
+            // cid + week
+            protos.add(genWithCidAndWeek("alg_cid_feature_week_action", cidStr, context.getWeek()));
+
+            // cid + hour
+            protos.add(genWithCidAndHour("alg_cid_feature_hour_action", cidStr, context.getHour()));
+
+            // cid + brand
+            protos.add(genWithCidAndBrand("alg_cid_feature_brand_action", cidStr, context.getMachineinfoBrand()));
+
+            // cid + wechatversion
+            protos.add(genWithCidAndWechatVersion("alg_cid_feature_weChatVersion_action", cidStr, context.getMachineinfoWechatversion()));
+
+            // cid + vid
+            protos.add(genWithCidAndVid("alg_cid_feature_vid_cf", cidStr, param.getVideoId().toString()));
+        }
+
+        for (String adVerId : adVerIdList) {
+            // adverid
+            protos.add(genWithAdVerId("alg_cid_feature_adver_action", adVerId));
+        }
+
+        // vid
+        protos.add(genWithVid("alg_cid_feature_vid_cf_rank", param.getVideoId().toString()));
+
+        // mid
+        protos.add(genWithMid("alg_mid_feature_ad_action", param.getMid()));
+        protos.add(genWithMid("alg_mid_feature_return_tags", param.getMid()));
+        protos.add(genWithMid("alg_mid_feature_share_tags", param.getMid()));
+
+        Map<String, String> featureMap = remoteService.getFeature(protos);
+        featureMap = this.featureStrCover(featureMap);
+        Feature feature = new Feature();
+
+        for (Map.Entry<String, String> entry : featureMap.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+            String[] split = key.split(":");
+            String prefix = split[0];
+            String tableName = split[1];
+            String id = split[2];
+            switch (prefix) {
+                case "c":
+                    Map<String, Map<String, String>> tableFeatureMap = feature.getCidFeature().getOrDefault(id, new HashMap<>());
+                    tableFeatureMap.put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getCidFeature().put(id, tableFeatureMap);
+                    break;
+                case "av":
+                    Map<String, Map<String, String>> avFeatureMap = feature.getAdVerFeature().getOrDefault(id, new HashMap<>());
+                    avFeatureMap.put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getAdVerFeature().put(id, avFeatureMap);
+                    break;
+                case "m":
+                    feature.getUserFeature().put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    break;
+                case "v":
+                    feature.getVideoFeature().put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    break;
+            }
+        }
+
+        return feature;
+    }
+
+    private FeatureKeyProto genWithCid(String table, String cid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkFormat, table, cid))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithAdVerId(String table, String adVerId) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(avUkFormat, table, adVerId))
+                .setTableName(table)
+                .putFieldValue("adverid", adVerId)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndRegion(String table, String cid, String region) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, region))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("region", region)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndAppType(String table, String cid, String appType) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, appType))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("apptype", appType)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndWeek(String table, String cid, String week) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, week))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("week", week)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndHour(String table, String cid, String hour) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, hour))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("hour", hour)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndBrand(String table, String cid, String brand) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, brand))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("brand", brand)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndWechatVersion(String table, String cid, String wechatVersion) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, wechatVersion))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("wechatversion", wechatVersion)
+                .build();
+    }
+
+    private FeatureKeyProto genWithCidAndVid(String table, String cid, String vid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, vid))
+                .setTableName(table)
+                .putFieldValue("cid", cid)
+                .putFieldValue("vid", vid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithVid(String table, String vid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(vidUkFormat, table, vid))
+                .setTableName(table)
+                .putFieldValue("vid", vid)
+                .build();
+    }
+
+    private FeatureKeyProto genWithMid(String table, String mid) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(midUkFormat, table, mid))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .build();
+    }
+
+    private Map<String, String> featureStrCover(Map<String, String> metaFeatureMap) {
+        Map<String, String> newFeatureMap = new HashMap<>();
+        for (Map.Entry<String, String> entry : metaFeatureMap.entrySet()) {
+            String key = entry.getKey();
+            try {
+                Map<String, String> valueMap = JSONUtils.fromJson(entry.getValue(), new TypeToken<Map<String, String>>() {
+                }, Collections.emptyMap());
+                String value = valueMap.getOrDefault("feature", "{}");
+                newFeatureMap.put(key, value);
+            } catch (Exception e) {
+                log.error("featureStrCover error: ", e);
+            }
+        }
+        return newFeatureMap;
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -2,12 +2,14 @@ package com.tzld.piaoquan.ad.engine.service.log.impl;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Sets;
 import com.tzld.commons.aliyun.log.AliyunLogManager;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.service.score.param.RecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -34,16 +36,25 @@ public class LogHubServiceImpl implements LogHubService {
     @Autowired
     private AliyunLogManager aliyunLogManager;
 
+    private static final Set<String> FEATURE_FIELD_SET = Sets.newHashSet();
+
     @Override
     public void scoreLogUpload(ScoreParam param, List<AdPlatformCreativeDTO> adIdList, List<AdRankItem> rankItems, RecommendRequestParam requestParam, String scoreStrategy, String abCode) {
         logUploadThreadPool.execute(new Runnable() {
             @Override
             public void run() {
+
+                AdRequestContext context = param.getRequestContext();
+
                 JSONObject logMap = new JSONObject();
-                logMap.put("pqtId", param.getPqtId());
+                logMap.put("pqtid", param.getPqtId());
                 logMap.put("mid", param.getMid());
-                logMap.put("videoId", param.getVideoId());
-                logMap.put("abCode", abCode);
+                logMap.put("vid", param.getVideoId());
+                logMap.put("abcode", param.getAdAbGroup());
+                logMap.put("expid", abCode);
+                logMap.put("apptype", context.getApptype());
+                logMap.put("extinfo", new JSONObject());
+
                 // 获取AB实验列表
                 Set<String> abExpCode = new HashSet<>();
                 if (CollectionUtils.isNotEmpty(requestParam.getAdAbExpArr())) {
@@ -53,41 +64,55 @@ public class LogHubServiceImpl implements LogHubService {
                         }
                     }
                 }
-                logMap.put("abExpCode", abExpCode);
+                logMap.put("expids", abExpCode);
 
                 List<JSONObject> scoreResult = new ArrayList<>();
                 for (AdRankItem rankItem : rankItems) {
                     JSONObject json = new JSONObject();
-                    json.put("adId", rankItem.getAdId());
                     json.put("cid", rankItem.getAdId());
                     json.put("score", rankItem.getScore());
-                    json.put("ext", rankItem.getExt());
-                    json.put("weight", rankItem.getWeight());
+                    JSONObject featureJson = new JSONObject();
+                    for (Map.Entry<String, String> entry : rankItem.getFeatureMap().entrySet()) {
+                        if (FEATURE_FIELD_SET.contains(entry.getKey())) {
+                            featureJson.put(entry.getKey(), entry.getValue());
+                        }
+                    }
+                    for (Map.Entry<String, Map<String, String>> entry : rankItem.getMetaFeatureMap().entrySet()) {
+                        if (FEATURE_FIELD_SET.contains(entry.getKey())) {
+                            featureJson.put(entry.getKey(), entry.getValue());
+                        }
+                    }
+                    json.put("feature", featureJson);
                     scoreResult.add(json);
                 }
-                logMap.put("scoreResult", JSON.toJSONString(scoreResult));
+
+                JSONObject result = new JSONObject();
+                result.put("score", scoreResult);
+                logMap.put("result", JSON.toJSONString(result));
 
                 AdRankItem top1 = rankItems.get(0);
-                logMap.put("top1_adId", top1.getAdId());
-                logMap.put("top1_cid", top1.getAdId());
-                logMap.put("top1_score", top1.getScore());
-                logMap.put("top1_ext", JSON.toJSONString(top1.getExt()));
-                logMap.put("top1_weight", top1.getWeight());
-
-                logMap.put("creativeList", JSON.toJSONString(adIdList));
-                logMap.put("adAbGroup", requestParam.getAdAbGroup());
-                logMap.put("scoreStrategy", scoreStrategy);
-                logMap.put("appType", requestParam.getAppType());
-
-                if (Objects.nonNull(requestParam.getStatisticsLog())) {
-                    logMap.put("earlyAdIds", requestParam.getStatisticsLog().getEarlyAdIds());
-                    logMap.put("earlyCidList", requestParam.getStatisticsLog().getEarlyCreativeIds());
-                    logMap.put("finalCidList", requestParam.getStatisticsLog().getFinalCreativeIds());
-                    logMap.put("commonFilterAfterAdIds", requestParam.getStatisticsLog().getCommonFilterAfterAdIds());
-                    logMap.put("commonFilterAfterCidList", requestParam.getStatisticsLog().getCommonFilterAfterCreativeIds());
-                    logMap.put("tacticsFilterAfterAdIds", requestParam.getStatisticsLog().getTacticsFilterAfterAdIds());
-                    logMap.put("tacticsFilterAfterCidList", requestParam.getStatisticsLog().getTacticsFilterAfterCreativeIds());
-                }
+                logMap.put("cid", top1.getAdId());
+                logMap.put("adverid", top1.getAdVerId());
+                logMap.put("adid", top1.getId());
+                logMap.put("campaignid", top1.getCampaignId());
+                logMap.put("score", top1.getScore());
+                Map<String, String> featureMap = top1.getFeatureMap();
+                featureMap.put("weight", String.valueOf(top1.getWeight()));
+                logMap.put("allfeature", JSON.toJSONString(featureMap));
+                logMap.put("metafeature", JSON.toJSONString(top1.getMetaFeatureMap()));
+                logMap.put("scoremap", JSON.toJSONString(top1.getScoreMap()));
+
+                // logMap.put("scorestrategy", scoreStrategy);
+                // logMap.put("creativeList", JSON.toJSONString(adIdList));
+                // if (Objects.nonNull(requestParam.getStatisticsLog())) {
+                //     extInfo.put("earlyAdIds", requestParam.getStatisticsLog().getEarlyAdIds());
+                //     extInfo.put("earlyCidList", requestParam.getStatisticsLog().getEarlyCreativeIds());
+                //     extInfo.put("finalCidList", requestParam.getStatisticsLog().getFinalCreativeIds());
+                //     extInfo.put("commonFilterAfterAdIds", requestParam.getStatisticsLog().getCommonFilterAfterAdIds());
+                //     extInfo.put("commonFilterAfterCidList", requestParam.getStatisticsLog().getCommonFilterAfterCreativeIds());
+                //     extInfo.put("tacticsFilterAfterAdIds", requestParam.getStatisticsLog().getTacticsFilterAfterAdIds());
+                //     extInfo.put("tacticsFilterAfterCidList", requestParam.getStatisticsLog().getTacticsFilterAfterCreativeIds());
+                // }
 
                 aliyunLogManager.sendLog(project, logStore, "", logMap);
             }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/container/PredictPidContainer.java

@@ -65,17 +65,17 @@ public class PredictPidContainer {
 
     @PostConstruct
     private void init(){
-        instanceClient();
-        final Runnable task = new Runnable() {
-            public void run() {
-                try {
-                    loadAndCalIfNeed();
-                }catch (Exception e){
-                    e.printStackTrace();
-                }
-            }
-        };
-        scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
+        // instanceClient();
+        // final Runnable task = new Runnable() {
+        //     public void run() {
+        //         try {
+        //             loadAndCalIfNeed();
+        //         }catch (Exception e){
+        //             e.printStackTrace();
+        //         }
+        //     }
+        // };
+        // scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
     }
 
     private void instanceClient(){

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/container/TopOneVideoContainer.java

@@ -36,17 +36,17 @@ public class TopOneVideoContainer {
     @PostConstruct
     public void init(){
 //        refreshTopVideoIdCache();
-        timer = new Timer();
-        timer.schedule(new TimerTask() {
-            @Override
-            public void run() {
-                try {
-                    refreshTopVideoIdCache();
-                }catch (Exception e){
-                    log.error("svc=timerTask taskName=refreshTopVideoIdCache e={}",Arrays.toString(e.getStackTrace()));
-                }
-            }
-        }, 0);
+//         timer = new Timer();
+//         timer.schedule(new TimerTask() {
+//             @Override
+//             public void run() {
+//                 try {
+//                     refreshTopVideoIdCache();
+//                 }catch (Exception e){
+//                     log.error("svc=timerTask taskName=refreshTopVideoIdCache e={}",Arrays.toString(e.getStackTrace()));
+//                 }
+//             }
+//         }, 0);
 
     }
     public void refreshTopVideoIdCache(){

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/remote/FeatureV2RemoteService.java

@@ -0,0 +1,32 @@
+package com.tzld.piaoquan.ad.engine.service.remote;
+
+import com.tzld.piaoquan.recommend.feature.client.FeatureV2Client;
+import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * @author dyp
+ */
+@Component
+@Slf4j
+public class FeatureV2RemoteService {
+
+    @Autowired
+    private FeatureV2Client client;
+
+    public Map<String, String> getFeature(List<FeatureKeyProto> protos) {
+        if (CollectionUtils.isEmpty(protos)) {
+            return Collections.emptyMap();
+        }
+        return client.multiGetFeature(protos);
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService.java

@@ -12,10 +12,12 @@ import java.util.List;
 public interface RankService {
 
     AdRankItem adItemRank(RankRecommendRequestParam request);
+
     AdRankItem adItemRankWithVideoAdThompson(RankRecommendRequestParam request);
+
     AdPlatformCreativeDTO adBidRank(BidRankRecommendRequestParam request);
 
     AdPlatformCreativeDTO adBidRankNewPid(BidRankRecommendRequestParam request);
 
-    List<AdRankItem> rank (ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems, String configFile);
+    List<AdRankItem> rank(ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems, String configFile);
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService680.java

@@ -0,0 +1,530 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.commons.util.ExtractorUtils;
+import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
+import com.tzld.piaoquan.ad.engine.service.feature.Feature;
+import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
+import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+public class RankService680 {
+
+    @Autowired
+    private FeatureService featureService;
+
+    private Map<String, double[]> bucketsMap = new HashMap<>();
+
+    private Map<String, Double> bucketsLen = new HashMap<>();
+
+    public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+
+        long ts = System.currentTimeMillis() / 1000;
+
+
+        // 特征处理
+        Feature feature = this.getFeature(scoreParam, request);
+        Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+        Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
+        Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
+        Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+
+        Map<String, String> userFeatureMap = new HashMap<>();
+        Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
+        List<TupleMapEntry<Tuple5>> midActionList = this.handleC1Feature(c1Feature, userFeatureMap);
+
+        Map<String, Double> midTimeDiffMap = this.parseC1FeatureListToTimeDiffMap(midActionList, ts);
+        Map<String, Double> actionStaticMap = this.parseC1FeatureListToActionStaticMap(midActionList);
+
+        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
+
+        Map<String, Map<String, Double>> vidRankMaps = this.parseD2FeatureMap(d2Feature);
+
+        Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+
+
+        List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
+        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+            AdRankItem adRankItem = new AdRankItem();
+            adRankItem.setAdId(dto.getCreativeId());
+            adRankItem.setCreativeCode(dto.getCreativeCode());
+            adRankItem.setAdVerId(dto.getAdVerId());
+            adRankItem.setVideoId(request.getVideoId());
+            adRankItem.setCpa(dto.getCpa());
+            adRankItem.setId(dto.getAdId());
+            adRankItem.setCampaignId(dto.getCampaignId());
+
+            String cidStr = dto.getCreativeId().toString();
+            Map<String, String> cidFeatureMap = new HashMap<>();
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+            Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+
+            this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+
+            this.handleB2ToB5AndB8Feature(cidFeature, adVerFeature, cidFeatureMap);
+
+            this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+
+            this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+
+            this.handleD1Feature(d1Feature, cidFeatureMap);
+
+            this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+
+            String title = b1Feature.getOrDefault("cidtitle", "");
+            this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
+
+            adRankItem.setFeatureMap(cidFeatureMap);
+
+            adRankItems.add(adRankItem);
+
+        }
+
+        // 分桶
+        this.readBucketFile();
+        userFeatureMap = this.featureBucket(userFeatureMap);
+        for (AdRankItem adRankItem : adRankItems) {
+            Map<String, String> featureMap = adRankItem.getFeatureMap();
+            adRankItem.setFeatureMap(this.featureBucket(featureMap));
+        }
+
+        // 打分排序
+        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.LR_ROV_SCORE_20240626)
+                .scoring(new HashMap<>(), userFeatureMap, adRankItems);
+
+        for (AdRankItem item : result) {
+            item.setScore(item.getLrScore() * item.getCpa());
+            item.getFeatureMap().putAll(userFeatureMap);
+            if (MapUtils.isNotEmpty(videoFeature)) {
+                item.getMetaFeatureMap().putAll(videoFeature);
+            }
+            if (MapUtils.isNotEmpty(userFeature)) {
+                item.getMetaFeatureMap().putAll(userFeature);
+            }
+            if (allAdVerFeature.containsKey(item.getAdVerId())) {
+                item.getMetaFeatureMap().putAll(allAdVerFeature.get(item.getAdVerId()));
+            }
+            if (allCidFeature.containsKey(String.valueOf(item.getAdId()))) {
+                item.getMetaFeatureMap().putAll(allCidFeature.get(String.valueOf(item.getAdId())));
+            }
+        }
+
+        Collections.sort(result);
+
+        return result;
+    }
+
+    private Feature getFeature(ScoreParam param, RankRecommendRequestParam request) {
+        List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
+        List<String> cidList = adIdList.stream()
+                .map(AdPlatformCreativeDTO::getCreativeId)
+                .map(Object::toString)
+                .collect(Collectors.toList());
+
+        List<String> adVerIdList = adIdList.stream()
+                .map(AdPlatformCreativeDTO::getAdVerId)
+                .distinct()
+                .collect(Collectors.toList());
+        return featureService.getFeature(cidList, adVerIdList, param);
+    }
+
+    private void handleB1Feature(Map<String, String> b1Feature, Map<String, String> cidFeatureMap, String cid) {
+        cidFeatureMap.put("cid_" + cid, "1");
+        if (StringUtils.isNotBlank(b1Feature.get("adid"))) {
+            String adId = b1Feature.get("adid");
+            cidFeatureMap.put("adid_" + adId, "1");
+        }
+        if (StringUtils.isNotBlank(b1Feature.get("adverid"))) {
+            String adVerId = b1Feature.get("adverid");
+            cidFeatureMap.put("adverid_" + adVerId, "1");
+        }
+        if (StringUtils.isNotBlank(b1Feature.get("targeting_conversion"))) {
+            String targetingConversion = b1Feature.get("targeting_conversion");
+            cidFeatureMap.put("targeting_conversion_" + targetingConversion, "1");
+        }
+        if (StringUtils.isNotBlank(b1Feature.get("cpa"))) {
+            String cpa = b1Feature.get("cpa");
+            cidFeatureMap.put("cpa", cpa);
+        }
+    }
+
+    private void handleB2ToB5AndB8Feature(Map<String, Map<String, String>> c1Feature, Map<String, Map<String, String>> adVerFeature, Map<String, String> cidFeatureMap) {
+        Map<String, String> b2Feature = adVerFeature.getOrDefault("alg_cid_feature_adver_action", new HashMap<>());
+        Map<String, String> b3Feature = c1Feature.getOrDefault("alg_cid_feature_cid_action", new HashMap<>());
+        Map<String, String> b4Feature = c1Feature.getOrDefault("alg_cid_feature_region_action", new HashMap<>());
+        Map<String, String> b5Feature = c1Feature.getOrDefault("alg_cid_feature_app_action", new HashMap<>());
+        Map<String, String> b8Feature = c1Feature.getOrDefault("alg_cid_feature_brand_action", new HashMap<>());
+
+        List<String> timeList = Arrays.asList("3h", "6h", "12h", "1d", "3d", "7d");
+        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
+                new Tuple2<>(b2Feature, "b2"),
+                new Tuple2<>(b3Feature, "b3"),
+                new Tuple2<>(b4Feature, "b4"),
+                new Tuple2<>(b5Feature, "b5"),
+                new Tuple2<>(b8Feature, "b8")
+        );
+        for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
+            Map<String, String> feature = tuple2.f1;
+            String prefix = tuple2.f2;
+            for (String time : timeList) {
+                double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
+                double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
+                double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            }
+        }
+
+    }
+
+    private void handleB6ToB7Feature(Map<String, Map<String, String>> c1Feature, Map<String, String> cidFeatureMap) {
+        Map<String, String> b6Feature = c1Feature.getOrDefault("alg_cid_feature_week_action", new HashMap<>());
+        Map<String, String> b7Feature = c1Feature.getOrDefault("alg_cid_feature_hour_action", new HashMap<>());
+
+        List<String> timeList = Arrays.asList("7d", "14d");
+        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
+                new Tuple2<>(b6Feature, "b6"),
+                new Tuple2<>(b7Feature, "b7")
+        );
+        for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
+            Map<String, String> feature = tuple2.f1;
+            String prefix = tuple2.f2;
+            for (String time : timeList) {
+                double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
+                double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
+                double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
+                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+                cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
+                cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+            }
+        }
+
+    }
+
+    private List<TupleMapEntry<Tuple5>> handleC1Feature(Map<String, String> c1Feature, Map<String, String> featureMap) {
+
+        // 用户特征
+        List<TupleMapEntry<Tuple5>> midActionList = new ArrayList<>();
+        if (c1Feature.containsKey("action")) {
+            String action = c1Feature.get("action");
+            midActionList = Arrays.stream(action.split(","))
+                    .map(r -> {
+                        String[] rList = r.split(":");
+                        Tuple5 tuple5 = new Tuple5(rList[1], rList[2], rList[3], rList[4], rList[5]);
+                        return new TupleMapEntry<>(rList[0], tuple5);
+                    })
+                    // TODO 倒排
+                    .sorted((a, b) -> Integer.compare(Integer.parseInt(b.value.f1), Integer.parseInt(a.value.f1)))
+                    .collect(Collectors.toList());
+        }
+
+        double viewAll = midActionList.size();
+        double clickAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f2)).sum();
+        double converAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f3)).sum();
+        double incomeAll = midActionList.stream().mapToInt(e -> Integer.parseInt(e.value.f4)).sum();
+        featureMap.put("viewAll", String.valueOf(viewAll));
+        featureMap.put("clickAll", String.valueOf(clickAll));
+        featureMap.put("converAll", String.valueOf(converAll));
+        featureMap.put("incomeAll", String.valueOf(incomeAll));
+        featureMap.put("ctr_all", String.valueOf(NumUtil.div(clickAll, viewAll)));
+        featureMap.put("ctcvr_all", String.valueOf(NumUtil.div(converAll, viewAll)));
+        featureMap.put("cvr_all", String.valueOf(NumUtil.div(clickAll, converAll)));
+        featureMap.put("ecpm_all", String.valueOf(NumUtil.div(incomeAll * 1000, viewAll)));
+
+        return midActionList;
+    }
+
+    private void handleC1UIFeature(Map<String, Double> midTimeDiffMap, Map<String, Double> midActionStatic, Map<String, String> featureMap, String cid) {
+        if (midTimeDiffMap.containsKey("timediff_view_" + cid)) {
+            featureMap.put("timediff_view", String.valueOf(midTimeDiffMap.getOrDefault("timediff_view_" + cid, 0.0)));
+        }
+        if (midTimeDiffMap.containsKey("timediff_click_" + cid)) {
+            featureMap.put("timediff_click", String.valueOf(midTimeDiffMap.getOrDefault("timediff_click_" + cid, 0.0)));
+        }
+        if (midTimeDiffMap.containsKey("timediff_conver_" + cid)) {
+            featureMap.put("timediff_conver", String.valueOf(midTimeDiffMap.getOrDefault("timediff_conver_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_view_" + cid)) {
+            featureMap.put("actionstatic_view", String.valueOf(midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_click_" + cid)) {
+            featureMap.put("actionstatic_click", String.valueOf(midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_conver_" + cid)) {
+            featureMap.put("actionstatic_conver", String.valueOf(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_income_" + cid)) {
+            featureMap.put("actionstatic_income", String.valueOf(midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0)));
+        }
+        if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
+            double ctr = NumUtil.div(
+                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+            );
+            featureMap.put("actionstatic_ctr", String.valueOf(ctr));
+        }
+        if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("timediff_conver_" + cid)) {
+            double ctcvr = NumUtil.div(
+                    midActionStatic.getOrDefault("timediff_conver_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+            );
+            featureMap.put("actionstatic_ctcvr", String.valueOf(ctcvr));
+        }
+        if (midActionStatic.containsKey("actionstatic_conver_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
+            double cvr = NumUtil.div(
+                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
+                    midActionStatic.getOrDefault("timediff_conver_" + cid, 0.0)
+            );
+            featureMap.put("actionstatic_cvr", String.valueOf(cvr));
+        }
+    }
+
+    private void handleD1Feature(Map<String, String> d1Feature, Map<String, String> featureMap) {
+        for (String prefix : Arrays.asList("3h", "6h", "12h", "1d", "3d", "7d")) {
+            double view = Double.parseDouble(d1Feature.getOrDefault("ad_view_" + prefix, "0"));
+            double click = Double.parseDouble(d1Feature.getOrDefault("ad_click_" + prefix, "0"));
+            double conver = Double.parseDouble(d1Feature.getOrDefault("ad_conversion_" + prefix, "0"));
+            double income = Double.parseDouble(d1Feature.getOrDefault("ad_income_" + prefix, "0"));
+            featureMap.put("d1_feature_" + prefix + "_ctr", String.valueOf(NumUtil.div(click, view)));
+            featureMap.put("d1_feature_" + prefix + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
+            featureMap.put("d1_feature_" + prefix + "_cvr", String.valueOf(NumUtil.div(conver, click)));
+            featureMap.put("d1_feature_" + prefix + "_conver", String.valueOf(conver));
+            featureMap.put("d1_feature_" + prefix + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+        }
+    }
+
+    private void handleD2Feature(Map<String, Map<String, Double>> vidRankMaps, Map<String, String> featureMap, String cid) {
+        if (MapUtils.isEmpty(vidRankMaps)) {
+            return;
+        }
+
+        List<String> prefixes1 = Arrays.asList("ctr", "ctcvr", "ecpm");
+        List<String> prefixes2 = Arrays.asList("1d", "3d", "7d", "14d");
+
+        for (String prefix1 : prefixes1) {
+            for (String prefix2 : prefixes2) {
+                String combinedKey = prefix1 + "_" + prefix2;
+                if (vidRankMaps.containsKey(combinedKey)) {
+                    Double rank = vidRankMaps.get(combinedKey).getOrDefault(cid, 0.0);
+                    if (rank >= 1.0) {
+                        featureMap.put("vid_rank_" + combinedKey, String.valueOf(NumUtil.div(1, rank)));
+                    }
+                }
+            }
+        }
+    }
+
+    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title, Map<String, String> featureMap) {
+        if (StringUtils.isEmpty(title)) {
+            return;
+        }
+
+        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(
+                new Tuple2<>(e1Feature, "e1"),
+                new Tuple2<>(e2Feature, "e2")
+        );
+
+        List<String> tagsFieldList = Arrays.asList("tags_3d", "tags_7d", "tags_14d");
+        for (Tuple2<Map<String, String>, String> tuple2 : tuple2List) {
+            Map<String, String> feature = tuple2.f1;
+            String prefix = tuple2.f2;
+            if (MapUtils.isEmpty(feature)) {
+                continue;
+            }
+
+            for (String tagsField : tagsFieldList) {
+                if (StringUtils.isNotEmpty(feature.get(tagsField))) {
+                    String tags = feature.get(tagsField);
+                    Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    featureMap.put(prefix + "_" + tagsField + "_matchnum", String.valueOf(doubles[0]));
+                    featureMap.put(prefix + "_" + tagsField + "_maxscore", String.valueOf(doubles[1]));
+                    featureMap.put(prefix + "_" + tagsField + "_avgscore", String.valueOf(doubles[2]));
+                }
+            }
+        }
+    }
+
+    private Map<String, Double> parseC1FeatureListToTimeDiffMap(List<TupleMapEntry<Tuple5>> midActionList, long ts) {
+        Map<String, Double> midTimeDiffMap = new HashMap<>();
+        for (TupleMapEntry<Tuple5> entry : midActionList) {
+            String cid = entry.key;
+            double tsHistory = Double.parseDouble(entry.value.f1);
+            double click = Double.parseDouble(entry.value.f2);
+            double conver = Double.parseDouble(entry.value.f3);
+            double d = (ts - tsHistory) / 3600 / 24;
+            if (!midTimeDiffMap.containsKey("timediff_view_" + cid)) {
+                midTimeDiffMap.put("timediff_view_" + cid, NumUtil.div(1, d));
+            }
+            if (!midTimeDiffMap.containsKey("timediff_click_" + cid) && click > 0) {
+                midTimeDiffMap.put("timediff_click_" + cid, NumUtil.div(1, d));
+            }
+            if (!midTimeDiffMap.containsKey("timediff_conver_" + cid) && conver > 0) {
+                midTimeDiffMap.put("timediff_conver_" + cid, NumUtil.div(1, d));
+            }
+        }
+        return midTimeDiffMap;
+    }
+
+    private Map<String, Double> parseC1FeatureListToActionStaticMap(List<TupleMapEntry<Tuple5>> midActionList) {
+        Map<String, Double> midActionStaticsMap = new HashMap<>();
+        for (TupleMapEntry<Tuple5> entry : midActionList) {
+            String cid = entry.key;
+            double click = Double.parseDouble(entry.value.f2);
+            double conver = Double.parseDouble(entry.value.f3);
+            double income = Double.parseDouble(entry.value.f4);
+
+            Double viewSum = midActionStaticsMap.getOrDefault("actionstatic_view_" + cid, 0.0);
+            midActionStaticsMap.put("actionstatic_view_" + cid, 1 + viewSum);
+
+            Double clickSum = midActionStaticsMap.getOrDefault("actionstatic_click_" + cid, 0.0);
+            midActionStaticsMap.put("actionstatic_click_" + cid, clickSum + click);
+
+            Double converSum = midActionStaticsMap.getOrDefault("actionstatic_conver_" + cid, 0.0);
+            midActionStaticsMap.put("actionstatic_conver_" + cid, converSum + conver);
+
+            Double incomSum = midActionStaticsMap.getOrDefault("actionstatic_income_" + cid, 0.0);
+            midActionStaticsMap.put("actionstatic_income_" + cid, incomSum + income);
+        }
+
+        return midActionStaticsMap;
+    }
+
+    private Map<String, Map<String, Double>> parseD2FeatureMap(Map<String, String> d2Feature) {
+        Map<String, Map<String, Double>> vidRankMaps = new HashMap<>();
+        for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+            Map<String, Double> valueMap = Arrays.stream(value.split(","))
+                    .map(r -> r.split(":"))
+                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
+            vidRankMaps.put(key, valueMap);
+        }
+        return vidRankMaps;
+    }
+
+    private void readBucketFile() {
+        if (MapUtils.isNotEmpty(bucketsMap)) {
+            return;
+        }
+        synchronized (this) {
+            InputStream resourceStream = RankService680.class.getClassLoader().getResourceAsStream("20240622_ad_bucket_249.txt");
+            if (resourceStream != null) {
+                try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
+                    Map<String, double[]> bucketsMap = new HashMap<>();
+                    Map<String, Double> bucketsLen = new HashMap<>();
+                    String line;
+                    while ((line = reader.readLine()) != null) {
+                        // 替换空格和换行符，过滤空行
+                        line = line.replace(" ", "").replaceAll("\n", "");
+                        if (!line.isEmpty()) {
+                            String[] rList = line.split("\t");
+                            if (rList.length == 3) {
+                                String key = rList[0];
+                                double value1 = Double.parseDouble(rList[1]);
+                                bucketsLen.put(key, value1);
+                                double[] value2 = Arrays.stream(rList[2].split(","))
+                                        .mapToDouble(Double::valueOf)
+                                        .toArray();
+                                bucketsMap.put(key, value2);
+                            }
+                        }
+                    }
+                    this.bucketsMap = bucketsMap;
+                    this.bucketsLen = bucketsLen;
+                } catch (IOException e) {
+                    log.error("something is wrong in parse bucket file:" + e);
+                }
+            } else {
+                log.error("no bucket file");
+            }
+        }
+    }
+
+    private Map<String, String> featureBucket(Map<String, String> featureMap) {
+        Map<String, String> newFeatureMap = new HashMap<>(featureMap.size());
+        for (Map.Entry<String, String> entry : featureMap.entrySet()) {
+            String name = entry.getKey();
+            double score = Double.parseDouble(entry.getValue());
+            // 注意：0值、不在分桶文件中的特征，会被过滤掉。
+            if (score > 1E-8) {
+                if (this.bucketsMap.containsKey(name)) {
+                    double[] buckets = this.bucketsMap.get(name);
+                    int bucketNum = buckets.length + 1;
+                    Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
+                    newFeatureMap.put(name, String.valueOf(scoreNew));
+                } else {
+                    newFeatureMap.put(name, String.valueOf(score));
+                }
+            }
+        }
+
+        return newFeatureMap;
+    }
+
+    public static class Tuple5 {
+        public String f1;
+        public String f2;
+        public String f3;
+        public String f4;
+        public String f5;
+
+        public Tuple5(String f1, String f2, String f3, String f4, String f5) {
+            this.f1 = f1;
+            this.f2 = f2;
+            this.f3 = f3;
+            this.f4 = f4;
+            this.f5 = f5;
+        }
+    }
+
+    public static class TupleMapEntry<T> {
+        public String key;
+        public T value;
+
+        public TupleMapEntry(String key, T value) {
+            this.key = key;
+            this.value = value;
+        }
+    }
+
+    public static class Tuple2<F1, F2> {
+        public F1 f1;
+
+        public F2 f2;
+
+        public Tuple2(F1 first, F2 name) {
+            this.f1 = first;
+            this.f2 = name;
+        }
+
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VideoAdThompsonScorerV2.java

@@ -43,52 +43,6 @@ public class VideoAdThompsonScorerV2 {
     Random random = new Random();
     Gson gson = new Gson();
 
-    // public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
-    //     List<AdRankItem> result = new LinkedList<>();
-    //     String jsonStr;
-    //     CreativeStatistic statistic = null;
-    //     List<String> redisKey = new LinkedList<>();
-    //     adIdList.forEach(creativeDTO -> {
-    //         redisKey.add(redisCreativeStatisticsPrefix + creativeDTO.getCreativeId());
-    //     });
-    //     List<String> values = redisHelper.getValues(redisKey);
-    //     double score = 0d;
-    //     int i = 0;
-    //     for (AdPlatformCreativeDTO dto : adIdList) {
-    //         AdRankItem item = new AdRankItem();
-    //         item.setVideoId(param.getVideoId());
-    //         item.setCpa(dto.getCpa());
-    //         item.setAdId(dto.getCreativeId());
-    //         try {
-    //             jsonStr = values.get(i);
-    //             if (jsonStr == null) {
-    //                 score = exp663Param.getOrDefault("randomMin", 0.000001)
-    //                         + (random.nextDouble() *
-    //                         (exp663Param.getOrDefault("randomMax", 0.00001) - exp663Param.getOrDefault("randomMin", 0.000001)));
-    //             } else {
-    //                 statistic = gson.fromJson(jsonStr, CreativeStatistic.class);
-    //                 score = (exp663Param.getOrDefault("alpha", 1d) + Long.parseLong(statistic.getOrder()))
-    //                         / (exp663Param.getOrDefault("beta", 10000d) + Long.parseLong(statistic.getExp()));
-    //             }
-    //             double s1 = score;
-    //             score = score * dto.getCpa() * dto.getBid1() * dto.getBid2();
-    //
-    //             Map<String, Object> ext = this.extMap(statistic, "663", dto.getCpa(), null,
-    //                     exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
-    //             ext.put("s1", s1);
-    //             item.setExt(ext);
-    //             item.setScore(score);
-    //             item.setScore_type(663);
-    //         } catch (Exception e) {
-    //             log.error("svc=thompsonScorerByExp663 {}", gson.toJson(e.getStackTrace()));
-    //         }
-    //         result.add(item);
-    //         i++;
-    //     }
-    //     Collections.sort(result);
-    //     return result;
-    // }
-
     public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
         List<AdRankItem> result = new LinkedList<>();
         Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(redisCreativeStatisticsPrefix, adIdList);
@@ -244,75 +198,6 @@ public class VideoAdThompsonScorerV2 {
         Collections.sort(result);
         return result;
     }
-    // public List<AdRankItem> thompsonScorerByExp666(ScoreParam param,List<AdPlatformCreativeDTO> adIdList){
-    //     List<AdRankItem> result=new LinkedList<>();
-    //     String jsonStr;
-    //     CreativeStatistic statistic = null;
-    //
-    //     List<String> redisKey=new LinkedList<>();
-    //     adIdList.forEach(creativeDTO -> {
-    //         redisKey.add(redisCreativeStatisticsPrefix+creativeDTO.getCreativeId());
-    //     });
-    //     List<String> values=redisHelper.getValues(redisKey);
-    //     List<String> combineKeys=new LinkedList<>();
-    //     adIdList.forEach(creativeDTO -> {
-    //         combineKeys.add(redisVideoCreativeStatisticsPrefix+param.getVideoId()+"_"+creativeDTO.getCreativeId());
-    //     });
-    //     List<String> combineValues=redisHelper.getValues(combineKeys);
-    //     int i=0;
-    //     double score=0d;
-    //     for(AdPlatformCreativeDTO dto:adIdList){
-    //         String statisticType = "vid+cid";
-    //         AdRankItem item=new AdRankItem();
-    //         item.setVideoId(param.getVideoId());
-    //         item.setCpa(dto.getCpa());
-    //         item.setAdId(dto.getCreativeId());
-    //         try {
-    //             jsonStr=combineValues.get(i);
-    //             if(jsonStr==null){
-    //                 jsonStr=values.get(i);
-    //                 if(jsonStr==null){
-    //                     statisticType = "";
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
-    //                 }else {
-    //                     statisticType = "cid";
-    //                     statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-    //                 }
-    //             }else {
-    //                 statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-    //                 if(Double.parseDouble(statistic.getExp())>exp666Param.getOrDefault("viewThreshold",5000d)){
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-    //                 } else if( values.get(i)!=null) {
-    //                     statistic =gson.fromJson(values.get(i),CreativeStatistic.class);
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-    //                 }else {
-    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
-    //                 }
-    //             }
-    //             double s1 = score;
-    //             score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
-    //             item.setScore(score);
-    //             item.setScore_type(666);
-    //
-    //             Map<String, Object> ext = this.extMap(statistic, "666", dto.getCpa(), exp666Param.getOrDefault("viewThreshold",5000d), exp663Param.getOrDefault("alpha", 1d),
-    //                     exp663Param.getOrDefault("beta", 10000d), exp666Param.getOrDefault("beta_k",9d));
-    //             ext.put("s1", s1);
-    //             ext.put("group_field", statisticType);
-    //             item.setExt(ext);
-    //
-    //         }catch (Exception e){
-    //             log.error("svc=thompsonScorerByExp666 {}",gson.toJson(e.getStackTrace()));
-    //         }
-    //         result.add(item);
-    //         i++;
-    //     }
-    //     Collections.sort(result);
-    //     return result;
-    // }
 
     public List<AdRankItem> thompsonScorerByExp666(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
         List<AdRankItem> result = new LinkedList<>();

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCtrLRScorer.java

@@ -19,6 +19,7 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.*;
 
 
@@ -62,6 +63,11 @@ public class VlogAdCtrLRScorer extends BaseLRModelScorer {
         return result;
     }
 
+    @Override
+    public List<AdRankItem> scoring(Map<String, String> sceneFeatureMap, Map<String, String> userFeatureMap, List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
     private List<AdRankItem> rankByJava(final List<AdRankItem> items,
                                         final AdRequestContext requestContext,
                                         final UserAdFeature user) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCvrLRAdjustingScorer.java

@@ -9,13 +9,12 @@ import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.NotImplementedException;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.concurrent.*;
 
 
@@ -52,6 +51,11 @@ public class VlogAdCvrLRAdjustingScorer extends AbstractScorer {
         return result;
     }
 
+    @Override
+    public List<AdRankItem> scoring(Map<String, String> sceneFeatureMap, Map<String, String> userFeatureMap, List<AdRankItem> rankItems) {
+        throw new NotImplementedException();
+    }
+
     private List<AdRankItem> rankByJava(final List<AdRankItem> items,
                                         final AdRequestContext requestContext,
                                         final UserAdFeature user) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogRovFMScorer.java

@@ -0,0 +1,160 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseFMModelScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.FMModel;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+
+public class VlogRovFMScorer extends BaseFMModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogRovFMScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public VlogRovFMScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userAdFeature,
+                                    final List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    final List<AdRankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        FMModel model = (FMModel) this.getModel();
+
+        List<AdRankItem> result = rankByJava(sceneFeatureMap, userFeatureMap, rankItems);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result != null ? result.size() : 0,
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                      final Map<String, String> userFeatureMap,
+                                      final List<AdRankItem> items) {
+        long startTime = System.currentTimeMillis();
+        FMModel model = (FMModel) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (int i = 0; i < items.size(); i++) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", items.get(i), items.get(i));
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("ctr ranker java execute time: [{}]", System.currentTimeMillis() - startTime);
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items != null ? items.size() : 0,
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final FMModel model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex), ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // 等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException {},{}", ExceptionUtils.getFullStackTrace(e));
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final FMModel model,
+                            final AdRankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        // Map<String, String> featureMap = new HashMap<>();
+        // if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+        //     featureMap.putAll(item.getFeatureMap());
+        // }
+        // if (MapUtils.isNotEmpty(userFeatureMap)) {
+        //     featureMap.putAll(userFeatureMap);
+        // }
+        // if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+        //     featureMap.putAll(sceneFeatureMap);
+        // }
+
+        double pro = 0.0;
+        // if (MapUtils.isNotEmpty(featureMap)) {
+        //     try {
+        //         pro = model.score(featureMap);
+        //         // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+        //     } catch (Exception e) {
+        //         LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+        //     }
+        // }
+        // item.setScoreRov(pro);
+        // item.getScoresMap().put("RovFMScore", pro);
+        // item.setAllFeatureMap(featureMap);
+        return pro;
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogRovLRScorer.java

@@ -0,0 +1,155 @@
+package com.tzld.piaoquan.ad.engine.service.score;
+
+
+import com.tzld.piaoquan.ad.engine.commons.score.BaseLRV2ModelScorer;
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerConfigInfo;
+import com.tzld.piaoquan.ad.engine.commons.score.model.LRV2Model;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+
+public class VlogRovLRScorer extends BaseLRV2ModelScorer {
+
+    private static final int LOCAL_TIME_OUT = 150;
+    private final static Logger LOGGER = LoggerFactory.getLogger(VlogRovLRScorer.class);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+
+
+    public VlogRovLRScorer(ScorerConfigInfo configInfo) {
+        super(configInfo);
+    }
+
+    @Override
+    public List<AdRankItem> scoring(final ScoreParam param,
+                                    final UserAdFeature userFeature,
+                                    final List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    final List<AdRankItem> rankItems) {
+        if (CollectionUtils.isEmpty(rankItems)) {
+            return rankItems;
+        }
+
+        long startTime = System.currentTimeMillis();
+        LRV2Model model = (LRV2Model) this.getModel();
+        LOGGER.debug("model size: [{}]", model.getModelSize());
+
+        List<AdRankItem> result = rankByJava(sceneFeatureMap, userFeatureMap, rankItems);
+
+        LOGGER.debug("ctr ranker time java items size={}, time={} ", result.size(),
+                System.currentTimeMillis() - startTime);
+
+        return result;
+    }
+
+    private List<AdRankItem> rankByJava(final Map<String, String> sceneFeatureMap,
+                                        final Map<String, String> userFeatureMap,
+                                        final List<AdRankItem> items) {
+        long startTime = System.currentTimeMillis();
+        LRV2Model model = (LRV2Model) this.getModel();
+        // 所有都参与打分，按照ctr排序
+        multipleCtrScore(items, userFeatureMap, sceneFeatureMap, model);
+
+        // debug log
+        if (LOGGER.isDebugEnabled()) {
+            for (AdRankItem item : items) {
+                LOGGER.debug("before enter feeds model predict ctr score [{}] [{}]", item, item);
+            }
+        }
+
+        Collections.sort(items);
+
+        LOGGER.debug("[ctr ranker time java] items size={}, cost={} ", items.size(),
+                System.currentTimeMillis() - startTime);
+        return items;
+    }
+
+    private void multipleCtrScore(final List<AdRankItem> items,
+                                  final Map<String, String> userFeatureMap,
+                                  final Map<String, String> sceneFeatureMap,
+                                  final LRV2Model model) {
+
+        List<Callable<Object>> calls = new ArrayList<Callable<Object>>();
+        for (int index = 0; index < items.size(); index++) {
+            final int fIndex = index;
+            calls.add(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    try {
+                        calcScore(model, items.get(fIndex), userFeatureMap, sceneFeatureMap);
+                    } catch (Exception e) {
+                        LOGGER.error("ctr exception: [{}] [{}]", items.get(fIndex).getAdId(), ExceptionUtils.getFullStackTrace(e));
+                    }
+                    return new Object();
+                }
+            });
+        }
+
+        List<Future<Object>> futures = null;
+        try {
+            futures = executorService.invokeAll(calls, LOCAL_TIME_OUT, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            LOGGER.error("execute invoke fail: {}", ExceptionUtils.getFullStackTrace(e));
+        }
+
+        // 等待所有请求的结果返回, 超时也返回
+        int cancel = 0;
+        if (futures != null) {
+            for (Future<Object> future : futures) {
+                try {
+                    if (!future.isDone() || future.isCancelled() || future.get() == null) {
+                        cancel++;
+                    }
+                } catch (InterruptedException e) {
+                    LOGGER.error("InterruptedException: ", e);
+                } catch (ExecutionException e) {
+                    LOGGER.error("ExecutionException {},{}", sceneFeatureMap.size(),
+                            ExceptionUtils.getFullStackTrace(e));
+                }
+            }
+        }
+    }
+
+    public double calcScore(final LRV2Model lrModel,
+                            final AdRankItem item,
+                            final Map<String, String> userFeatureMap,
+                            final Map<String, String> sceneFeatureMap) {
+
+
+        Map<String, String> featureMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+            featureMap.putAll(item.getFeatureMap());
+        }
+        if (MapUtils.isNotEmpty(userFeatureMap)) {
+            featureMap.putAll(userFeatureMap);
+        }
+        if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+            featureMap.putAll(sceneFeatureMap);
+        }
+
+        double pro = 0.0;
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                pro = lrModel.score(featureMap);
+                // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.getScoreMap().put("rovLrScore", pro);
+        item.setLrScore(pro);
+        return pro;
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/container/PidLambdaForCpcContainer.java

@@ -69,17 +69,17 @@ public class PidLambdaForCpcContainer {
 
     @PostConstruct
     private void init(){
-        instanceClient();
-        final Runnable task = new Runnable() {
-            public void run() {
-                try {
-                    loadAndCalIfNeed();
-                }catch (Exception e){
-                    e.printStackTrace();
-                }
-            }
-        };
-        scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
+        // instanceClient();
+        // final Runnable task = new Runnable() {
+        //     public void run() {
+        //         try {
+        //             loadAndCalIfNeed();
+        //         }catch (Exception e){
+        //             e.printStackTrace();
+        //         }
+        //     }
+        // };
+        // scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
     }
 
     private void instanceClient(){

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/RequestConvert.java

@@ -1,27 +1,36 @@
 package com.tzld.piaoquan.ad.engine.service.score.convert;
 
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.service.score.param.RecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
-import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
-import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
 
 public class RequestConvert {
 
-    public static ScoreParam requestConvert(RecommendRequestParam request){
-        AdRequestContext context=new AdRequestContext();
+    public static ScoreParam requestConvert(RecommendRequestParam request) {
+        AdRequestContext context = new AdRequestContext();
         context.setApptype(request.getAppType().toString());
         context.setMachineinfoBrand(request.getMachineInfo().getBrand());
         context.setMachineinfoModel(request.getMachineInfo().getModel());
         context.setMachineinfoSdkversion(request.getMachineInfo().getSdkVersion());
         context.setMachineinfoWchatversion(request.getMachineInfo().getWeChatVersion());
-        LocalDateTime date=LocalDateTime.now();
-        context.setHour(date.getHour()+"");
+
+        LocalDateTime date = LocalDateTime.now();
+        String hourStr = date.format(DateTimeFormatter.ofPattern("HH"));
+        context.setHour(hourStr);
         context.setDay(date.format(DateTimeFormatter.ofPattern("yyyyMMdd")));
-        context.setWeek(date.getDayOfWeek().getValue()+"");
-        ScoreParam scoreParam=new ScoreParam();
+        context.setWeek(String.valueOf(date.getDayOfWeek().getValue()));
+        context.setRegion(request.getRegion().replace("省", ""));
+        context.setCity(request.getCity().replace("市", ""));
+
+        ScoreParam scoreParam = new ScoreParam();
+        scoreParam.setAdAbGroup(request.getAdAbGroup());
         scoreParam.setRequestContext(context);
         scoreParam.setVideoId(request.getVideoId());
         scoreParam.setNewExpGroup(request.getNewExpGroup());
@@ -30,4 +39,16 @@ public class RequestConvert {
         return scoreParam;
     }
 
+    public static List<AdRankItem> adIdConvertRankItem(List<AdPlatformCreativeDTO> creativeDTOList, RecommendRequestParam request) {
+        List<AdRankItem> adRankItems = new ArrayList<>(creativeDTOList.size());
+        for (AdPlatformCreativeDTO dto : creativeDTOList) {
+            AdRankItem adRankItem = new AdRankItem();
+            adRankItem.setAdId(dto.getCreativeId());
+            adRankItem.setCreativeCode(dto.getCreativeCode());
+            adRankItem.setAdVerId(dto.getAdVerId());
+            adRankItem.setVideoId(request.getVideoId());
+            adRankItem.setWeight(dto.getWeight());
+        }
+        return adRankItems;
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/dto/AdDirectionScore.java

@@ -0,0 +1,17 @@
+package com.tzld.piaoquan.ad.engine.service.score.dto;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.Map;
+
+@Data
+@Builder
+public class AdDirectionScore {
+
+    private Double exponent;
+
+
+    private Map<String, String> scoreDetail;
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/dto/AdPlatformCreativeDTO.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.ad.engine.service.score.dto;
 
-import com.alibaba.fastjson.JSON;
 import lombok.Builder;
 import lombok.Data;
 import lombok.ToString;
@@ -14,7 +13,7 @@ public class AdPlatformCreativeDTO {
 
     private String creativeCode;
 
-    private Integer bidType;
+    private String adVerId;
 
     private Double pctr;
 
@@ -31,20 +30,13 @@ public class AdPlatformCreativeDTO {
     private Double ecpm2;
 
     private double weight;
-
-    public static void main(String[] args) {
-        System.out.println(JSON.toJSONString(AdPlatformCreativeDTO.builder()
-                .creativeId(3366L)
-                .creativeCode("CREATIVE_17036704789851370")
-                .bidType(2)
-                .pctr(3.2)
-                .pcvr(1.2)
-                .cpa(40.1)
-                .bid1(1.3)
-                .bid2(2.1)
-                .ecpm1(0.1)
-                .ecpm2(0.2)
-                .build()));
-    }
+    // 计划id
+    private Long campaignId;
+    // 广告id
+    private Long adId;
+    /**
+     * 定向打分参数
+     */
+    private AdDirectionScore adDirectionScore;
 
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -4,11 +4,13 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.commons.util.AbUtil;
 import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import com.tzld.piaoquan.ad.engine.service.predict.helper.NewExpInfoHelper;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
 import com.tzld.piaoquan.ad.engine.service.remote.FeatureRemoteService;
 import com.tzld.piaoquan.ad.engine.service.score.RankService;
+import com.tzld.piaoquan.ad.engine.service.score.RankService680;
 import com.tzld.piaoquan.ad.engine.service.score.VideoAdThompsonScorerV2;
 import com.tzld.piaoquan.ad.engine.service.score.container.AdCreativeFeatureContainer;
 import com.tzld.piaoquan.ad.engine.service.score.container.PidLambdaContainer;
@@ -21,13 +23,13 @@ import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdItemFeature;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
+import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
@@ -38,19 +40,26 @@ import java.util.stream.Collectors;
 public class RankServiceImpl implements RankService {
 
     private final static Logger log = LoggerFactory.getLogger(RankServiceImpl.class);
-    DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd");
-    DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss");
+    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd");
+    private static final DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss");
+
+
     @Autowired
-    FeatureRemoteService featureRemoteService;
+    private FeatureRemoteService featureRemoteService;
     @Autowired
-    RankServiceThompsonImpl rankServiceThompson;
+    private RankServiceThompsonImpl rankServiceThompson;
     @Autowired
-    VideoAdThompsonScorerV2 videoAdThompsonScorerV2;
+    private VideoAdThompsonScorerV2 videoAdThompsonScorerV2;
     @Autowired
     private LogHubService logHubService;
+    @Autowired
+    private AdCreativeFeatureContainer adCreativeFeatureContainer;
 
     @Autowired
-    AdCreativeFeatureContainer adCreativeFeatureContainer;
+    private RankService680 fmRankService;
+    @Autowired
+    private TacticsAndLRModelScoreRankService tacticsAndFmModelScoreRankService;
+
     @Value("${ad.model.cpm.max:200}")
     Double cpmMax = 200d;
     @Value("${ad.model.cpm.min:30}")
@@ -60,16 +69,55 @@ public class RankServiceImpl implements RankService {
     @Value("${ad.cvr.adjusting.exp:652}")
     private String cvrAdjustingExpCode;
 
-    public AdRankItem adItemRank(RankRecommendRequestParam request) {
+    @Value("${fm.model.score.exp.code:680}")
+    private String fmModelScoreExpCode;
+
+    @Value("${tactics.fm.mode.score.exp.code:679}")
+    private String tacticsAndFmModelScoreExpCode;
+
+    public AdRankItem adItemRank(RankRecommendRequestParam requestParam) {
+        Set<String> expCodeSet = AbUtil.unfoldAllExpCode(requestParam.getAdAbExpArr());
+        Long appType = requestParam.getAppType();
+        Integer newExpGroup = requestParam.getNewExpGroup();
+        if (AbUtil.isInAbExp(expCodeSet, appType, newExpGroup, fmModelScoreExpCode)) {
+            return rankBy680(requestParam);
+        } else if (AbUtil.isInAbExp(expCodeSet, appType, newExpGroup, tacticsAndFmModelScoreExpCode)) {
+            return rankBy679(requestParam);
+        } else {
+            return adItemRankOld(requestParam);
+        }
+    }
+
+    private AdRankItem rankBy680(RankRecommendRequestParam request) {
+        ScoreParam scoreParam = RequestConvert.requestConvert(request);
+        List<AdRankItem> adRankItems = fmRankService.adItemRank(request, scoreParam);
+        // List<JSONObject> collect = adRankItems.stream().map(i -> {
+        //     JSONObject json = new JSONObject();
+        //     json.put("cid", i.getAdId());
+        //     json.put("lrScore", i.getLrScore());
+        //     json.put("score", i.getScore());
+        //     json.put("metaFeatureMap", i.getMetaFeatureMap());
+        //     json.put("allFeatureMap", i.getFeatureMap());
+        //     return json;
+        // }).collect(Collectors.toList());
+        // log.info("LR模型打分结果: {}", JSON.toJSONString(collect));
+        logHubService.scoreLogUpload(scoreParam, request.getAdIdList(), adRankItems, request, "LRModelScore", "680");
+        return adRankItems.get(0);
+    }
+
+    private AdRankItem rankBy679(RankRecommendRequestParam requestParam) {
+        ScoreParam scoreParam = RequestConvert.requestConvert(requestParam);
+        List<AdRankItem> adRankItems = tacticsAndFmModelScoreRankService.adItemRank(requestParam, scoreParam);
+        logHubService.scoreLogUpload(scoreParam, requestParam.getAdIdList(), adRankItems, requestParam, "directionAndLRModelScore", "679");
+        double score = adRankItems.get(0).getScore();
+        if (score > 0) {
+            return adRankItems.get(0);
+        }
+        return null;
+    }
+
+    public AdRankItem adItemRankOld(RankRecommendRequestParam request) {
         ScoreParam param = RequestConvert.requestConvert(request);
-        LocalDateTime currentTime = LocalDateTime.now();
-        int currentHour = currentTime.getHour();
-        int dayOfWeek = currentTime.getDayOfWeek().getValue();
-        param.getRequestContext().setHour(currentHour + "");
-        param.getRequestContext().setWeek(dayOfWeek + "");
-        param.getRequestContext().setRegion(request.getRegion().replace("省", ""));
-        param.getRequestContext().setCity(request.getCity().replace("市", ""));
-        param.getRequestContext().setDay(currentTime.format(dateFormatter));
 
         UserAdFeature userAdFeature = featureRemoteService.getUserAdFeature(request.getMid());
         if (userAdFeature == null) {
@@ -78,14 +126,14 @@ public class RankServiceImpl implements RankService {
         Map<Long, List<AdPlatformCreativeDTO>> groupMap = request
                 .getAdIdList()
                 .stream()
-                .collect(Collectors.groupingBy(creativeDTO -> creativeDTO.getCreativeId()));
+                .collect(Collectors.groupingBy(AdPlatformCreativeDTO::getCreativeId));
         Map<Long, AdRankItem> cache = adCreativeFeatureContainer.getAll(new ArrayList<>(groupMap.keySet()));
         List<AdRankItem> rankItems = Collections.emptyList();
         if (!cache.isEmpty()) {
             rankItems = new LinkedList<>(cache.values());
         }
         // 避免recommend-feature出问题
-        if (rankItems == null || rankItems.size() == 0) {
+        if (CollectionUtils.isEmpty(rankItems)) {
             rankItems = new LinkedList<>();
             for (Long adId : groupMap.keySet()) {
                 AdRankItem item = new AdRankItem();
@@ -96,15 +144,24 @@ public class RankServiceImpl implements RankService {
         }
         boolean inCpcPidExp = false;
         boolean inCvrAdjustingExp = false;
-        if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
-            for (Map<String, Object> map : request.getAdAbExpArr()) {
-                if (map.getOrDefault("abExpCode", "").equals(cpcPidExpCode)) {
-                    inCpcPidExp = true;
-                }
-                if (map.getOrDefault("abExpCode", "").equals(cvrAdjustingExpCode)) {
-                    inCvrAdjustingExp = true;
-                }
-            }
+        // if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
+        //     for (Map<String, Object> map : request.getAdAbExpArr()) {
+        //         if (map.getOrDefault("abExpCode", "").equals(cpcPidExpCode)) {
+        //             inCpcPidExp = true;
+        //         }
+        //         if (map.getOrDefault("abExpCode", "").equals(cvrAdjustingExpCode)) {
+        //             inCvrAdjustingExp = true;
+        //         }
+        //     }
+        // }
+        if (CollectionUtils.isNotEmpty(request.getAdAbExpArr())) {
+            Set<String> abExpCode = request.getAdAbExpArr().stream()
+                    .map(map -> map.get("abExpCode"))
+                    .filter(Objects::nonNull)
+                    .map(Object::toString)
+                    .collect(Collectors.toSet());
+            inCpcPidExp = abExpCode.contains(cpcPidExpCode);
+            inCvrAdjustingExp = abExpCode.contains(cvrAdjustingExpCode);
         }
         double lambda = -1d;
         if (inCpcPidExp) {
@@ -162,7 +219,7 @@ public class RankServiceImpl implements RankService {
             object.put("score", rankResult.get(0).getScore());
             object.put("pidLambda", rankResult.get(0).getPidLambda());
             object.put("lrsamples", rankResult.get(0).getLrSampleString());
-            object.put("dataTime", currentTime.format(timeFormatter));
+            object.put("dataTime", LocalDateTime.now().format(timeFormatter));
             object.put("creativeId", rankResult.get(0).getAdId());
             object.put("videoId", request.getVideoId());
             object.put("pqtId", request.getPqtId());
@@ -179,7 +236,7 @@ public class RankServiceImpl implements RankService {
             }
 
             for (AdRankItem adRankItem : rankResult) {
-                adRankItem.getExt().put("userAdFeature", JSON.toJSONString(userAdFeature));
+                // adRankItem.getItemFeature().put("userAdFeature", JSON.toJSONString(userAdFeature));
             }
             // 日志上报
             logHubService.scoreLogUpload(param, request.getAdIdList(), rankResult, request,
@@ -204,12 +261,21 @@ public class RankServiceImpl implements RankService {
         param.getRequestContext().setDay(currentTime.format(dateFormatter));
 
         Set<String> expCodes = new HashSet<>();
-        if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
-            for (Map<String, Object> map : request.getAdAbExpArr()) {
-                String expCode = map.getOrDefault("abExpCode", "").toString();
-                expCodes.add(expCode);
-            }
+        // if (request.getAdAbExpArr() != null && request.getAdAbExpArr().size() != 0) {
+        //     for (Map<String, Object> map : request.getAdAbExpArr()) {
+        //         String expCode = map.getOrDefault("abExpCode", "").toString();
+        //         expCodes.add(expCode);
+        //     }
+        // }
+
+        if (CollectionUtils.isNotEmpty(request.getAdAbExpArr())) {
+            expCodes = request.getAdAbExpArr().stream()
+                    .map(m -> m.get("abExpCode"))
+                    .filter(Objects::nonNull)
+                    .map(Object::toString)
+                    .collect(Collectors.toSet());
         }
+
         ThresholdPredictModelParam modelParam = ThresholdPredictModelParam.builder()
                 .build();
         // 兜底方案

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/TacticsAndLRModelScoreRankService.java

@@ -0,0 +1,76 @@
+package com.tzld.piaoquan.ad.engine.service.score.impl;
+
+import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.service.score.RankService680;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdDirectionScore;
+import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
+import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
+import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+public class TacticsAndLRModelScoreRankService {
+
+    @Autowired
+    private RankService680 rankService680;
+
+    public List<AdRankItem> adItemRank(RankRecommendRequestParam requestParam, ScoreParam scoreParam) {
+
+        // LR模型打分结果
+        List<AdRankItem> result = rankService680.adItemRank(requestParam, scoreParam);
+
+        Map<Long, AdDirectionScore> adDirectionScoreMap = requestParam.getAdIdList().stream()
+                .collect(Collectors.toMap(AdPlatformCreativeDTO::getCreativeId, AdPlatformCreativeDTO::getAdDirectionScore));
+
+        for (AdRankItem adRankItem : result) {
+
+            double s1 = this.calcDirectionScore(adRankItem, adDirectionScoreMap.get(adRankItem.getAdId()));
+            adRankItem.setAdDirectionScore(s1);
+
+            // 模型的打分结果已经乘CPA，此处不需要重复乘CPA
+            double s2 = adRankItem.getScore();
+            adRankItem.setScore(s1 * s2);
+
+        }
+
+        Collections.sort(result);
+        return result;
+    }
+
+    private double calcDirectionScore(AdRankItem adRankItem, AdDirectionScore adDirectionScore) {
+        if (Objects.isNull(adDirectionScore)) {
+            adRankItem.setAdDirectionScore(1);
+            return 1;
+        }
+        double exponent = Objects.nonNull(adDirectionScore.getExponent()) ? adDirectionScore.getExponent() : 1;
+        Map<String, String> scoreDetail = adDirectionScore.getScoreDetail();
+
+        double network = Double.parseDouble(scoreDetail.getOrDefault("network", "1"));
+        double phoneModel = Double.parseDouble(scoreDetail.getOrDefault("phoneModel", "1"));
+        double area = Double.parseDouble(scoreDetail.getOrDefault("area", "1"));
+        double peoplePackage = Double.parseDouble(scoreDetail.getOrDefault("peoplePackage", "1"));
+        double excludeConv = Double.parseDouble(scoreDetail.getOrDefault("excludeConv", "1"));
+        double excludeClick = Double.parseDouble(scoreDetail.getOrDefault("excludeClick", "1"));
+        double excludeView = Double.parseDouble(scoreDetail.getOrDefault("excludeView", "1"));
+
+        double excludeMin = Math.min(excludeConv, Math.min(excludeClick, excludeView));
+
+        double s1 = (network * phoneModel * area * peoplePackage * excludeMin);
+        double s2 = Math.pow(s1, exponent);
+        adRankItem.getScoreMap().put("adDirectionScore", s2);
+        adRankItem.setAdDirectionScore(s2);
+
+        Map<String, String> scoreDetailMap = new HashMap<>(scoreDetail);
+        scoreDetailMap.put("exponent", String.valueOf(exponent));
+        adRankItem.getMetaFeatureMap().put("adDirectionScoreDetail", scoreDetailMap);
+
+        return s2;
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/RankRecommendRequestParam.java

@@ -2,10 +2,12 @@ package com.tzld.piaoquan.ad.engine.service.score.param;
 
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 
 import java.util.List;
 
 @Data
+@EqualsAndHashCode(callSuper = true)
 public class RankRecommendRequestParam extends RecommendRequestParam{
      List<AdPlatformCreativeDTO> adIdList;
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/RecommendRequestParam.java

@@ -9,19 +9,19 @@ import java.util.Map;
 @Data
 @ToString
 public class RecommendRequestParam {
-    MachineInfoParam machineInfo;
+    private MachineInfoParam machineInfo;
 
-    Long videoId;
-    Long appType;
-    String mid;
-    //省-中文
-    String region = "-1";
-    //市-中文
-    String city = "-1";
-    Integer newExpGroup;
-    String pqtId;
-    List<Map> adAbExpArr ;
-    String adAbGroup;
+    private Long videoId;
+    private Long appType;
+    private String mid;
+    // 省-中文
+    private String region = "-1";
+    // 市-中文
+    private String city = "-1";
+    private Integer newExpGroup;
+    private String pqtId;
+    private List<Map<String, String>> adAbExpArr;
+    private String adAbGroup;
 
     private StatisticsLogParam statisticsLog;
 }