Merge branch 'feature_dyp' of algorithm/ad-engine into master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/thread/ThreadPoolFactory.java

@@ -18,20 +18,20 @@ public final class ThreadPoolFactory {
             new LinkedBlockingQueue<>(1000),
             new ThreadFactoryBuilder().setNameFormat("DEFAULT-%d").build(),
             new ThreadPoolExecutor.AbortPolicy());
-    public final static ExecutorService RECALL = new CommonThreadPoolExecutor(
-            128,
-            128,
+    private final static ExecutorService FEATURE = new CommonThreadPoolExecutor(
+            256,
+            256,
             0L, TimeUnit.SECONDS,
             new LinkedBlockingQueue<>(1000),
-            new ThreadFactoryBuilder().setNameFormat("RecallService-%d").build(),
+            new ThreadFactoryBuilder().setNameFormat("Feature-%d").build(),
             new ThreadPoolExecutor.AbortPolicy());
 
     public static ExecutorService defaultPool() {
         return DEFAULT;
     }
 
-    public static ExecutorService recallPool() {
-        return RECALL;
+    public static ExecutorService feature() {
+        return FEATURE;
     }
 
 }

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/AdRecommendController.java

@@ -35,22 +35,23 @@ public class AdRecommendController {
             map.put("msg", "score error");
             return map;
         }
-
-        AdRankItem rankResult = rankService.adItemRank(request);
-
-        if (Objects.isNull(rankResult)) {
-            map.put("code", "1");
-            map.put("msg", "score error");
-            return map;
+        try {
+            AdRankItem rankResult = rankService.adItemRank(request);
+            if (Objects.nonNull(rankResult)) {
+                map.put("code", "0");
+                map.put("msg", "success");
+                Map<String, Object> contentMap = new HashMap<>();
+                contentMap.put("adId", rankResult.getAdId());
+                contentMap.put("adScore", rankResult.getScore());
+                map.put("content", contentMap);
+                return map;
+            }
+        } catch (Exception e) {
+            log.error("模型排序异常: ", e);
         }
 
-
-        map.put("code", "0");
-        map.put("msg", "success");
-        Map<String, Object> contentMap = new HashMap<>();
-        contentMap.put("adId", rankResult.getAdId());
-        contentMap.put("adScore", rankResult.getScore());
-        map.put("content", contentMap);
+        map.put("code", "1");
+        map.put("msg", "score error");
         return map;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService683.java

@@ -12,8 +12,7 @@ import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
+import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
 import java.io.BufferedReader;
@@ -23,11 +22,12 @@ import java.io.InputStreamReader;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Slf4j
-@Service
+@Component
 public class RankService683 extends RankStrategyXGBBasic {
 
     @ApolloJsonValue("${no.postback.conversion.adverids:[]}")
@@ -37,9 +37,6 @@ public class RankService683 extends RankStrategyXGBBasic {
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
-    @Value("${similarity.concurrent: false}")
-    private boolean similarityConcurrent;
-
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
@@ -47,6 +44,7 @@ public class RankService683 extends RankStrategyXGBBasic {
 
         long start = System.currentTimeMillis();
         // 特征处理
+        // feature1
         Feature feature = this.getFeature(scoreParam, request);
 
         Map<String, Map<String, String>> userFeature = feature.getUserFeature();
@@ -72,130 +70,118 @@ public class RankService683 extends RankStrategyXGBBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
-        List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
+        List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
-        if (similarityConcurrent) {
-            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+        List<Future<AdRankItem>> futures = new ArrayList<>();
+        CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
+        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+            Future<AdRankItem> future = ThreadPoolFactory.feature().submit(() -> {
                 AdRankItem adRankItem = new AdRankItem();
-                adRankItem.setAdId(dto.getCreativeId());
-                adRankItem.setCreativeCode(dto.getCreativeCode());
-                adRankItem.setAdVerId(dto.getAdVerId());
-                adRankItem.setVideoId(request.getVideoId());
-                adRankItem.setCpa(dto.getCpa());
-                adRankItem.setId(dto.getAdId());
-                adRankItem.setCampaignId(dto.getCampaignId());
-                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                adRankItem.setRandom(random.nextInt(1000));
-
-                String cidStr = dto.getCreativeId().toString();
-                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
-                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-
-                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-
-                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-
-                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-
-                this.handleD1Feature(d1Feature, cidFeatureMap);
-
-                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
-
-                // adRankItem.setFeatureMap(cidFeatureMap);
-
-                adRankItems.add(adRankItem);
-
-            }
-
-
-            CountDownLatch cdl = new CountDownLatch(adRankItems.size());
-            for (AdRankItem item : adRankItems) {
-                ThreadPoolFactory.defaultPool().submit(() -> {
-                    try {
-                        String cidStr = String.valueOf(item.getAdId());
-                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-                        String title = b1Feature.getOrDefault("cidtitle", "");
-                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
-                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
-                    } finally{
-                        cdl.countDown();
-                    }
-                });
-            }
+                try {
+                    adRankItem.setAdId(dto.getCreativeId());
+                    adRankItem.setCreativeCode(dto.getCreativeCode());
+                    adRankItem.setAdVerId(dto.getAdVerId());
+                    adRankItem.setVideoId(request.getVideoId());
+                    adRankItem.setCpa(dto.getCpa());
+                    adRankItem.setId(dto.getAdId());
+                    adRankItem.setCampaignId(dto.getCampaignId());
+                    adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setRandom(random.nextInt(1000));
+
+                    String cidStr = dto.getCreativeId().toString();
+                    Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                    Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                    Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+                    Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+                    Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                    this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                    this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                    this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                    this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                    this.handleD1Feature(d1Feature, cidFeatureMap);
+                    this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                    return adRankItem;
+                } finally {
+                    cdl1.countDown();
+                }
+            });
+            futures.add(future);
+        }
+        try {
+            cdl1.await(300, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+        for (Future<AdRankItem> future : futures) {
             try {
-                cdl.await(200, TimeUnit.MILLISECONDS);
+                if (future.isDone()) {
+                    adRankItems.add(future.get());
+                }
             } catch (Exception e) {
-                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
-            }
-        } else {
-            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-                AdRankItem adRankItem = new AdRankItem();
-                adRankItem.setAdId(dto.getCreativeId());
-                adRankItem.setCreativeCode(dto.getCreativeCode());
-                adRankItem.setAdVerId(dto.getAdVerId());
-                adRankItem.setVideoId(request.getVideoId());
-                adRankItem.setCpa(dto.getCpa());
-                adRankItem.setId(dto.getAdId());
-                adRankItem.setCampaignId(dto.getCampaignId());
-                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                adRankItem.setRandom(random.nextInt(1000));
-
-                String cidStr = dto.getCreativeId().toString();
-                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
-                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-
-                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-
-                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-
-                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-
-                this.handleD1Feature(d1Feature, cidFeatureMap);
-
-                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
-
-                String title = b1Feature.getOrDefault("cidtitle", "");
-                this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
-                this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
-
-                // adRankItem.setFeatureMap(cidFeatureMap);
-
-                adRankItems.add(adRankItem);
-
+                log.error("Feature handle error", e);
             }
         }
 
         long time2 = System.currentTimeMillis();
+        // feature3
+        CountDownLatch cdl2 = new CountDownLatch(adRankItems.size() * 2);
+        for (AdRankItem item : adRankItems) {
+            String cidStr = String.valueOf(item.getAdId());
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+            String title = b1Feature.getOrDefault("cidtitle", "");
+            ThreadPoolFactory.defaultPool().submit(() -> {
+                try {
+                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                } finally {
+                    cdl2.countDown();
+                }
+            });
+            ThreadPoolFactory.defaultPool().submit(() -> {
+                try {
+                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                } finally {
+                    cdl2.countDown();
+                }
+            });
+        }
+        long time31 = System.currentTimeMillis();
+        try {
+            cdl2.await(150, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+
+        // feature4
+        long time3 = System.currentTimeMillis();
         // 分桶
         this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
+        CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
         for (AdRankItem adRankItem : adRankItems) {
-            Map<String, String> featureMap = adRankItem.getFeatureMap();
-            adRankItem.setFeatureMap(this.featureBucket(featureMap));
+            ThreadPoolFactory.feature().submit(() -> {
+                try {
+                    Map<String, String> featureMap = adRankItem.getFeatureMap();
+                    adRankItem.setFeatureMap(this.featureBucket(featureMap));
+                } finally {
+                    cdl4.countDown();
+                }
+            });
         }
-
-
-
+        try {
+            cdl4.await(100, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+        long time4 = System.currentTimeMillis();
         // 打分排序
-        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_683)
-                .scoring(sceneFeatureMap, userFeatureMap, adRankItems);
-
-        long time3 = System.currentTimeMillis();
+        // getScorerPipeline
+        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_683).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
+        long time5 = System.currentTimeMillis();
+        // loop
         for (AdRankItem item : result) {
             item.setScore(item.getLrScore() * item.getCpa());
             item.getScoreMap().put("cpa", item.getCpa());
@@ -235,9 +221,10 @@ public class RankService683 extends RankStrategyXGBBasic {
             }
         }
 
-        long time4 = System.currentTimeMillis();
-        log.info("cost={}, feature1={}, feature2={}, getScorerPipeline={}, loop={}",
-                time4 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3);
+        log.info("cost={}, feature1={}, feature2={}, feature31={}, feature32={}, feature4={}, getScorerPipeline={}, " +
+                        "adIdSize={}, adRankItemsSize={}",
+                time5 - start, time1 - start, time2 - time1, time31 - time2, time3 - time31, time4 - time3,
+                time5 - time4, request.getAdIdList().size(), adRankItems.size());
 
         result.sort(ComparatorUtil.equalsRandomComparator());
 
@@ -288,13 +275,11 @@ public class RankService683 extends RankStrategyXGBBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
-                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -613,41 +598,4 @@ public class RankService683 extends RankStrategyXGBBasic {
         return newFeatureMap;
     }
 
-    // public static class Tuple5 {
-    //     public String f1;
-    //     public String f2;
-    //     public String f3;
-    //     public String f4;
-    //     public String f5;
-    //
-    //     public Tuple5(String f1, String f2, String f3, String f4, String f5) {
-    //         this.f1 = f1;
-    //         this.f2 = f2;
-    //         this.f3 = f3;
-    //         this.f4 = f4;
-    //         this.f5 = f5;
-    //     }
-    // }
-    //
-    // public static class TupleMapEntry<T> {
-    //     public String key;
-    //     public T value;
-    //
-    //     public TupleMapEntry(String key, T value) {
-    //         this.key = key;
-    //         this.value = value;
-    //     }
-    // }
-    //
-    // public static class Tuple2<F1, F2> {
-    //     public F1 f1;
-    //
-    //     public F2 f2;
-    //
-    //     public Tuple2(F1 first, F2 name) {
-    //         this.f1 = first;
-    //         this.f2 = name;
-    //     }
-    //
-    // }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankStrategyXGBAutoUpdateModel688.java

@@ -12,7 +12,6 @@ import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
@@ -23,6 +22,7 @@ import java.io.InputStreamReader;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -37,9 +37,6 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
-    @Value("${similarity.concurrent: false}")
-    private boolean similarityConcurrent;
-
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
@@ -52,6 +49,7 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
 
         long start = System.currentTimeMillis();
         // 特征处理
+        // feature1
         Feature feature = this.getFeature(scoreParam, request);
 
         Map<String, Map<String, String>> userFeature = feature.getUserFeature();
@@ -77,126 +75,118 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
-        List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
+        List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
-        if (similarityConcurrent) {
-            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+        List<Future<AdRankItem>> futures = new ArrayList<>();
+        CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
+        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+            Future<AdRankItem> future = ThreadPoolFactory.feature().submit(() -> {
                 AdRankItem adRankItem = new AdRankItem();
-                adRankItem.setAdId(dto.getCreativeId());
-                adRankItem.setCreativeCode(dto.getCreativeCode());
-                adRankItem.setAdVerId(dto.getAdVerId());
-                adRankItem.setVideoId(request.getVideoId());
-                adRankItem.setCpa(dto.getCpa());
-                adRankItem.setId(dto.getAdId());
-                adRankItem.setCampaignId(dto.getCampaignId());
-                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                adRankItem.setRandom(random.nextInt(1000));
-
-                String cidStr = dto.getCreativeId().toString();
-                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
-                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-
-                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-
-                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-
-                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-
-                this.handleD1Feature(d1Feature, cidFeatureMap);
-
-                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
-
-                // adRankItem.setFeatureMap(cidFeatureMap);
-
-                adRankItems.add(adRankItem);
-
-            }
-
-
-            CountDownLatch cdl = new CountDownLatch(adRankItems.size());
-            for (AdRankItem item : adRankItems) {
-                ThreadPoolFactory.defaultPool().submit(() -> {
-                    try {
-                        String cidStr = String.valueOf(item.getAdId());
-                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-                        String title = b1Feature.getOrDefault("cidtitle", "");
-                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
-                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
-                    } finally{
-                        cdl.countDown();
-                    }
-                });
-            }
+                try {
+                    adRankItem.setAdId(dto.getCreativeId());
+                    adRankItem.setCreativeCode(dto.getCreativeCode());
+                    adRankItem.setAdVerId(dto.getAdVerId());
+                    adRankItem.setVideoId(request.getVideoId());
+                    adRankItem.setCpa(dto.getCpa());
+                    adRankItem.setId(dto.getAdId());
+                    adRankItem.setCampaignId(dto.getCampaignId());
+                    adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setRandom(random.nextInt(1000));
+
+                    String cidStr = dto.getCreativeId().toString();
+                    Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                    Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                    Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+                    Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+                    Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                    this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                    this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                    this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                    this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                    this.handleD1Feature(d1Feature, cidFeatureMap);
+                    this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                    return adRankItem;
+                } finally {
+                    cdl1.countDown();
+                }
+            });
+            futures.add(future);
+        }
+        try {
+            cdl1.await(300, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+        for (Future<AdRankItem> future : futures) {
             try {
-                cdl.await(200, TimeUnit.MILLISECONDS);
+                if (future.isDone()) {
+                    adRankItems.add(future.get());
+                }
             } catch (Exception e) {
-                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
-            }
-        } else {
-            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-                AdRankItem adRankItem = new AdRankItem();
-                adRankItem.setAdId(dto.getCreativeId());
-                adRankItem.setCreativeCode(dto.getCreativeCode());
-                adRankItem.setAdVerId(dto.getAdVerId());
-                adRankItem.setVideoId(request.getVideoId());
-                adRankItem.setCpa(dto.getCpa());
-                adRankItem.setId(dto.getAdId());
-                adRankItem.setCampaignId(dto.getCampaignId());
-                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                adRankItem.setRandom(random.nextInt(1000));
-
-                String cidStr = dto.getCreativeId().toString();
-                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
-                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-
-                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-
-                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-
-                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-
-                this.handleD1Feature(d1Feature, cidFeatureMap);
-
-                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
-
-                String title = b1Feature.getOrDefault("cidtitle", "");
-                this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
-                this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
-
-                // adRankItem.setFeatureMap(cidFeatureMap);
-
-                adRankItems.add(adRankItem);
-
+                log.error("Feature handle error", e);
             }
         }
 
         long time2 = System.currentTimeMillis();
+        // feature3
+        CountDownLatch cdl2 = new CountDownLatch(adRankItems.size() * 2);
+        for (AdRankItem item : adRankItems) {
+            String cidStr = String.valueOf(item.getAdId());
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+            String title = b1Feature.getOrDefault("cidtitle", "");
+            ThreadPoolFactory.defaultPool().submit(() -> {
+                try {
+                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                } finally {
+                    cdl2.countDown();
+                }
+            });
+            ThreadPoolFactory.defaultPool().submit(() -> {
+                try {
+                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                } finally {
+                    cdl2.countDown();
+                }
+            });
+        }
+        long time31 = System.currentTimeMillis();
+        try {
+            cdl2.await(150, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+
+        // feature4
+        long time3 = System.currentTimeMillis();
         // 分桶
         this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
+        CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
         for (AdRankItem adRankItem : adRankItems) {
-            Map<String, String> featureMap = adRankItem.getFeatureMap();
-            adRankItem.setFeatureMap(this.featureBucket(featureMap));
+            ThreadPoolFactory.feature().submit(() -> {
+                try {
+                    Map<String, String> featureMap = adRankItem.getFeatureMap();
+                    adRankItem.setFeatureMap(this.featureBucket(featureMap));
+                } finally {
+                    cdl4.countDown();
+                }
+            });
         }
-
+        try {
+            cdl4.await(100, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+        long time4 = System.currentTimeMillis();
         // 打分排序
+        // getScorerPipeline
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20240909).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
-        long time3 = System.currentTimeMillis();
+        long time5 = System.currentTimeMillis();
+        // loop
         for (AdRankItem item : result) {
             item.setScore(item.getLrScore() * item.getCpa());
             item.getScoreMap().put("cpa", item.getCpa());
@@ -236,9 +226,10 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
             }
         }
 
-        long time4 = System.currentTimeMillis();
-        log.info("cost={}, feature1={}, feature2={}, getScorerPipeline={}, loop={}",
-                time4 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3);
+        log.info("cost={}, feature1={}, feature2={}, feature31={}, feature32={}, feature4={}, getScorerPipeline={}, " +
+                        "adIdSize={}, adRankItemsSize={}",
+                time5 - start, time1 - start, time2 - time1, time31 - time2, time3 - time31, time4 - time3,
+                time5 - time4, request.getAdIdList().size(), adRankItems.size());
 
         result.sort(ComparatorUtil.equalsRandomComparator());
 
@@ -274,7 +265,14 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
         Map<String, String> b9Feature = c1Feature.getOrDefault("alg_cid_feature_weChatVersion_action", new HashMap<>());
 
         List<String> timeList = Arrays.asList("1h", "2h", "3h", "6h", "12h", "1d", "3d", "7d", "yesterday", "today");
-        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(new Tuple2<>(b2Feature, "b2"), new Tuple2<>(b3Feature, "b3"), new Tuple2<>(b4Feature, "b4"), new Tuple2<>(b5Feature, "b5"), new Tuple2<>(b8Feature, "b8"), new Tuple2<>(b9Feature, "b9"));
+        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
+                new Tuple2<>(b2Feature, "b2"),
+                new Tuple2<>(b3Feature, "b3"),
+                new Tuple2<>(b4Feature, "b4"),
+                new Tuple2<>(b5Feature, "b5"),
+                new Tuple2<>(b8Feature, "b8"),
+                new Tuple2<>(b9Feature, "b9")
+        );
         for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
             Map<String, String> feature = tuple2.f1;
             String prefix = tuple2.f2;
@@ -282,13 +280,11 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
-                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -303,7 +299,10 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
         Map<String, String> b7Feature = c1Feature.getOrDefault("alg_cid_feature_hour_action", new HashMap<>());
 
         List<String> timeList = Arrays.asList("7d", "14d");
-        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(new Tuple2<>(b6Feature, "b6"), new Tuple2<>(b7Feature, "b7"));
+        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
+                new Tuple2<>(b6Feature, "b6"),
+                new Tuple2<>(b7Feature, "b7")
+        );
         for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
             Map<String, String> feature = tuple2.f1;
             String prefix = tuple2.f2;
@@ -333,13 +332,15 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
         List<TupleMapEntry<Tuple5>> midActionList = new ArrayList<>();
         if (c1Feature.containsKey("action")) {
             String action = c1Feature.get("action");
-            midActionList = Arrays.stream(action.split(",")).map(r -> {
+            midActionList = Arrays.stream(action.split(","))
+                    .map(r -> {
                         String[] rList = r.split(":");
                         Tuple5 tuple5 = new Tuple5(rList[1], rList[2], rList[3], rList[4], rList[5]);
                         return new TupleMapEntry<>(rList[0], tuple5);
                     })
                     // TODO 倒排
-                    .sorted((a, b) -> Integer.compare(Integer.parseInt(b.value.f1), Integer.parseInt(a.value.f1))).collect(Collectors.toList());
+                    .sorted((a, b) -> Integer.compare(Integer.parseInt(b.value.f1), Integer.parseInt(a.value.f1)))
+                    .collect(Collectors.toList());
         }
 
         double viewAll = midActionList.size();
@@ -381,7 +382,10 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
             featureMap.put("actionstatic_income", String.valueOf(midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0)));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            double ctr = NumUtil.div(midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0), midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0));
+            double ctr = NumUtil.div(
+                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+            );
             featureMap.put("actionstatic_ctr", String.valueOf(ctr));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_conver_" + cid)) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/XGBoostScorer.java

@@ -21,7 +21,7 @@ public class XGBoostScorer extends BaseXGBoostModelScorer {
 
     private static final int LOCAL_TIME_OUT = 150;
     private final static Logger LOGGER = LoggerFactory.getLogger(XGBoostScorer.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(256);
 
 
     public XGBoostScorer(ScorerConfigInfo configInfo) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/XGBoostScorer683.java

@@ -21,7 +21,7 @@ public class XGBoostScorer683 extends BaseXGBoostModelScorer {
 
     private static final int LOCAL_TIME_OUT = 150;
     private final static Logger LOGGER = LoggerFactory.getLogger(XGBoostScorer683.class);
-    private static final ExecutorService executorService = Executors.newFixedThreadPool(128);
+    private static final ExecutorService executorService = Executors.newFixedThreadPool(256);
 
 
     public XGBoostScorer683(ScorerConfigInfo configInfo) {