Merge branch 'feature_dyp' of algorithm/ad-engine into master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/thread/ThreadPoolFactory.java

@@ -12,8 +12,8 @@ import java.util.concurrent.TimeUnit;
  */
 public final class ThreadPoolFactory {
     private final static ExecutorService DEFAULT = new CommonThreadPoolExecutor(
-            32,
-            32,
+            512,
+            512,
             0L, TimeUnit.SECONDS,
             new LinkedBlockingQueue<>(1000),
             new ThreadFactoryBuilder().setNameFormat("DEFAULT-%d").build(),

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/ComparatorUtil.java

@@ -15,7 +15,7 @@ public class ComparatorUtil {
                     return comparison;
                 }
 
-                return Long.compare(o1.hashCode(), o2.hashCode());
+                return Integer.compare(o1.getRandom(), o2.getRandom());
 
                 // 数字相等时随机排列
                 // return new Random().nextInt(3) - 1; // 产生-1、0或1的随机值

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService683.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
@@ -11,6 +12,7 @@ import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.xm.Similarity;
 
@@ -19,6 +21,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -32,12 +37,15 @@ public class RankService683 extends RankStrategyXGBBasic {
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
+    @Value("${similarity.concurrent: false}")
+    private boolean similarityConcurrent;
 
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
         long ts = System.currentTimeMillis() / 1000;
 
+        long start = System.currentTimeMillis();
         // 特征处理
         Feature feature = this.getFeature(scoreParam, request);
 
@@ -62,51 +70,117 @@ public class RankService683 extends RankStrategyXGBBasic {
         Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
 
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
+        long time1 = System.currentTimeMillis();
 
         List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
-        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-            AdRankItem adRankItem = new AdRankItem();
-            adRankItem.setAdId(dto.getCreativeId());
-            adRankItem.setCreativeCode(dto.getCreativeCode());
-            adRankItem.setAdVerId(dto.getAdVerId());
-            adRankItem.setVideoId(request.getVideoId());
-            adRankItem.setCpa(dto.getCpa());
-            adRankItem.setId(dto.getAdId());
-            adRankItem.setCampaignId(dto.getCampaignId());
-            adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+        Random random = new Random();
+        if (similarityConcurrent) {
+            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+                AdRankItem adRankItem = new AdRankItem();
+                adRankItem.setAdId(dto.getCreativeId());
+                adRankItem.setCreativeCode(dto.getCreativeCode());
+                adRankItem.setAdVerId(dto.getAdVerId());
+                adRankItem.setVideoId(request.getVideoId());
+                adRankItem.setCpa(dto.getCpa());
+                adRankItem.setId(dto.getAdId());
+                adRankItem.setCampaignId(dto.getCampaignId());
+                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                adRankItem.setRandom(random.nextInt(1000));
 
-            String cidStr = dto.getCreativeId().toString();
-            Map<String, String> cidFeatureMap = new HashMap<>();
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                String cidStr = dto.getCreativeId().toString();
+                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
 
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
 
-            Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
 
-            this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
 
-            this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
 
-            this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
 
-            this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
 
-            this.handleD1Feature(d1Feature, cidFeatureMap);
+                this.handleD1Feature(d1Feature, cidFeatureMap);
 
-            this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
 
-            String title = b1Feature.getOrDefault("cidtitle", "");
-            this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
+                // adRankItem.setFeatureMap(cidFeatureMap);
 
-            this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
+                adRankItems.add(adRankItem);
 
-            adRankItem.setFeatureMap(cidFeatureMap);
+            }
+
+
+            CountDownLatch cdl = new CountDownLatch(adRankItems.size());
+            for (AdRankItem item : adRankItems) {
+                ThreadPoolFactory.defaultPool().submit(() -> {
+                    try {
+                        String cidStr = String.valueOf(item.getAdId());
+                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                        String title = b1Feature.getOrDefault("cidtitle", "");
+                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                    } finally{
+                        cdl.countDown();
+                    }
+                });
+            }
+            try {
+                cdl.await(200, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+            }
+        } else {
+            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+                AdRankItem adRankItem = new AdRankItem();
+                adRankItem.setAdId(dto.getCreativeId());
+                adRankItem.setCreativeCode(dto.getCreativeCode());
+                adRankItem.setAdVerId(dto.getAdVerId());
+                adRankItem.setVideoId(request.getVideoId());
+                adRankItem.setCpa(dto.getCpa());
+                adRankItem.setId(dto.getAdId());
+                adRankItem.setCampaignId(dto.getCampaignId());
+                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                adRankItem.setRandom(random.nextInt(1000));
+
+                String cidStr = dto.getCreativeId().toString();
+                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+
+                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
 
-            adRankItems.add(adRankItem);
+                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
 
+                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+
+                this.handleD1Feature(d1Feature, cidFeatureMap);
+
+                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+
+                String title = b1Feature.getOrDefault("cidtitle", "");
+                this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
+                this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
+
+                // adRankItem.setFeatureMap(cidFeatureMap);
+
+                adRankItems.add(adRankItem);
+
+            }
         }
 
+        long time2 = System.currentTimeMillis();
         // 分桶
         this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
@@ -115,9 +189,13 @@ public class RankService683 extends RankStrategyXGBBasic {
             adRankItem.setFeatureMap(this.featureBucket(featureMap));
         }
 
+
+
         // 打分排序
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_683)
                 .scoring(sceneFeatureMap, userFeatureMap, adRankItems);
+
+        long time3 = System.currentTimeMillis();
         for (AdRankItem item : result) {
             item.setScore(item.getLrScore() * item.getCpa());
             item.getScoreMap().put("cpa", item.getCpa());
@@ -157,6 +235,10 @@ public class RankService683 extends RankStrategyXGBBasic {
             }
         }
 
+        long time4 = System.currentTimeMillis();
+        log.info("cost={}, feature1={}, feature2={}, getScorerPipeline={}, loop={}",
+                time4 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3);
+
         result.sort(ComparatorUtil.equalsRandomComparator());
 
         return result;
@@ -511,7 +593,7 @@ public class RankService683 extends RankStrategyXGBBasic {
     }
 
     private Map<String, String> featureBucket(Map<String, String> featureMap) {
-        Map<String, String> newFeatureMap = new HashMap<>(featureMap.size());
+        Map<String, String> newFeatureMap = new ConcurrentHashMap<>(featureMap.size());
         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
             String name = entry.getKey();
             double score = Double.parseDouble(entry.getValue());

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService687.java

@@ -3,16 +3,16 @@ package com.tzld.piaoquan.ad.engine.service.score;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
-import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.xm.Similarity;
 
@@ -21,6 +21,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -34,6 +37,9 @@ public class RankService687 extends RankStrategyXGBBasic {
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
+    @Value("${similarity.concurrent: false}")
+    private boolean similarityConcurrent;
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
@@ -44,6 +50,7 @@ public class RankService687 extends RankStrategyXGBBasic {
             scoreParam.getRequestContext().setMachineinfoBrand(brand + "-n");
         }
 
+        long start = System.currentTimeMillis();
         // 特征处理
         Feature feature = this.getFeature(scoreParam, request);
 
@@ -68,51 +75,131 @@ public class RankService687 extends RankStrategyXGBBasic {
         Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
 
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
+        long time1 = System.currentTimeMillis();
 
         List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
-        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-            AdRankItem adRankItem = new AdRankItem();
-            adRankItem.setAdId(dto.getCreativeId());
-            adRankItem.setCreativeCode(dto.getCreativeCode());
-            adRankItem.setAdVerId(dto.getAdVerId());
-            adRankItem.setVideoId(request.getVideoId());
-            adRankItem.setCpa(dto.getCpa());
-            adRankItem.setId(dto.getAdId());
-            adRankItem.setCampaignId(dto.getCampaignId());
-            adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+        Random random = new Random();
+        if (similarityConcurrent) {
+            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+                AdRankItem adRankItem = new AdRankItem();
+                adRankItem.setAdId(dto.getCreativeId());
+                adRankItem.setCreativeCode(dto.getCreativeCode());
+                adRankItem.setAdVerId(dto.getAdVerId());
+                adRankItem.setVideoId(request.getVideoId());
+                adRankItem.setCpa(dto.getCpa());
+                adRankItem.setId(dto.getAdId());
+                adRankItem.setCampaignId(dto.getCampaignId());
+                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                adRankItem.setRandom(random.nextInt(1000));
+
+                String cidStr = dto.getCreativeId().toString();
+                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+
+                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+
+                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
 
-            String cidStr = dto.getCreativeId().toString();
-            Map<String, String> cidFeatureMap = new HashMap<>();
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
 
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+                this.handleD1Feature(d1Feature, cidFeatureMap);
 
-            Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
 
-            this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
 
-            this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
 
-            this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                // adRankItem.setFeatureMap(cidFeatureMap);
 
-            this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                adRankItems.add(adRankItem);
+
+            }
+
+
+            CountDownLatch cdl = new CountDownLatch(adRankItems.size());
+            for (AdRankItem item : adRankItems) {
+                ThreadPoolFactory.defaultPool().submit(() -> {
+                    try {
+                        String cidStr = String.valueOf(item.getAdId());
+                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                        String title = b1Feature.getOrDefault("cidtitle", "");
+                        long time21 = System.currentTimeMillis();
+                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                        long time22 = System.currentTimeMillis();
+                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                        long time23 = System.currentTimeMillis();
+                        log.info("cost={} handleE1AndE2Feature={} handleD3AndB1Feature={}", time23 - time21, time22 - time21, time23 - time22);
+                    } finally{
+                        cdl.countDown();
+                    }
+                });
+            }
+            try {
+                cdl.await(200, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+            }
+        } else {
+            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+                long time20 = System.currentTimeMillis();
+                AdRankItem adRankItem = new AdRankItem();
+                adRankItem.setAdId(dto.getCreativeId());
+                adRankItem.setCreativeCode(dto.getCreativeCode());
+                adRankItem.setAdVerId(dto.getAdVerId());
+                adRankItem.setVideoId(request.getVideoId());
+                adRankItem.setCpa(dto.getCpa());
+                adRankItem.setId(dto.getAdId());
+                adRankItem.setCampaignId(dto.getCampaignId());
+                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                adRankItem.setRandom(random.nextInt(1000));
 
-            this.handleD1Feature(d1Feature, cidFeatureMap);
+                String cidStr = dto.getCreativeId().toString();
+                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
 
-            this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
 
-            String title = b1Feature.getOrDefault("cidtitle", "");
-            this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
+                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
 
-            this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
+                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
 
-            adRankItem.setFeatureMap(cidFeatureMap);
+                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
 
-            adRankItems.add(adRankItem);
+                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
 
+                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+
+                this.handleD1Feature(d1Feature, cidFeatureMap);
+
+                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+
+                String title = b1Feature.getOrDefault("cidtitle", "");
+                long time21 = System.currentTimeMillis();
+                this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
+                long time22 = System.currentTimeMillis();
+                this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
+                long time23 = System.currentTimeMillis();
+                log.info("cost={} other={} handleE1AndE2Feature={} handleD3AndB1Feature={}", time23 - time20,
+                        time21 - time20, time22 - time21, time23 - time22);
+
+                // adRankItem.setFeatureMap(cidFeatureMap);
+
+                adRankItems.add(adRankItem);
+
+            }
         }
 
+        //
+
+        long time2 = System.currentTimeMillis();
         // 分桶
         this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
@@ -121,9 +208,13 @@ public class RankService687 extends RankStrategyXGBBasic {
             adRankItem.setFeatureMap(this.featureBucket(featureMap));
         }
 
+
+
         // 打分排序
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_683)
                 .scoring(sceneFeatureMap, userFeatureMap, adRankItems);
+
+        long time3 = System.currentTimeMillis();
         for (AdRankItem item : result) {
             item.setScore(item.getLrScore() * item.getCpa());
             item.getScoreMap().put("cpa", item.getCpa());
@@ -163,6 +254,10 @@ public class RankService687 extends RankStrategyXGBBasic {
             }
         }
 
+        long time4 = System.currentTimeMillis();
+        log.info("cost={}, feature1={}, feature2={}, getScorerPipeline={}, loop={}",
+                time4 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3);
+
         result.sort(ComparatorUtil.equalsRandomComparator());
 
         return result;
@@ -517,7 +612,7 @@ public class RankService687 extends RankStrategyXGBBasic {
     }
 
     private Map<String, String> featureBucket(Map<String, String> featureMap) {
-        Map<String, String> newFeatureMap = new HashMap<>(featureMap.size());
+        Map<String, String> newFeatureMap = new ConcurrentHashMap<>(featureMap.size());
         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
             String name = entry.getKey();
             double score = Double.parseDouble(entry.getValue());
@@ -536,5 +631,4 @@ public class RankService687 extends RankStrategyXGBBasic {
 
         return newFeatureMap;
     }
-
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankStrategyXGBAutoUpdateModel688.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
@@ -11,6 +12,7 @@ import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
@@ -19,6 +21,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -32,6 +37,9 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
+    @Value("${similarity.concurrent: false}")
+    private boolean similarityConcurrent;
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
@@ -42,6 +50,7 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
             scoreParam.getRequestContext().setMachineinfoBrand(brand + "-n");
         }
 
+        long start = System.currentTimeMillis();
         // 特征处理
         Feature feature = this.getFeature(scoreParam, request);
 
@@ -66,51 +75,117 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
         Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
 
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
+        long time1 = System.currentTimeMillis();
 
         List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
-        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-            AdRankItem adRankItem = new AdRankItem();
-            adRankItem.setAdId(dto.getCreativeId());
-            adRankItem.setCreativeCode(dto.getCreativeCode());
-            adRankItem.setAdVerId(dto.getAdVerId());
-            adRankItem.setVideoId(request.getVideoId());
-            adRankItem.setCpa(dto.getCpa());
-            adRankItem.setId(dto.getAdId());
-            adRankItem.setCampaignId(dto.getCampaignId());
-            adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+        Random random = new Random();
+        if (similarityConcurrent) {
+            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+                AdRankItem adRankItem = new AdRankItem();
+                adRankItem.setAdId(dto.getCreativeId());
+                adRankItem.setCreativeCode(dto.getCreativeCode());
+                adRankItem.setAdVerId(dto.getAdVerId());
+                adRankItem.setVideoId(request.getVideoId());
+                adRankItem.setCpa(dto.getCpa());
+                adRankItem.setId(dto.getAdId());
+                adRankItem.setCampaignId(dto.getCampaignId());
+                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                adRankItem.setRandom(random.nextInt(1000));
+
+                String cidStr = dto.getCreativeId().toString();
+                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+
+                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+
+                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+
+                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+
+                this.handleD1Feature(d1Feature, cidFeatureMap);
+
+                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
 
-            String cidStr = dto.getCreativeId().toString();
-            Map<String, String> cidFeatureMap = new HashMap<>();
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                // adRankItem.setFeatureMap(cidFeatureMap);
 
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+                adRankItems.add(adRankItem);
 
-            Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+            }
+
+
+            CountDownLatch cdl = new CountDownLatch(adRankItems.size());
+            for (AdRankItem item : adRankItems) {
+                ThreadPoolFactory.defaultPool().submit(() -> {
+                    try {
+                        String cidStr = String.valueOf(item.getAdId());
+                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                        String title = b1Feature.getOrDefault("cidtitle", "");
+                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                    } finally{
+                        cdl.countDown();
+                    }
+                });
+            }
+            try {
+                cdl.await(200, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+            }
+        } else {
+            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+                AdRankItem adRankItem = new AdRankItem();
+                adRankItem.setAdId(dto.getCreativeId());
+                adRankItem.setCreativeCode(dto.getCreativeCode());
+                adRankItem.setAdVerId(dto.getAdVerId());
+                adRankItem.setVideoId(request.getVideoId());
+                adRankItem.setCpa(dto.getCpa());
+                adRankItem.setId(dto.getAdId());
+                adRankItem.setCampaignId(dto.getCampaignId());
+                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                adRankItem.setRandom(random.nextInt(1000));
+
+                String cidStr = dto.getCreativeId().toString();
+                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
 
-            this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
 
-            this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
 
-            this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
 
-            this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
 
-            this.handleD1Feature(d1Feature, cidFeatureMap);
+                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
 
-            this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                this.handleD1Feature(d1Feature, cidFeatureMap);
 
-            String title = b1Feature.getOrDefault("cidtitle", "");
-            this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
+                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
 
-            this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
+                String title = b1Feature.getOrDefault("cidtitle", "");
+                this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
+                this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
 
-            adRankItem.setFeatureMap(cidFeatureMap);
+                // adRankItem.setFeatureMap(cidFeatureMap);
 
-            adRankItems.add(adRankItem);
+                adRankItems.add(adRankItem);
 
+            }
         }
 
+        long time2 = System.currentTimeMillis();
         // 分桶
         this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
@@ -121,6 +196,7 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
 
         // 打分排序
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20240909).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
+        long time3 = System.currentTimeMillis();
         for (AdRankItem item : result) {
             item.setScore(item.getLrScore() * item.getCpa());
             item.getScoreMap().put("cpa", item.getCpa());
@@ -160,6 +236,10 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
             }
         }
 
+        long time4 = System.currentTimeMillis();
+        log.info("cost={}, feature1={}, feature2={}, getScorerPipeline={}, loop={}",
+                time4 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3);
+
         result.sort(ComparatorUtil.equalsRandomComparator());
 
         return result;
@@ -486,7 +566,7 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
     }
 
     private Map<String, String> featureBucket(Map<String, String> featureMap) {
-        Map<String, String> newFeatureMap = new HashMap<>(featureMap.size());
+        Map<String, String> newFeatureMap = new ConcurrentHashMap<>(featureMap.size());
         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
             String name = entry.getKey();
             double score = Double.parseDouble(entry.getValue());