优化

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/thread/ThreadPoolFactory.java

@@ -27,8 +27,8 @@ public final class ThreadPoolFactory {
             new ThreadPoolExecutor.AbortPolicy());
 
     private final static ExecutorService FEATURE_OPTIMIZED = new CommonThreadPoolExecutor(
-            256,
-            256,
+            64,
+            64,
             0L, TimeUnit.SECONDS,
             new LinkedBlockingQueue<>(5000),
             new ThreadFactoryBuilder().setNameFormat("Optimized-%d").build(),
@@ -44,8 +44,8 @@ public final class ThreadPoolFactory {
             new ThreadPoolExecutor.AbortPolicy());
 
     private final static ExecutorService SCORE_OPTIMIZED = new CommonThreadPoolExecutor(
-            64,
-            64,
+            32,
+            32,
             0L, TimeUnit.SECONDS,
             new LinkedBlockingQueue<>(5000),
             new ThreadFactoryBuilder().setNameFormat("Score-Opt-%d").build(),

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -152,145 +152,302 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         Map<String, GuaranteeView> map = getGuaranteeViewMap(request, isGuaranteedFlow);
         Map<Long, CorrectCpaParam> correctCpaMap = getCorrectCpaParamMap(request, scoreParam, reqFeature);
         List<AdRankItem> adRankItems = new ArrayList<>();
-        Random random = new Random();
-        List<Future<AdRankItem>> futures = new ArrayList<>();
-        CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
 
         // 优化：根据开关选择线程池
         ExecutorService featurePool = optimizeSwitch ? ThreadPoolFactory.featureOptimized() : ThreadPoolFactory.feature();
 
-        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-            Future<AdRankItem> future = featurePool.submit(() -> {
-
-                AdRankItem adRankItem = new AdRankItem();
-                try {
-                    adRankItem.setAdId(dto.getCreativeId());
-                    adRankItem.setCreativeCode(dto.getCreativeCode());
-                    adRankItem.setAdVerId(dto.getAdVerId());
-                    adRankItem.setVideoId(request.getVideoId());
-                    adRankItem.setCpa(dto.getCpa());
-                    adRankItem.setId(dto.getAdId());
-                    adRankItem.setCampaignId(dto.getCampaignId());
-                    adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                    adRankItem.setSkuId(dto.getSkuId());
-                    adRankItem.setCustomerId(dto.getCustomerId());
-                    adRankItem.setProfession(dto.getProfession());
-
-                    // 优化：使用 ThreadLocalRandom 减少锁竞争
-                    if (optimizeSwitch) {
-                        adRankItem.setRandom(java.util.concurrent.ThreadLocalRandom.current().nextInt(1000));
-                    } else {
-                        adRankItem.setRandom(random.nextInt(1000));
+        if (optimizeSwitch) {
+            // 方案C：批量处理优化 - 将 N 个任务合并为 N/batchSize 个批次任务
+            int batchSize = 100;
+            List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
+            int totalSize = adIdList.size();
+            int batchCount = (totalSize + batchSize - 1) / batchSize;
+
+            CountDownLatch cdl1 = new CountDownLatch(batchCount);
+            List<Future<List<AdRankItem>>> batchFutures = new ArrayList<>();
+
+            for (int i = 0; i < totalSize; i += batchSize) {
+                int fromIndex = i;
+                int toIndex = Math.min(i + batchSize, totalSize);
+                List<AdPlatformCreativeDTO> batch = adIdList.subList(fromIndex, toIndex);
+
+                Future<List<AdRankItem>> future = featurePool.submit(() -> {
+                    List<AdRankItem> batchResult = new ArrayList<>(batch.size());
+                    try {
+                        for (AdPlatformCreativeDTO dto : batch) {
+                            AdRankItem adRankItem = new AdRankItem();
+                            adRankItem.setAdId(dto.getCreativeId());
+                            adRankItem.setCreativeCode(dto.getCreativeCode());
+                            adRankItem.setAdVerId(dto.getAdVerId());
+                            adRankItem.setVideoId(request.getVideoId());
+                            adRankItem.setCpa(dto.getCpa());
+                            adRankItem.setId(dto.getAdId());
+                            adRankItem.setCampaignId(dto.getCampaignId());
+                            adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                            adRankItem.setSkuId(dto.getSkuId());
+                            adRankItem.setCustomerId(dto.getCustomerId());
+                            adRankItem.setProfession(dto.getProfession());
+                            adRankItem.setRandom(ThreadLocalRandom.current().nextInt(1000));
+
+                            if (noApiAdVerIds.contains(dto.getAdVerId())) {
+                                adRankItem.getExt().put("isApi", "0");
+                            } else {
+                                adRankItem.getExt().put("isApi", "1");
+                            }
+                            adRankItem.getExt().put("recallsources", dto.getRecallSources());
+                            adRankItem.getExt().put("correctCpaMap", correctCpaMap.get(dto.getAdId()));
+                            adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
+                            setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow, reqFeature);
+
+                            String cidStr = dto.getCreativeId().toString();
+                            Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+                            Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(dto.getSkuId()), new HashMap<>());
+                            Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                            this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                            this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                            this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                            this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                            this.handleD1Feature(d1Feature, cidFeatureMap);
+                            this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                            this.handleH1AndH2Feature(skuFeature, adVerFeature, cidFeatureMap);
+                            cidFeatureMap.put("cid", dto.getCreativeId() != null ? String.valueOf(dto.getCreativeId()) : "");
+                            cidFeatureMap.put("adid", dto.getAdId() != null ? String.valueOf(dto.getAdId()) : "");
+                            cidFeatureMap.put("adverid", dto.getAdVerId() != null ? dto.getAdVerId() : "");
+                            cidFeatureMap.put("profession", dto.getProfession() != null ? dto.getProfession() : "");
+                            cidFeatureMap.put("category_name", dto.getCategoryName() != null ? dto.getCategoryName() : "");
+                            cidFeatureMap.put("material_md5", dto.getMaterialMd5() != null ? dto.getMaterialMd5() : "");
+
+                            if (CollectionUtils.isNotEmpty(DnnCidDataHelper.getCidSet()) && !DnnCidDataHelper.getCidSet().contains(adRankItem.getAdId())) {
+                                cidFeatureMap.put("cid", "");
+                                cidFeatureMap.put("adid", "");
+                                cidFeatureMap.put("adverid", "");
+                            }
+                            batchResult.add(adRankItem);
+                        }
+                    } finally {
+                        cdl1.countDown();
                     }
+                    return batchResult;
+                });
+                batchFutures.add(future);
+            }
 
-                    if (noApiAdVerIds.contains(dto.getAdVerId())) {
-                        adRankItem.getExt().put("isApi", "0");
-                    } else {
-                        adRankItem.getExt().put("isApi", "1");
+            try {
+                cdl1.await(500, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                log.error("handleFeature batch wait timeout", e);
+            }
+
+            for (Future<List<AdRankItem>> future : batchFutures) {
+                try {
+                    if (future.isDone()) {
+                        adRankItems.addAll(future.get());
                     }
-                    adRankItem.getExt().put("recallsources", dto.getRecallSources());
+                } catch (Exception e) {
+                    log.error("Feature batch handle error", e);
+                }
+            }
+        } else {
+            // 原逻辑：每个广告一个任务
+            Random random = new Random();
+            List<Future<AdRankItem>> futures = new ArrayList<>();
+            CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
+
+            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+                Future<AdRankItem> future = featurePool.submit(() -> {
+                    AdRankItem adRankItem = new AdRankItem();
+                    try {
+                        adRankItem.setAdId(dto.getCreativeId());
+                        adRankItem.setCreativeCode(dto.getCreativeCode());
+                        adRankItem.setAdVerId(dto.getAdVerId());
+                        adRankItem.setVideoId(request.getVideoId());
+                        adRankItem.setCpa(dto.getCpa());
+                        adRankItem.setId(dto.getAdId());
+                        adRankItem.setCampaignId(dto.getCampaignId());
+                        adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                        adRankItem.setSkuId(dto.getSkuId());
+                        adRankItem.setCustomerId(dto.getCustomerId());
+                        adRankItem.setProfession(dto.getProfession());
+                        adRankItem.setRandom(random.nextInt(1000));
 
-                    // 优化：直接存对象，避免循环内 JSON 序列化
-                    if (optimizeSwitch) {
-                        adRankItem.getExt().put("correctCpaMap", correctCpaMap.get(dto.getAdId()));
-                    } else {
+                        if (noApiAdVerIds.contains(dto.getAdVerId())) {
+                            adRankItem.getExt().put("isApi", "0");
+                        } else {
+                            adRankItem.getExt().put("isApi", "1");
+                        }
+                        adRankItem.getExt().put("recallsources", dto.getRecallSources());
                         adRankItem.getExt().put("correctCpaMap", JSONObject.toJSONString(correctCpaMap.get(dto.getAdId())));
+                        adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
+                        setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow, reqFeature);
+
+                        String cidStr = dto.getCreativeId().toString();
+                        Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                        Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+                        Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(dto.getSkuId()), new HashMap<>());
+                        Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                        this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                        this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                        this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                        this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                        this.handleD1Feature(d1Feature, cidFeatureMap);
+                        this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                        this.handleH1AndH2Feature(skuFeature, adVerFeature, cidFeatureMap);
+                        cidFeatureMap.put("cid", dto.getCreativeId() != null ? String.valueOf(dto.getCreativeId()) : "");
+                        cidFeatureMap.put("adid", dto.getAdId() != null ? String.valueOf(dto.getAdId()) : "");
+                        cidFeatureMap.put("adverid", dto.getAdVerId() != null ? dto.getAdVerId() : "");
+                        cidFeatureMap.put("profession", dto.getProfession() != null ? dto.getProfession() : "");
+                        cidFeatureMap.put("category_name", dto.getCategoryName() != null ? dto.getCategoryName() : "");
+                        cidFeatureMap.put("material_md5", dto.getMaterialMd5() != null ? dto.getMaterialMd5() : "");
+
+                        if (CollectionUtils.isNotEmpty(DnnCidDataHelper.getCidSet()) && !DnnCidDataHelper.getCidSet().contains(adRankItem.getAdId())) {
+                            cidFeatureMap.put("cid", "");
+                            cidFeatureMap.put("adid", "");
+                            cidFeatureMap.put("adverid", "");
+                        }
+                        return adRankItem;
+                    } finally {
+                        cdl1.countDown();
                     }
+                });
+                futures.add(future);
+            }
 
-                    adRankItem.getExt().put("correctionFactor", correctCpaMap.get(dto.getAdId()).getCorrectionFactor());
-                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow, reqFeature);
-                    String cidStr = dto.getCreativeId().toString();
-                    Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                    Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                    Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-                    Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-                    Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(dto.getSkuId()), new HashMap<>());
-                    Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-                    this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-                    this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-                    this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-                    this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-                    this.handleD1Feature(d1Feature, cidFeatureMap);
-                    this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
-                    this.handleH1AndH2Feature(skuFeature, adVerFeature, cidFeatureMap);
-                    cidFeatureMap.put("cid", dto.getCreativeId() != null ? String.valueOf(dto.getCreativeId()) : "");
-                    cidFeatureMap.put("adid", dto.getAdId() != null ? String.valueOf(dto.getAdId()) : "");
-                    cidFeatureMap.put("adverid", dto.getAdVerId() != null ? dto.getAdVerId() : "");
-                    cidFeatureMap.put("profession", dto.getProfession() != null ? dto.getProfession() : "");
-                    cidFeatureMap.put("category_name", dto.getCategoryName() != null ? dto.getCategoryName() : "");
-                    cidFeatureMap.put("material_md5", dto.getMaterialMd5() != null ? dto.getMaterialMd5() : "");
-                    //DNN模型没训练过的cid才不传入广告相关的稀疏特征
-                    if (CollectionUtils.isNotEmpty(DnnCidDataHelper.getCidSet()) && !DnnCidDataHelper.getCidSet().contains(adRankItem.getAdId())) {
-                        cidFeatureMap.put("cid", "");
-                        cidFeatureMap.put("adid", "");
-                        cidFeatureMap.put("adverid", "");
-                    }
-                    return adRankItem;
-                } finally {
-                    cdl1.countDown();
-                }
-            });
-            futures.add(future);
-        }
-        try {
-            cdl1.await(300, TimeUnit.MILLISECONDS);
-        } catch (Exception e) {
-            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
-        }
-        for (Future<AdRankItem> future : futures) {
             try {
-                if (future.isDone()) {
-                    adRankItems.add(future.get());
-                }
+                cdl1.await(300, TimeUnit.MILLISECONDS);
             } catch (Exception e) {
-                log.error("Feature handle error", e);
+                log.error("handleFeature wait timeout", e);
             }
-        }
 
-        long time2 = System.currentTimeMillis();
-        // feature3 - 优化：合并两个任务为一个，减少线程调度开销
-        CountDownLatch cdl2 = new CountDownLatch(adRankItems.size());
-        for (AdRankItem item : adRankItems) {
-            String cidStr = String.valueOf(item.getAdId());
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-            String title = b1Feature.getOrDefault("cidtitle", "");
-            ThreadPoolFactory.defaultPool().submit(() -> {
+            for (Future<AdRankItem> future : futures) {
                 try {
-                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap(), scoreParam);
-                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap(), scoreParam);
-                } finally {
-                    cdl2.countDown();
+                    if (future.isDone()) {
+                        adRankItems.add(future.get());
+                    }
+                } catch (Exception e) {
+                    log.error("Feature handle error", e);
                 }
-            });
+            }
         }
-        try {
-            cdl2.await(150, TimeUnit.MILLISECONDS);
-        } catch (Exception e) {
-            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+
+        long time2 = System.currentTimeMillis();
+        // feature3 - similar 阶段
+        if (optimizeSwitch) {
+            // 方案C：批量处理优化
+            int batchSize = 100;
+            int totalSize = adRankItems.size();
+            int batchCount = (totalSize + batchSize - 1) / batchSize;
+            CountDownLatch cdl2 = new CountDownLatch(batchCount);
+
+            for (int i = 0; i < totalSize; i += batchSize) {
+                int fromIndex = i;
+                int toIndex = Math.min(i + batchSize, totalSize);
+                List<AdRankItem> batch = adRankItems.subList(fromIndex, toIndex);
+
+                featurePool.submit(() -> {
+                    try {
+                        for (AdRankItem item : batch) {
+                            String cidStr = String.valueOf(item.getAdId());
+                            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                            String title = b1Feature.getOrDefault("cidtitle", "");
+                            this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap(), scoreParam);
+                            this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap(), scoreParam);
+                        }
+                    } finally {
+                        cdl2.countDown();
+                    }
+                });
+            }
+
+            try {
+                cdl2.await(300, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                log.error("similar batch wait timeout", e);
+            }
+        } else {
+            // 原逻辑
+            CountDownLatch cdl2 = new CountDownLatch(adRankItems.size());
+            for (AdRankItem item : adRankItems) {
+                String cidStr = String.valueOf(item.getAdId());
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                String title = b1Feature.getOrDefault("cidtitle", "");
+                ThreadPoolFactory.defaultPool().submit(() -> {
+                    try {
+                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap(), scoreParam);
+                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap(), scoreParam);
+                    } finally {
+                        cdl2.countDown();
+                    }
+                });
+            }
+            try {
+                cdl2.await(150, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+            }
         }
 
         long time3 = System.currentTimeMillis();
         // 分桶
         userFeatureMap = this.featureBucket(userFeatureMap);
-        CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
-        for (AdRankItem adRankItem : adRankItems) {
-            featurePool.submit(() -> {
-                try {
-                    Map<String, String> featureMap = adRankItem.getFeatureMap();
-                    adRankItem.setFeatureMap(this.featureBucket(featureMap));
-                } finally {
-                    cdl4.countDown();
-                }
-            });
-        }
-        try {
-            cdl4.await(100, TimeUnit.MILLISECONDS);
-        } catch (Exception e) {
-            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+
+        if (optimizeSwitch) {
+            // 方案C：批量处理优化
+            int batchSize = 100;
+            int totalSize = adRankItems.size();
+            int batchCount = (totalSize + batchSize - 1) / batchSize;
+            CountDownLatch cdl4 = new CountDownLatch(batchCount);
+
+            for (int i = 0; i < totalSize; i += batchSize) {
+                int fromIndex = i;
+                int toIndex = Math.min(i + batchSize, totalSize);
+                List<AdRankItem> batch = adRankItems.subList(fromIndex, toIndex);
+
+                featurePool.submit(() -> {
+                    try {
+                        for (AdRankItem adRankItem : batch) {
+                            Map<String, String> featureMap = adRankItem.getFeatureMap();
+                            adRankItem.setFeatureMap(this.featureBucket(featureMap));
+                        }
+                    } finally {
+                        cdl4.countDown();
+                    }
+                });
+            }
+
+            try {
+                cdl4.await(200, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                log.error("bucketFeature batch wait timeout", e);
+            }
+        } else {
+            // 原逻辑
+            CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
+            for (AdRankItem adRankItem : adRankItems) {
+                featurePool.submit(() -> {
+                    try {
+                        Map<String, String> featureMap = adRankItem.getFeatureMap();
+                        adRankItem.setFeatureMap(this.featureBucket(featureMap));
+                    } finally {
+                        cdl4.countDown();
+                    }
+                });
+            }
+            try {
+                cdl4.await(100, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                log.error("bucketFeature wait timeout", e);
+            }
         }
+
         long time4 = System.currentTimeMillis();
         // 打分排序
         // getScorerPipeline