ad cost

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/thread/ThreadPoolFactory.java

@@ -25,13 +25,6 @@ public final class ThreadPoolFactory {
             new LinkedBlockingQueue<>(1000),
             new ThreadFactoryBuilder().setNameFormat("Feature-%d").build(),
             new ThreadPoolExecutor.AbortPolicy());
-    public final static ExecutorService RECALL = new CommonThreadPoolExecutor(
-            128,
-            128,
-            0L, TimeUnit.SECONDS,
-            new LinkedBlockingQueue<>(1000),
-            new ThreadFactoryBuilder().setNameFormat("RecallService-%d").build(),
-            new ThreadPoolExecutor.AbortPolicy());
 
     public static ExecutorService defaultPool() {
         return DEFAULT;
@@ -41,8 +34,4 @@ public final class ThreadPoolFactory {
         return FEATURE;
     }
 
-    public static ExecutorService recallPool() {
-        return RECALL;
-    }
-
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankStrategyXGBAutoUpdateModel688.java

@@ -16,6 +16,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
+import javax.annotation.PostConstruct;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -41,6 +42,11 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
     @Value("${similarity.concurrent: false}")
     private boolean similarityConcurrent;
 
+//    @PostConstruct
+//    public void init() {
+//        readBucketFile();
+//    }
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
@@ -81,176 +87,129 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
 
         List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
-        long time2 = 0;
-        // feature2
-        if (similarityConcurrent) {
-            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+        List<Future<AdRankItem>> futures = new ArrayList<>();
+        CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
+        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+            Future<AdRankItem> future = ThreadPoolFactory.feature().submit(() -> {
                 AdRankItem adRankItem = new AdRankItem();
-                adRankItem.setAdId(dto.getCreativeId());
-                adRankItem.setCreativeCode(dto.getCreativeCode());
-                adRankItem.setAdVerId(dto.getAdVerId());
-                adRankItem.setVideoId(request.getVideoId());
-                adRankItem.setCpa(dto.getCpa());
-                adRankItem.setId(dto.getAdId());
-                adRankItem.setCampaignId(dto.getCampaignId());
-                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                adRankItem.setRandom(random.nextInt(1000));
-
-                String cidStr = dto.getCreativeId().toString();
-                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
-                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-
-                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-
-                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-
-                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-
-                this.handleD1Feature(d1Feature, cidFeatureMap);
-
-                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
-
+                try {
+                    long time20 = System.currentTimeMillis();
+                    adRankItem.setAdId(dto.getCreativeId());
+                    adRankItem.setCreativeCode(dto.getCreativeCode());
+                    adRankItem.setAdVerId(dto.getAdVerId());
+                    adRankItem.setVideoId(request.getVideoId());
+                    adRankItem.setCpa(dto.getCpa());
+                    adRankItem.setId(dto.getAdId());
+                    adRankItem.setCampaignId(dto.getCampaignId());
+                    adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setRandom(random.nextInt(1000));
+
+                    String cidStr = dto.getCreativeId().toString();
+                    Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                    Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                    Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+                    Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+                    Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                    long time21 = System.currentTimeMillis();
+                    this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                    long time22 = System.currentTimeMillis();
+                    this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                    long time23 = System.currentTimeMillis();
+                    this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                    long time24 = System.currentTimeMillis();
+                    this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                    long time25 = System.currentTimeMillis();
+                    this.handleD1Feature(d1Feature, cidFeatureMap);
+                    long time26 = System.currentTimeMillis();
+                    this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                    long time27 = System.currentTimeMillis();
+
+                    log.info("cost={}, other={}, handleB1Feature={}, handleB2ToB5AndB8ToB9Feature={}, " +
+                                    "handleB6ToB7Feature={}, " +
+                                    "handleC1UIFeature={}, handleD1Feature={}, handleD2Feature={}",
+                            time27 - time20, time21 - time20, time22 - time21,
+                            time23 - time22, time24 - time23, time25 - time24, time26 - time25, time27 - time26);
+                    return adRankItem;
+                } finally {
+                    cdl1.countDown();
+                }
                 // adRankItem.setFeatureMap(cidFeatureMap);
-
-                adRankItems.add(adRankItem);
-
-            }
-
-            time2 = System.currentTimeMillis();
-            // feature3
-            CountDownLatch cdl = new CountDownLatch(adRankItems.size());
-            for (AdRankItem item : adRankItems) {
-                ThreadPoolFactory.defaultPool().submit(() -> {
-                    try {
-                        String cidStr = String.valueOf(item.getAdId());
-                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-                        String title = b1Feature.getOrDefault("cidtitle", "");
-                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
-                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
-                    } finally {
-                        cdl.countDown();
-                    }
-                });
-            }
+            });
+            futures.add(future);
+        }
+        try {
+            cdl1.await(300, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+        for (Future<AdRankItem> future : futures) {
             try {
-                cdl.await(200, TimeUnit.MILLISECONDS);
+                if (future.isDone()) {
+                    adRankItems.add(future.get());
+                }
             } catch (Exception e) {
-                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+                log.error("Feature handle error", e);
             }
-        } else {
+        }
 
-            List<Future<AdRankItem>> futures = new ArrayList<>();
-            CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
-            for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-                Future<AdRankItem> future = ThreadPoolFactory.feature().submit(() -> {
-                    AdRankItem adRankItem = new AdRankItem();
-                    try {
-                        long time20 = System.currentTimeMillis();
-                        adRankItem.setAdId(dto.getCreativeId());
-                        adRankItem.setCreativeCode(dto.getCreativeCode());
-                        adRankItem.setAdVerId(dto.getAdVerId());
-                        adRankItem.setVideoId(request.getVideoId());
-                        adRankItem.setCpa(dto.getCpa());
-                        adRankItem.setId(dto.getAdId());
-                        adRankItem.setCampaignId(dto.getCampaignId());
-                        adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-                        adRankItem.setRandom(random.nextInt(1000));
-
-                        String cidStr = dto.getCreativeId().toString();
-                        Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-                        Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
-                        Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-                        long time21 = System.currentTimeMillis();
-                        this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-                        long time22 = System.currentTimeMillis();
-                        this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
-                        long time23 = System.currentTimeMillis();
-                        this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
-                        long time24 = System.currentTimeMillis();
-                        this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
-                        long time25 = System.currentTimeMillis();
-                        this.handleD1Feature(d1Feature, cidFeatureMap);
-                        long time26 = System.currentTimeMillis();
-                        this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
-                        long time27 = System.currentTimeMillis();
-
-                        log.info("cost={}, other={}, handleB1Feature={}, handleB2ToB5AndB8ToB9Feature={}, " +
-                                        "handleB6ToB7Feature={}, " +
-                                        "handleC1UIFeature={}, handleD1Feature={}, handleD2Feature={}",
-                                time27 - time20, time21 - time20,time22 - time21,
-                                time23 - time22, time24 - time23, time25 - time24, time26 - time25, time27 - time26);
-                        return adRankItem;
-                    } finally {
-                        cdl1.countDown();
-                    }
-                    // adRankItem.setFeatureMap(cidFeatureMap);
-                });
-                futures.add(future);
-            }
-            try {
-                cdl1.await(300, TimeUnit.MILLISECONDS);
-            } catch (Exception e) {
-                log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
-            }
-            for (Future<AdRankItem> future : futures) {
+        long time2 = System.currentTimeMillis();
+        // feature3
+        CountDownLatch cdl2 = new CountDownLatch(adRankItems.size());
+        for (AdRankItem item : adRankItems) {
+            ThreadPoolFactory.defaultPool().submit(() -> {
                 try {
-                    if (future.isDone()) {
-                        adRankItems.add(future.get());
-                    }
-                } catch (Exception e) {
-                    log.error("Feature handle error", e);
+                    String cidStr = String.valueOf(item.getAdId());
+                    Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                    Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                    String title = b1Feature.getOrDefault("cidtitle", "");
+                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                } finally {
+                    cdl2.countDown();
                 }
-            }
+            });
+        }
+        try {
+            cdl2.await(200, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
 
-            time2 = System.currentTimeMillis();
-            // feature3
-            CountDownLatch cdl2 = new CountDownLatch(adRankItems.size());
-            for (AdRankItem item : adRankItems) {
-                ThreadPoolFactory.defaultPool().submit(() -> {
+        // feature4
+        long time3 = System.currentTimeMillis();
+        // 分桶
+        if (similarityConcurrent) {
+            this.readBucketFile();
+            userFeatureMap = this.featureBucket(userFeatureMap);
+            CountDownLatch cdl3 = new CountDownLatch(adRankItems.size());
+            for (AdRankItem adRankItem : adRankItems) {
+                ThreadPoolFactory.feature().submit(() -> {
                     try {
-                        String cidStr = String.valueOf(item.getAdId());
-                        Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                        Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-                        String title = b1Feature.getOrDefault("cidtitle", "");
-                        this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
-                        this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
-                    } finally {
-                        cdl2.countDown();
+                        Map<String, String> featureMap = adRankItem.getFeatureMap();
+                        adRankItem.setFeatureMap(this.featureBucket(featureMap));
+                    } finally{
+                        cdl3.countDown();
                     }
                 });
             }
             try {
-                cdl2.await(200, TimeUnit.MILLISECONDS);
+                cdl3.await(100, TimeUnit.MILLISECONDS);
             } catch (Exception e) {
                 log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
             }
-        }
-
 
-        // feature4
-        long time3 = System.currentTimeMillis();
-        // 分桶
-        this.readBucketFile();
-        long time41 = System.currentTimeMillis();
-        userFeatureMap = this.featureBucket(userFeatureMap);
-        for (AdRankItem adRankItem : adRankItems) {
-            Map<String, String> featureMap = adRankItem.getFeatureMap();
-            adRankItem.setFeatureMap(this.featureBucket(featureMap));
-        }
-        long time42 = System.currentTimeMillis();
+        } else {
+            this.readBucketFile();
+            userFeatureMap = this.featureBucket(userFeatureMap);
+            for (AdRankItem adRankItem : adRankItems) {
+                Map<String, String> featureMap = adRankItem.getFeatureMap();
+                adRankItem.setFeatureMap(this.featureBucket(featureMap));
+            }
+        }
+        long time4 = System.currentTimeMillis();
         // 打分排序
         // getScorerPipeline
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20240909).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
@@ -296,10 +255,9 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
         }
 
         long time6 = System.currentTimeMillis();
-        log.info("cost={}, feature1={}, feature2={}, feature3={}, feature41={}, feature42={}， getScorerPipeline={}, " +
+        log.info("cost={}, feature1={}, feature2={}, feature3={}, feature4={}, getScorerPipeline={}, " +
                         "loop={}, adIdSize={}, adRankItemsSize={}",
-                time6 - start, time1 - start, time2 - time1, time3 - time2, time41 - time3,
-                time42 - time41, time5 - time42,
+                time6 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3, time5 - time4,
                 time6 - time5, request.getAdIdList().size(), adRankItems.size());
 
         result.sort(ComparatorUtil.equalsRandomComparator());