ad cost

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankStrategyXGBAutoUpdateModel688.java

@@ -12,6 +12,7 @@ import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
@@ -20,10 +21,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -37,8 +35,220 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
+    @Value("${similarity.concurrent: false}")
+    private boolean similarityConcurrent;
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+        if (similarityConcurrent) {
+            return adItemRankNew(request, scoreParam);
+        } else {
+            return adItemRankOld(request, scoreParam);
+        }
+    }
+
+    private List<AdRankItem> adItemRankNew(RankRecommendRequestParam request, ScoreParam scoreParam) {
+
+        long ts = System.currentTimeMillis() / 1000;
+
+        String brand = scoreParam.getRequestContext().getMachineinfoBrand();
+        if (StringUtils.isNotEmpty(brand)) {
+            scoreParam.getRequestContext().setMachineinfoBrand(brand + "-n");
+        }
+
+        long start = System.currentTimeMillis();
+        // 特征处理
+        // feature1
+        Feature feature = this.getFeature(scoreParam, request);
+
+        Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+        Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
+        Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
+        Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+
+        Map<String, String> userFeatureMap = new HashMap<>();
+        Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
+        List<TupleMapEntry<Tuple5>> midActionList = this.handleC1Feature(c1Feature, userFeatureMap);
+
+        Map<String, Double> midTimeDiffMap = this.parseC1FeatureListToTimeDiffMap(midActionList, ts);
+        Map<String, Double> actionStaticMap = this.parseC1FeatureListToActionStaticMap(midActionList);
+
+        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
+        Map<String, String> d3Feature = videoFeature.getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+
+        Map<String, Map<String, Double>> vidRankMaps = this.parseD2FeatureMap(d2Feature);
+
+        Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
+        Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+
+        Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
+        long time1 = System.currentTimeMillis();
+
+        List<AdRankItem> adRankItems = new ArrayList<>();
+        Random random = new Random();
+        List<Callable<AdRankItem>> task1s = new ArrayList<>();
+        for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
+            task1s.add(() -> {
+                AdRankItem adRankItem = new AdRankItem();
+                adRankItem.setAdId(dto.getCreativeId());
+                adRankItem.setCreativeCode(dto.getCreativeCode());
+                adRankItem.setAdVerId(dto.getAdVerId());
+                adRankItem.setVideoId(request.getVideoId());
+                adRankItem.setCpa(dto.getCpa());
+                adRankItem.setId(dto.getAdId());
+                adRankItem.setCampaignId(dto.getCampaignId());
+                adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                adRankItem.setRandom(random.nextInt(1000));
+
+                String cidStr = dto.getCreativeId().toString();
+                Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+
+                Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
+
+                Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+
+                this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                this.handleD1Feature(d1Feature, cidFeatureMap);
+                this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                return adRankItem;
+            });
+        }
+        try {
+            List<Future<AdRankItem>> futures = ThreadPoolFactory.feature().invokeAll(task1s, 100,
+                    TimeUnit.MILLISECONDS);
+            for (Future<AdRankItem> future : futures) {
+                try {
+                    if (future.isDone()) {
+                        adRankItems.add(future.get());
+                    }
+                } catch (Exception e) {
+                    log.error("Feature handle error", e);
+                }
+            }
+        } catch (Exception e) {
+            log.error("handle feature error", e);
+            return Collections.emptyList();
+        }
+
+
+        long time2 = System.currentTimeMillis();
+        // feature3
+        List<Future<String>> futures = new ArrayList<>();
+        CountDownLatch cdl2 = new CountDownLatch(adRankItems.size() * 2);
+        for (AdRankItem item : adRankItems) {
+            String cidStr = String.valueOf(item.getAdId());
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+            String title = b1Feature.getOrDefault("cidtitle", "");
+            futures.add(ThreadPoolFactory.defaultPool().submit(() -> {
+                try {
+                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap());
+                    return "";
+                } finally {
+                    cdl2.countDown();
+                }
+            }));
+            futures.add(ThreadPoolFactory.defaultPool().submit(() -> {
+                try {
+                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
+                    return "";
+                } finally {
+                    cdl2.countDown();
+                }
+            }));
+        }
+        try {
+            cdl2.await(150, TimeUnit.MILLISECONDS);
+            for (Future<String> future : futures) {
+                if (!future.isDone()) {
+                    future.cancel(true);
+                }
+            }
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+
+        long time3 = System.currentTimeMillis();
+        // 分桶
+        this.readBucketFile();
+        userFeatureMap = this.featureBucket(userFeatureMap);
+        CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
+        for (AdRankItem adRankItem : adRankItems) {
+            ThreadPoolFactory.feature().submit(() -> {
+                try {
+                    Map<String, String> featureMap = adRankItem.getFeatureMap();
+                    adRankItem.setFeatureMap(this.featureBucket(featureMap));
+                } finally {
+                    cdl4.countDown();
+                }
+            });
+        }
+        try {
+            cdl4.await(100, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+        long time4 = System.currentTimeMillis();
+        // 打分排序
+        // getScorerPipeline
+        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20240909).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
+        long time5 = System.currentTimeMillis();
+        // loop
+        for (AdRankItem item : result) {
+            item.setScore(item.getLrScore() * item.getCpa());
+            item.getScoreMap().put("cpa", item.getCpa());
+            item.getScoreMap().put("cpm", item.getCpm());
+            item.getFeatureMap().putAll(userFeatureMap);
+            item.getFeatureMap().putAll(sceneFeatureMap);
+
+            // 没有转化回传的广告主，使用后台配置的CPM
+            if (noPostbackConversionAdVerIds.contains(item.getAdVerId())) {
+                item.setScore(item.getCpm() / 1000);
+            }
+
+            for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
+
+            for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
+
+            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(item.getAdVerId(), new HashMap<>());
+            for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
+
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
+            for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
+        }
+
+        log.info("cost={}, feature1={}, feature2={}, feature3={}, feature4={}, getScorerPipeline={}, " +
+                        "adIdSize={}, adRankItemsSize={}",
+                time5 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3,
+                time5 - time4, request.getAdIdList().size(), adRankItems.size());
+
+        result.sort(ComparatorUtil.equalsRandomComparator());
+
+        return result;
+    }
+
+    private List<AdRankItem> adItemRankOld(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
         long ts = System.currentTimeMillis() / 1000;