Merge branch 'feature_20240925_zhaohaipeng_ad_time' of algorithm/ad-engine into master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/enums/RedisPrefixEnum.java

@@ -0,0 +1,24 @@
+package com.tzld.piaoquan.ad.engine.commons.enums;
+
+public enum RedisPrefixEnum {
+
+    ADVANCE_SHOW_AD_FLAG("ad:advance:show:ad:flag:%s", "是否提前出广告标识,0-否；1-是"),
+    ADVER_IS_API_EQ_0_IDS("ad:adver:isapi:0", "未回传广告主ID集合"),
+    ;
+    private String prefix;
+    private String desc;
+
+    RedisPrefixEnum(String prefix, String desc) {
+        this.prefix = prefix;
+        this.desc = desc;
+    }
+
+    public String getPrefix() {
+        return prefix;
+    }
+
+    public String getDesc() {
+        return desc;
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/redis/AdRedisHelper.java

@@ -0,0 +1,54 @@
+package com.tzld.piaoquan.ad.engine.commons.redis;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+@Component
+public class AdRedisHelper {
+    private final static Logger log = LoggerFactory.getLogger(AdRedisHelper.class);
+
+    @Resource(name = "adRedisTemplate")
+    private RedisTemplate<String, String> adRedisTemplate;
+
+    public Set<String> sMembers(String key) {
+        return adRedisTemplate.opsForSet().members(key);
+    }
+
+    public boolean set(String key, String value, long time) {
+        try {
+            if (time > 0) {
+                adRedisTemplate.opsForValue().set(key, value, time, TimeUnit.SECONDS);
+            } else {
+                adRedisTemplate.opsForValue().set(key, value);
+            }
+            return true;
+        } catch (Exception e) {
+            log.error("adRedisHelper set error", e);
+            return false;
+        }
+    }
+
+    public String get(String key) {
+        String val = adRedisTemplate.opsForValue().get(key);
+        if (StringUtils.isNotBlank(val)) {
+            return val;
+        }
+        return null;
+    }
+
+    public boolean del(String key) {
+        return Boolean.TRUE.equals(adRedisTemplate.delete(key));
+    }
+
+    public boolean containsKey(String key) {
+        return Boolean.TRUE.equals(adRedisTemplate.hasKey(key));
+    }
+
+}

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/redis/AlgorithmRedisHelper.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.ad.engine.commons.redis;
 
-import com.google.common.collect.Collections2;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/redis/RedisTemplateConfig.java

@@ -44,12 +44,30 @@ public class RedisTemplateConfig {
         return new RedisStandaloneConfiguration();
     }
 
+    @Primary
     @Bean("algorithmRedisFactory")
-    public LettuceConnectionFactory algorithmRedisFactory(GenericObjectPoolConfig<LettucePoolingClientConfiguration> config, RedisStandaloneConfiguration algorithmRedisConfig) {
+    public LettuceConnectionFactory algorithmRedisFactory(GenericObjectPoolConfig<LettucePoolingClientConfiguration> config,@Qualifier("algorithmRedisConfig") RedisStandaloneConfiguration algorithmRedisConfig) {
         LettuceClientConfiguration lettuceClientConfiguration = LettucePoolingClientConfiguration.builder().poolConfig(config).build();
         return new LettuceConnectionFactory(algorithmRedisConfig, lettuceClientConfiguration);
     }
 
+    @Bean(name = "adRedisTemplate")
+    public RedisTemplate<String, String> adRedisTemplate(@Qualifier("adRedisFactory") RedisConnectionFactory adRedisFactory) {
+        return buildRedisTemplateByString(adRedisFactory);
+    }
+
+    @Bean
+    @ConfigurationProperties(prefix = "spring.redis-ad")
+    public RedisStandaloneConfiguration adRedisConfig() {
+        return new RedisStandaloneConfiguration();
+    }
+
+    @Bean("adRedisFactory")
+    public LettuceConnectionFactory adRedisFactory(GenericObjectPoolConfig<LettucePoolingClientConfiguration> config,@Qualifier("adRedisConfig") RedisStandaloneConfiguration adRedisConfig) {
+        LettuceClientConfiguration lettuceClientConfiguration = LettucePoolingClientConfiguration.builder().poolConfig(config).build();
+        return new LettuceConnectionFactory(adRedisConfig, lettuceClientConfiguration);
+    }
+
     /**
      * 构建redisTemplate 使用string序列化
      *

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/impl/PredictModelServiceImpl.java

@@ -4,14 +4,15 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.alibaba.fastjson.TypeReference;
 import com.tzld.piaoquan.ad.engine.commons.enums.AppTypeEnum;
+import com.tzld.piaoquan.ad.engine.commons.enums.RedisPrefixEnum;
+import com.tzld.piaoquan.ad.engine.commons.redis.AdRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
-import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import com.tzld.piaoquan.ad.engine.service.predict.PredictModelService;
+import com.tzld.piaoquan.ad.engine.service.predict.config.AbConfig;
 import com.tzld.piaoquan.ad.engine.service.predict.config.NewExpUserGroupConfig;
 import com.tzld.piaoquan.ad.engine.service.predict.config.RoiModelConfig;
 import com.tzld.piaoquan.ad.engine.service.predict.constant.RuleRedisKeyConst;
-import com.tzld.piaoquan.ad.engine.service.predict.PredictModelService;
-import com.tzld.piaoquan.ad.engine.service.predict.config.AbConfig;
 import com.tzld.piaoquan.ad.engine.service.predict.container.AbTestConfigContainer;
 import com.tzld.piaoquan.ad.engine.service.predict.container.ThresholdModelContainer;
 import com.tzld.piaoquan.ad.engine.service.predict.container.TopOneVideoContainer;
@@ -20,6 +21,7 @@ import com.tzld.piaoquan.ad.engine.service.predict.param.RoiThresholdPredictMode
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.RoiPredictModelRequestParam;
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.ThresholdPredictModelRequestParam;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
@@ -27,6 +29,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.*;
 
 //
@@ -54,8 +58,6 @@ public class PredictModelServiceImpl implements PredictModelService {
     @Value("${ad.predict.share0.exp.code:000}")
     private String adPredictNoShareUserExpCode;
 
-//    @Value("${ad.predict.90d.share0.exp.code:612}")
-//    private String adPredict90DNoShareUserExpCode;
 
     @Value("${ad.predict.immersion.exp.code:607}")
     private String adPredictImmersionExpCode;
@@ -70,6 +72,15 @@ public class PredictModelServiceImpl implements PredictModelService {
     @Value("${user.source.layer.ad.rate.exp.ids:673,676}")
     private String userSourceLayerAdRateExpIds;
 
+    @Value("${advance.show.ad.global.flag:false}")
+    private boolean advanceShowAdGlobalFlag;
+
+    @Value("${advance.show.ad.switch:true}")
+    private boolean advanceShowAdSwitch;
+
+    @Autowired
+    private AdRedisHelper adRedisHelper;
+
     List<Integer> appIdArr = Arrays.asList(new Integer[]{0, 3, 4, 5, 6, 17, 18, 19, 21, 22});
 
 
@@ -121,9 +132,7 @@ public class PredictModelServiceImpl implements PredictModelService {
                 result.put("no_ad_strategy", "no_ad_time_with_time_plan");
                 return result;
             } else if (
-                    (!condition1 || (condition1 && !condition2))
-                            &&
-                            (0 <= hourOfDay && hourOfDay < 8)) {
+                    (!condition1 || !condition2) && 0 <= hourOfDay && hourOfDay < 8 && !isAdvanceShowAd()) {
                 // 0点到8点 && （未开启555 || 555配置不包含abcode）
                 result.put("ad_predict", 1);
                 result.put("no_ad_strategy", "no_ad_time_with_fixed_time");
@@ -363,4 +372,27 @@ public class PredictModelServiceImpl implements PredictModelService {
         }
         return false;
     }
+
+    /**
+     * 是否提前出广告
+     *
+     * @return true-提前出，false-不提前出
+     */
+    private boolean isAdvanceShowAd() {
+        // 提前出广告是否生效，不生效表示不用提前出广告，返回false。默认生效
+        if (!advanceShowAdSwitch) {
+            return false;
+        }
+
+        // 提前出广告全局开关，如果开启不走定时任务的判断
+        if (advanceShowAdGlobalFlag) {
+            return true;
+        }
+
+        // 提前出广告标识，由ad后端的定时任务写入，为1表示提前出广告
+        String day = DateTimeFormatter.ofPattern("yyyyMMdd").format(LocalDateTime.now());
+        String redisKey = String.format(RedisPrefixEnum.ADVANCE_SHOW_AD_FLAG.getPrefix(), day);
+        String flag = adRedisHelper.get(redisKey);
+        return StringUtils.equals("1", flag);
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService680.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
+import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
@@ -11,7 +12,8 @@ import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.stereotype.Service;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
 import org.xm.Similarity;
 
 import java.io.BufferedReader;
@@ -19,25 +21,44 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Slf4j
-@Service
+@Component
 public class RankService680 extends RankStrategyXGBBasic {
 
-    @ApolloJsonValue("${no.postback.conversion.adverids:[]}")
-    private Set<String> noPostbackConversionAdVerIds;
 
     private Map<String, double[]> bucketsMap = new HashMap<>();
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
+    @Value("${word2vec.exp:694}")
+    private String word2vecExp;
+
+    @ApolloJsonValue("${rank.score.weight.680:{}}")
+    private Map<String, Double> weightMap;
+
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
+        Map<String, Double> weightParam = ObjUtil.nullOrDefault(weightMap, new HashMap<>());
+
+        Set<String> noApiAdVerIds = getNoApiAdVerIds();
+
         long ts = System.currentTimeMillis() / 1000;
 
+        String brand = scoreParam.getRequestContext().getMachineinfoBrand();
+        if (StringUtils.isNotEmpty(brand)) {
+            scoreParam.getRequestContext().setMachineinfoBrand(brand + "-n");
+        }
+
+        long start = System.currentTimeMillis();
         // 特征处理
+        // feature1
         Feature feature = this.getFeature(scoreParam, request);
 
         Map<String, Map<String, String>> userFeature = feature.getUserFeature();
@@ -61,72 +82,137 @@ public class RankService680 extends RankStrategyXGBBasic {
         Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
 
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
+        long time1 = System.currentTimeMillis();
 
-        List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
+        List<AdRankItem> adRankItems = new ArrayList<>();
+        Random random = new Random();
+        List<Future<AdRankItem>> futures = new ArrayList<>();
+        CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
         for (AdPlatformCreativeDTO dto : request.getAdIdList()) {
-            AdRankItem adRankItem = new AdRankItem();
-            adRankItem.setAdId(dto.getCreativeId());
-            adRankItem.setCreativeCode(dto.getCreativeCode());
-            adRankItem.setAdVerId(dto.getAdVerId());
-            adRankItem.setVideoId(request.getVideoId());
-            adRankItem.setCpa(dto.getCpa());
-            adRankItem.setId(dto.getAdId());
-            adRankItem.setCampaignId(dto.getCampaignId());
-            adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
-
-            String cidStr = dto.getCreativeId().toString();
-            Map<String, String> cidFeatureMap = new HashMap<>();
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
-
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
-            Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
-
-            this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
-
-            this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+            Future<AdRankItem> future = ThreadPoolFactory.feature().submit(() -> {
+                AdRankItem adRankItem = new AdRankItem();
+                try {
+                    adRankItem.setAdId(dto.getCreativeId());
+                    adRankItem.setCreativeCode(dto.getCreativeCode());
+                    adRankItem.setAdVerId(dto.getAdVerId());
+                    adRankItem.setVideoId(request.getVideoId());
+                    adRankItem.setCpa(dto.getCpa());
+                    adRankItem.setId(dto.getAdId());
+                    adRankItem.setCampaignId(dto.getCampaignId());
+                    adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setRandom(random.nextInt(1000));
+                    if (noApiAdVerIds.contains(dto.getAdVerId())) {
+                        adRankItem.getExt().put("isApi", "0");
+                    } else {
+                        adRankItem.getExt().put("isApi", "1");
+                    }
 
-            this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                    String cidStr = dto.getCreativeId().toString();
+                    Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
+                    Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+                    Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
 
-            this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                    Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
 
-            this.handleD1Feature(d1Feature, cidFeatureMap);
+                    Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
 
-            this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                    this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
+                    this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
+                    this.handleB6ToB7Feature(cidFeature, cidFeatureMap);
+                    this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
+                    this.handleD1Feature(d1Feature, cidFeatureMap);
+                    this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                    return adRankItem;
+                } finally {
+                    cdl1.countDown();
+                }
+            });
+            futures.add(future);
+        }
+        try {
+            cdl1.await(300, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+        for (Future<AdRankItem> future : futures) {
+            try {
+                if (future.isDone()) {
+                    adRankItems.add(future.get());
+                }
+            } catch (Exception e) {
+                log.error("Feature handle error", e);
+            }
+        }
 
+        long time2 = System.currentTimeMillis();
+        // feature3
+        CountDownLatch cdl2 = new CountDownLatch(adRankItems.size() * 2);
+        for (AdRankItem item : adRankItems) {
+            String cidStr = String.valueOf(item.getAdId());
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
+            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
             String title = b1Feature.getOrDefault("cidtitle", "");
-            this.handleE1AndE2Feature(e1Feature, e2Feature, title, cidFeatureMap);
-
-            this.handleD3AndB1Feature(d3Feature, title, cidFeatureMap);
-
-            adRankItem.setFeatureMap(cidFeatureMap);
-
-            adRankItems.add(adRankItem);
-
+            ThreadPoolFactory.defaultPool().submit(() -> {
+                try {
+                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap(), scoreParam);
+                } finally {
+                    cdl2.countDown();
+                }
+            });
+            ThreadPoolFactory.defaultPool().submit(() -> {
+                try {
+                    this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap(), scoreParam);
+                } finally {
+                    cdl2.countDown();
+                }
+            });
+        }
+        long time31 = System.currentTimeMillis();
+        try {
+            cdl2.await(150, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
         }
 
+        // feature4
+        long time3 = System.currentTimeMillis();
         // 分桶
         this.readBucketFile();
         userFeatureMap = this.featureBucket(userFeatureMap);
+        CountDownLatch cdl4 = new CountDownLatch(adRankItems.size());
         for (AdRankItem adRankItem : adRankItems) {
-            Map<String, String> featureMap = adRankItem.getFeatureMap();
-            adRankItem.setFeatureMap(this.featureBucket(featureMap));
+            ThreadPoolFactory.feature().submit(() -> {
+                try {
+                    Map<String, String> featureMap = adRankItem.getFeatureMap();
+                    adRankItem.setFeatureMap(this.featureBucket(featureMap));
+                } finally {
+                    cdl4.countDown();
+                }
+            });
         }
-
+        try {
+            cdl4.await(100, TimeUnit.MILLISECONDS);
+        } catch (Exception e) {
+            log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
+        }
+        long time4 = System.currentTimeMillis();
         // 打分排序
-        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF)
-                .scoring(sceneFeatureMap, userFeatureMap, adRankItems);
+        // getScorerPipeline
+        List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_20240909).scoring(sceneFeatureMap, userFeatureMap, adRankItems);
+        long time5 = System.currentTimeMillis();
+        // loop
+        double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
         for (AdRankItem item : result) {
             item.setScore(item.getLrScore() * item.getCpa());
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
+            item.getScoreMap().put("cpmCoefficient", cpmCoefficient);
             item.getFeatureMap().putAll(userFeatureMap);
             item.getFeatureMap().putAll(sceneFeatureMap);
 
             // 没有转化回传的广告主，使用后台配置的CPM
-            if (noPostbackConversionAdVerIds.contains(item.getAdVerId())) {
-                item.setScore(item.getCpm() / 1000);
+            if (noApiAdVerIds.contains(item.getAdVerId())) {
+                item.setScore(item.getCpm() * cpmCoefficient / 1000);
             }
 
             for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
@@ -156,6 +242,11 @@ public class RankService680 extends RankStrategyXGBBasic {
             }
         }
 
+        log.info("cost={}, feature1={}, feature2={}, feature31={}, feature32={}, feature4={}, getScorerPipeline={}, " +
+                        "adIdSize={}, adRankItemsSize={}",
+                time5 - start, time1 - start, time2 - time1, time31 - time2, time3 - time31, time4 - time3,
+                time5 - time4, request.getAdIdList().size(), adRankItems.size());
+
         result.sort(ComparatorUtil.equalsRandomComparator());
 
         return result;
@@ -205,13 +296,11 @@ public class RankService680 extends RankStrategyXGBBasic {
                 double view = Double.parseDouble(feature.getOrDefault("ad_view_" + time, "0"));
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
-                double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
                 double f2 = NumUtil.div(conver, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
                 cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
-                // cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
 
                 cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
                 cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
@@ -316,17 +405,11 @@ public class RankService680 extends RankStrategyXGBBasic {
             featureMap.put("actionstatic_ctr", String.valueOf(ctr));
         }
         if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_conver_" + cid)) {
-            double ctcvr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
-            );
+            double ctcvr = NumUtil.div(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0), midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0));
             featureMap.put("actionstatic_ctcvr", String.valueOf(ctcvr));
         }
         if (midActionStatic.containsKey("actionstatic_conver_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
-            double cvr = NumUtil.div(
-                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
-                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)
-            );
+            double cvr = NumUtil.div(midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0), midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0));
             featureMap.put("actionstatic_cvr", String.valueOf(cvr));
         }
     }
@@ -367,24 +450,28 @@ public class RankService680 extends RankStrategyXGBBasic {
         }
     }
 
-    private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap) {
+    private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap,
+                                      ScoreParam scoreParam) {
         if (MapUtils.isEmpty(d3Feature) || !d3Feature.containsKey("title") || StringUtils.isEmpty(cTitle)) {
             return;
         }
         String vTitle = d3Feature.get("title");
-        double score = Similarity.conceptSimilarity(cTitle, vTitle);
+        double score;
+        if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
+            score = SimilarityUtils.word2VecSimilarity(cTitle, vTitle);
+        } else {
+            score = Similarity.conceptSimilarity(cTitle, vTitle);
+        }
         featureMap.put("ctitle_vtitle_similarity", String.valueOf(score));
     }
 
-    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title, Map<String, String> featureMap) {
+    private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title,
+                                      Map<String, String> featureMap, ScoreParam scoreParam) {
         if (StringUtils.isEmpty(title)) {
             return;
         }
 
-        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(
-                new Tuple2<>(e1Feature, "e1"),
-                new Tuple2<>(e2Feature, "e2")
-        );
+        List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(new Tuple2<>(e1Feature, "e1"), new Tuple2<>(e2Feature, "e2"));
 
         List<String> tagsFieldList = Arrays.asList("tags_3d", "tags_7d", "tags_14d");
         for (Tuple2<Map<String, String>, String> tuple2 : tuple2List) {
@@ -397,7 +484,13 @@ public class RankService680 extends RankStrategyXGBBasic {
             for (String tagsField : tagsFieldList) {
                 if (StringUtils.isNotEmpty(feature.get(tagsField))) {
                     String tags = feature.get(tagsField);
-                    Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    //Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    Double[] doubles;
+                    if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
+                        doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
+                    } else {
+                        doubles = ExtractorUtils.funcC34567ForTags(tags, title);
+                    }
                     featureMap.put(prefix + "_" + tagsField + "_matchnum", String.valueOf(doubles[0]));
                     featureMap.put(prefix + "_" + tagsField + "_maxscore", String.valueOf(doubles[1]));
                     featureMap.put(prefix + "_" + tagsField + "_avgscore", String.valueOf(doubles[2]));
@@ -456,9 +549,7 @@ public class RankService680 extends RankStrategyXGBBasic {
         for (Map.Entry<String, String> entry : d2Feature.entrySet()) {
             String key = entry.getKey();
             String value = entry.getValue();
-            Map<String, Double> valueMap = Arrays.stream(value.split(","))
-                    .map(r -> r.split(":"))
-                    .collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
+            Map<String, Double> valueMap = Arrays.stream(value.split(",")).map(r -> r.split(":")).collect(Collectors.toMap(rList -> rList[0], rList -> Double.parseDouble(rList[2])));
             vidRankMaps.put(key, valueMap);
         }
         return vidRankMaps;
@@ -491,9 +582,7 @@ public class RankService680 extends RankStrategyXGBBasic {
                                 String key = rList[0];
                                 double value1 = Double.parseDouble(rList[1]);
                                 bucketsLen.put(key, value1);
-                                double[] value2 = Arrays.stream(rList[2].split(","))
-                                        .mapToDouble(Double::valueOf)
-                                        .toArray();
+                                double[] value2 = Arrays.stream(rList[2].split(",")).mapToDouble(Double::valueOf).toArray();
                                 bucketsMap.put(key, value2);
                             }
                         }
@@ -510,7 +599,7 @@ public class RankService680 extends RankStrategyXGBBasic {
     }
 
     private Map<String, String> featureBucket(Map<String, String> featureMap) {
-        Map<String, String> newFeatureMap = new HashMap<>(featureMap.size());
+        Map<String, String> newFeatureMap = new ConcurrentHashMap<>(featureMap.size());
         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
             String name = entry.getKey();
             double score = Double.parseDouble(entry.getValue());
@@ -530,41 +619,4 @@ public class RankService680 extends RankStrategyXGBBasic {
         return newFeatureMap;
     }
 
-    // public static class Tuple5 {
-    //     public String f1;
-    //     public String f2;
-    //     public String f3;
-    //     public String f4;
-    //     public String f5;
-    //
-    //     public Tuple5(String f1, String f2, String f3, String f4, String f5) {
-    //         this.f1 = f1;
-    //         this.f2 = f2;
-    //         this.f3 = f3;
-    //         this.f4 = f4;
-    //         this.f5 = f5;
-    //     }
-    // }
-    //
-    // public static class TupleMapEntry<T> {
-    //     public String key;
-    //     public T value;
-    //
-    //     public TupleMapEntry(String key, T value) {
-    //         this.key = key;
-    //         this.value = value;
-    //     }
-    // }
-    //
-    // public static class Tuple2<F1, F2> {
-    //     public F1 f1;
-    //
-    //     public F2 f2;
-    //
-    //     public Tuple2(F1 first, F2 name) {
-    //         this.f1 = first;
-    //         this.f2 = name;
-    //     }
-    //
-    // }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService683.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
@@ -30,15 +29,13 @@ import java.util.stream.Collectors;
 @Component
 public class RankService683 extends RankStrategyXGBBasic {
 
-    @ApolloJsonValue("${no.postback.conversion.adverids:[]}")
-    private Set<String> noPostbackConversionAdVerIds;
-
     private Map<String, double[]> bucketsMap = new HashMap<>();
 
     private Map<String, Double> bucketsLen = new HashMap<>();
 
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+        Set<String> noApiAdVerIds = getNoApiAdVerIds();
 
         long ts = System.currentTimeMillis() / 1000;
 
@@ -70,7 +67,7 @@ public class RankService683 extends RankStrategyXGBBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
-        List<AdRankItem> adRankItems = new ArrayList<>();
+        List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
         CountDownLatch cdl1 = new CountDownLatch(request.getAdIdList().size());
@@ -87,6 +84,11 @@ public class RankService683 extends RankStrategyXGBBasic {
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
                     adRankItem.setRandom(random.nextInt(1000));
+                    if (noApiAdVerIds.contains(dto.getAdVerId())) {
+                        adRankItem.getExt().put("isApi", "0");
+                    } else {
+                        adRankItem.getExt().put("isApi", "1");
+                    }
 
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
@@ -190,7 +192,7 @@ public class RankService683 extends RankStrategyXGBBasic {
             item.getFeatureMap().putAll(sceneFeatureMap);
 
             // 没有转化回传的广告主，使用后台配置的CPM
-            if (noPostbackConversionAdVerIds.contains(item.getAdVerId())) {
+            if (noApiAdVerIds.contains(item.getAdVerId())) {
                 item.setScore(item.getCpm() / 1000);
             }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService687.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
@@ -30,9 +29,6 @@ import java.util.stream.Collectors;
 @Service
 public class RankService687 extends RankStrategyXGBBasic {
 
-    @ApolloJsonValue("${no.postback.conversion.adverids:[]}")
-    private Set<String> noPostbackConversionAdVerIds;
-
     private Map<String, double[]> bucketsMap = new HashMap<>();
 
     private Map<String, Double> bucketsLen = new HashMap<>();
@@ -42,6 +38,7 @@ public class RankService687 extends RankStrategyXGBBasic {
 
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
+        Set<String> noApiAdVerIds = getNoApiAdVerIds();
 
         long ts = System.currentTimeMillis() / 1000;
 
@@ -91,6 +88,12 @@ public class RankService687 extends RankStrategyXGBBasic {
                 adRankItem.setCampaignId(dto.getCampaignId());
                 adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
                 adRankItem.setRandom(random.nextInt(1000));
+                if (noApiAdVerIds.contains(dto.getAdVerId())) {
+                    adRankItem.getExt().put("isApi", "0");
+                } else {
+                    adRankItem.getExt().put("isApi", "1");
+                }
+
 
                 String cidStr = dto.getCreativeId().toString();
                 Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
@@ -114,7 +117,6 @@ public class RankService687 extends RankStrategyXGBBasic {
                 this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
 
 
-
                 // adRankItem.setFeatureMap(cidFeatureMap);
 
                 adRankItems.add(adRankItem);
@@ -136,7 +138,7 @@ public class RankService687 extends RankStrategyXGBBasic {
                         this.handleD3AndB1Feature(d3Feature, title, item.getFeatureMap());
                         long time23 = System.currentTimeMillis();
                         log.info("cost={} handleE1AndE2Feature={} handleD3AndB1Feature={}", time23 - time21, time22 - time21, time23 - time22);
-                    } finally{
+                    } finally {
                         cdl.countDown();
                     }
                 });
@@ -209,7 +211,6 @@ public class RankService687 extends RankStrategyXGBBasic {
         }
 
 
-
         // 打分排序
         List<AdRankItem> result = ScorerUtils.getScorerPipeline(ScorerUtils.XGBOOST_SCORE_CONF_683)
                 .scoring(sceneFeatureMap, userFeatureMap, adRankItems);
@@ -223,7 +224,7 @@ public class RankService687 extends RankStrategyXGBBasic {
             item.getFeatureMap().putAll(sceneFeatureMap);
 
             // 没有转化回传的广告主，使用后台配置的CPM
-            if (noPostbackConversionAdVerIds.contains(item.getAdVerId())) {
+            if (noApiAdVerIds.contains(item.getAdVerId())) {
                 item.setScore(item.getCpm() / 1000);
             }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankStrategyXGBAutoUpdateModel688.java

@@ -1,6 +1,5 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
-import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
@@ -31,8 +30,6 @@ import java.util.stream.Collectors;
 @Component
 public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
 
-    @ApolloJsonValue("${no.postback.conversion.adverids:[]}")
-    private Set<String> noPostbackConversionAdVerIds;
 
     private Map<String, double[]> bucketsMap = new HashMap<>();
 
@@ -44,6 +41,8 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
     @Override
     public List<AdRankItem> adItemRank(RankRecommendRequestParam request, ScoreParam scoreParam) {
 
+        Set<String> noApiAdVerIds = getNoApiAdVerIds();
+
         long ts = System.currentTimeMillis() / 1000;
 
         String brand = scoreParam.getRequestContext().getMachineinfoBrand();
@@ -96,6 +95,11 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
                     adRankItem.setRandom(random.nextInt(1000));
+                    if (noApiAdVerIds.contains(dto.getAdVerId())) {
+                        adRankItem.getExt().put("isApi", "0");
+                    } else {
+                        adRankItem.getExt().put("isApi", "1");
+                    }
 
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
@@ -199,7 +203,7 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
             item.getFeatureMap().putAll(sceneFeatureMap);
 
             // 没有转化回传的广告主，使用后台配置的CPM
-            if (noPostbackConversionAdVerIds.contains(item.getAdVerId())) {
+            if (noApiAdVerIds.contains(item.getAdVerId())) {
                 item.setScore(item.getCpm() / 1000);
             }
 
@@ -555,7 +559,7 @@ public class RankStrategyXGBAutoUpdateModel688 extends RankStrategyXGBBasic {
             return;
         }
         synchronized (this) {
-            InputStream resourceStream = RankService687.class.getClassLoader().getResourceAsStream("20240718_ad_bucket_688.txt");
+            InputStream resourceStream = RankStrategyXGBAutoUpdateModel688.class.getClassLoader().getResourceAsStream("20240718_ad_bucket_688.txt");
             if (resourceStream != null) {
                 try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
                     Map<String, double[]> bucketsMap = new HashMap<>();

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankStrategyXGBBasic.java

@@ -1,20 +1,34 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.ad.engine.commons.enums.RedisPrefixEnum;
+import com.tzld.piaoquan.ad.engine.commons.redis.AdRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
+@Slf4j
 public abstract class RankStrategyXGBBasic implements RankStrategy {
 
+    @ApolloJsonValue("${no.postback.conversion.adverids:[]}")
+    private Set<String> noPostbackConversionAdVerIds;
+
+
     @Autowired
     private FeatureService featureService;
+    @Autowired
+    protected AdRedisHelper adRedisHelper;
+
 
     protected Feature getFeature(ScoreParam param, RankRecommendRequestParam request) {
         List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
@@ -31,6 +45,15 @@ public abstract class RankStrategyXGBBasic implements RankStrategy {
         return featureService.getFeature(cidList, adVerIdList, param);
     }
 
+    protected Set<String> getNoApiAdVerIds() {
+        Set<String> noApiAdVerIds = adRedisHelper.sMembers(RedisPrefixEnum.ADVER_IS_API_EQ_0_IDS.getPrefix());
+        log.info("Redis中缓存的没有回传广告主ID集合: {}", noApiAdVerIds);
+        if (CollectionUtils.isEmpty(noApiAdVerIds)) {
+            noApiAdVerIds = noPostbackConversionAdVerIds;
+        }
+        return noApiAdVerIds;
+    }
+
     protected static class Tuple5 {
         public String f1;
         public String f2;