Merge branch 'ljd/feature-20251230-youhua' of algorithm/ad-engine into master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/ExtractorUtils.java

@@ -64,18 +64,45 @@ public class ExtractorUtils {
     }
 
     public static Double[] funcC34567ForTagsNew(String tags, String title) {
+        return funcC34567ForTagsNewWithCache(tags, title, null);
+    }
+
+    /**
+     * 优化版本：支持缓存 title 和 tag 的分词结果
+     * @param tags 标签字符串，逗号分隔
+     * @param title 标题
+     * @param titleWords title 的分词结果缓存，如果为 null 则现场分词
+     * @param tagWordsCache tag 分词结果缓存（请求级别），可以为 null
+     * @return [匹配数量, 最大相似度, 平均相似度]
+     */
+    public static Double[] funcC34567ForTagsNewWithCache(String tags, String title, List<String> titleWords, Map<String, List<String>> tagWordsCache) {
+        if (tags == null || tags.isEmpty() || title == null || title.isEmpty()) {
+            return EMPTY_RESULT;
+        }
+
         String[] tagsList = tags.split(",");
         int d1 = 0;
-        List<String> d2 = new ArrayList<>();
         double d3 = 0.0;
         double d4 = 0.0;
 
+        // 只分词一次 title，复用结果
+        List<String> cachedTitleWords = (titleWords != null) ? titleWords : SimilarityUtils.segment(title);
+
         for (String tag : tagsList) {
+            if (tag == null || tag.isEmpty()) {
+                continue;
+            }
             if (title.contains(tag)) {
                 d1++;
-                d2.add(tag);
             }
-            double score = SimilarityUtils.word2VecSimilarity(tag, title);
+            // 使用请求级别的 tag 分词缓存
+            List<String> tagWords;
+            if (tagWordsCache != null) {
+                tagWords = tagWordsCache.computeIfAbsent(tag, SimilarityUtils::segment);
+            } else {
+                tagWords = SimilarityUtils.segment(tag);
+            }
+            float score = SimilarityUtils.word2VecSimilarityWithWords(tagWords, cachedTitleWords);
             if (score > d3) {
                 d3 = score;
             }
@@ -84,10 +111,21 @@ public class ExtractorUtils {
 
         d4 = (tagsList.length > 0) ? d4 / tagsList.length : d4;
 
-        // 使用数组来返回多个值
         return new Double[]{(double) d1, d3, d4};
     }
 
+    /**
+     * 兼容旧接口
+     */
+    public static Double[] funcC34567ForTagsNewWithCache(String tags, String title, List<String> titleWords) {
+        return funcC34567ForTagsNewWithCache(tags, title, titleWords, null);
+    }
+
+    /**
+     * 空结果常量，避免重复创建数组
+     */
+    private static final Double[] EMPTY_RESULT = new Double[]{0.0, 0.0, 0.0};
+
     public static Double calDiv(double a, double b) {
         if (a == 0 || b == 0) {
             return 0D;

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/SimilarityUtils.java

@@ -52,12 +52,51 @@ public final class SimilarityUtils {
         }
     }
 
-
+    /**
+     * 原始方法，保持兼容
+     */
     public static float word2VecSimilarity(String str1, String str2) {
         List<String> words1 = Segment.getWords(str1);
         List<String> words2 = Segment.getWords(str2);
         return vec.sentenceSimilarity(words1, words2);
     }
 
+    /**
+     * 优化版本：复用已分词的结果，避免重复分词
+     * @param words1 已分词的词列表
+     * @param words2 已分词的词列表
+     * @return 相似度分数
+     */
+    public static float word2VecSimilarityWithWords(List<String> words1, List<String> words2) {
+        if (words1 == null || words1.isEmpty() || words2 == null || words2.isEmpty()) {
+            return 0.0f;
+        }
+        return vec.sentenceSimilarity(words1, words2);
+    }
+
+    /**
+     * 优化版本：对一个文本与多个文本计算相似度，复用第一个文本的分词结果
+     * @param str1 第一个文本
+     * @param str2 第二个文本
+     * @param str1Words 第一个文本已分词的结果（如果为null则现场分词）
+     * @return 相似度分数
+     */
+    public static float word2VecSimilarityWithCache(String str1, String str2, List<String> str1Words) {
+        List<String> words1 = (str1Words != null) ? str1Words : Segment.getWords(str1);
+        List<String> words2 = Segment.getWords(str2);
+        return vec.sentenceSimilarity(words1, words2);
+    }
+
+    /**
+     * 分词方法，供外部缓存使用
+     * @param text 待分词文本
+     * @return 分词结果
+     */
+    public static List<String> segment(String text) {
+        if (text == null || text.isEmpty()) {
+            return java.util.Collections.emptyList();
+        }
+        return Segment.getWords(text);
+    }
 
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -39,6 +39,12 @@ import static com.tzld.piaoquan.ad.engine.commons.math.Const.*;
 @Component
 public class RankStrategyBy688 extends RankStrategyBasic {
 
+    /**
+     * 空 Map 常量，避免频繁创建空 HashMap
+     */
+    private static final Map<String, String> EMPTY_STRING_MAP = Collections.emptyMap();
+    private static final Map<String, Map<String, String>> EMPTY_NESTED_MAP = Collections.emptyMap();
+
     private Map<String, double[]> bucketsMap = new HashMap<>();
 
     private Map<String, Double> bucketsLen = new HashMap<>();
@@ -105,22 +111,22 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
 
         Map<String, String> userFeatureMap = new HashMap<>();
-        Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", new HashMap<>());
+        Map<String, String> c1Feature = userFeature.getOrDefault("alg_mid_feature_ad_action", EMPTY_STRING_MAP);
         List<TupleMapEntry<Tuple5>> midActionList = this.handleC1Feature(c1Feature, userFeatureMap);
 
         Map<String, Double> midTimeDiffMap = this.parseC1FeatureListToTimeDiffMap(midActionList, ts);
         Map<String, Double> actionStaticMap = this.parseC1FeatureListToActionStaticMap(midActionList);
 
-        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", new HashMap<>());
-        Map<String, String> d3Feature = videoFeature.getOrDefault("alg_vid_feature_basic_info", new HashMap<>());
+        Map<String, String> d2Feature = videoFeature.getOrDefault("alg_cid_feature_vid_cf_rank", EMPTY_STRING_MAP);
+        Map<String, String> d3Feature = videoFeature.getOrDefault("alg_vid_feature_basic_info", EMPTY_STRING_MAP);
 
         Map<String, Map<String, Double>> vidRankMaps = this.parseD2FeatureMap(d2Feature);
 
-        Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", new HashMap<>());
-        Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", new HashMap<>());
+        Map<String, String> e1Feature = userFeature.getOrDefault("alg_mid_feature_return_tags", EMPTY_STRING_MAP);
+        Map<String, String> e2Feature = userFeature.getOrDefault("alg_mid_feature_share_tags", EMPTY_STRING_MAP);
 
-        Map<String, String> g1Feature = userFeature.getOrDefault("mid_return_video_cate", new HashMap<>());
-        Map<String, String> g2Feature = userFeature.getOrDefault("mid_share_video_cate", new HashMap<>());
+        Map<String, String> g1Feature = userFeature.getOrDefault("mid_return_video_cate", EMPTY_STRING_MAP);
+        Map<String, String> g2Feature = userFeature.getOrDefault("mid_share_video_cate", EMPTY_STRING_MAP);
 
 
         userFeatureMap.put("brand", reqFeature.getOrDefault("brand", ""));
@@ -185,12 +191,12 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt(), isGuaranteedFlow, reqFeature);
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
-                    Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-                    Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+                    Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, EMPTY_NESTED_MAP);
+                    Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", EMPTY_STRING_MAP);
 
-                    Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-                    Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(dto.getSkuId()), new HashMap<>());
-                    Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
+                    Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), EMPTY_NESTED_MAP);
+                    Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(dto.getSkuId()), EMPTY_NESTED_MAP);
+                    Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", EMPTY_STRING_MAP);
 
                     this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
                     this.handleB2ToB5AndB8ToB9Feature(cidFeature, adVerFeature, cidFeatureMap);
@@ -235,15 +241,17 @@ public class RankStrategyBy688 extends RankStrategyBasic {
 
         long time2 = System.currentTimeMillis();
         // feature3
+        // 请求级别的 tag 分词缓存，所有广告共享（同一用户的 tags 相同）
+        Map<String, List<String>> tagWordsCache = new ConcurrentHashMap<>();
         CountDownLatch cdl2 = new CountDownLatch(adRankItems.size() * 2);
         for (AdRankItem item : adRankItems) {
             String cidStr = String.valueOf(item.getAdId());
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
-            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, EMPTY_NESTED_MAP);
+            Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", EMPTY_STRING_MAP);
             String title = b1Feature.getOrDefault("cidtitle", "");
             ThreadPoolFactory.defaultPool().submit(() -> {
                 try {
-                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap(), scoreParam);
+                    this.handleE1AndE2Feature(e1Feature, e2Feature, title, item.getFeatureMap(), scoreParam, tagWordsCache);
                 } finally {
                     cdl2.countDown();
                 }
@@ -299,8 +307,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             double calibratedScore = originalScore / (originalScore + (1 - originalScore) / negSampleRate);
             // 该创意尚未在模型中训练，打分不可靠
             if (CollectionUtils.isNotEmpty(DnnCidDataHelper.getCidSet()) && !DnnCidDataHelper.getCidSet().contains(item.getAdId())) {
-                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
-                Map<String, String> b3Feature = cidFeature.getOrDefault("alg_cid_feature_cid_action", new HashMap<>());
+                Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), EMPTY_NESTED_MAP);
+                Map<String, String> b3Feature = cidFeature.getOrDefault("alg_cid_feature_cid_action", EMPTY_STRING_MAP);
                 double view = Double.parseDouble(b3Feature.getOrDefault("ad_view_14d", "0"));
                 double conver = Double.parseDouble(b3Feature.getOrDefault("ad_conversion_14d", "0"));
                 double smoothCxr = NumUtil.divSmoothV1(conver, view, 1.64);
@@ -440,12 +448,12 @@ public class RankStrategyBy688 extends RankStrategyBasic {
     }
 
     private void handleB2ToB5AndB8ToB9Feature(Map<String, Map<String, String>> c1Feature, Map<String, Map<String, String>> adVerFeature, Map<String, String> cidFeatureMap) {
-        Map<String, String> b2Feature = adVerFeature.getOrDefault("alg_cid_feature_adver_action", new HashMap<>());
-        Map<String, String> b3Feature = c1Feature.getOrDefault("alg_cid_feature_cid_action", new HashMap<>());
-        Map<String, String> b4Feature = c1Feature.getOrDefault("alg_cid_feature_region_action", new HashMap<>());
-        Map<String, String> b5Feature = c1Feature.getOrDefault("alg_cid_feature_app_action", new HashMap<>());
-        Map<String, String> b8Feature = c1Feature.getOrDefault("alg_cid_feature_brand_action", new HashMap<>());
-        Map<String, String> b9Feature = c1Feature.getOrDefault("alg_cid_feature_weChatVersion_action", new HashMap<>());
+        Map<String, String> b2Feature = adVerFeature.getOrDefault("alg_cid_feature_adver_action", EMPTY_STRING_MAP);
+        Map<String, String> b3Feature = c1Feature.getOrDefault("alg_cid_feature_cid_action", EMPTY_STRING_MAP);
+        Map<String, String> b4Feature = c1Feature.getOrDefault("alg_cid_feature_region_action", EMPTY_STRING_MAP);
+        Map<String, String> b5Feature = c1Feature.getOrDefault("alg_cid_feature_app_action", EMPTY_STRING_MAP);
+        Map<String, String> b8Feature = c1Feature.getOrDefault("alg_cid_feature_brand_action", EMPTY_STRING_MAP);
+        Map<String, String> b9Feature = c1Feature.getOrDefault("alg_cid_feature_weChatVersion_action", EMPTY_STRING_MAP);
 
         List<String> timeList = Arrays.asList("1h", "2h", "3h", "6h", "12h", "1d", "3d", "7d", "yesterday", "today");
         List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
@@ -483,8 +491,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
     }
 
     private void handleB6ToB7Feature(Map<String, Map<String, String>> c1Feature, Map<String, String> cidFeatureMap) {
-        Map<String, String> b6Feature = c1Feature.getOrDefault("alg_cid_feature_week_action", new HashMap<>());
-        Map<String, String> b7Feature = c1Feature.getOrDefault("alg_cid_feature_hour_action", new HashMap<>());
+        Map<String, String> b6Feature = c1Feature.getOrDefault("alg_cid_feature_week_action", EMPTY_STRING_MAP);
+        Map<String, String> b7Feature = c1Feature.getOrDefault("alg_cid_feature_hour_action", EMPTY_STRING_MAP);
 
         List<String> timeList = Arrays.asList("7d", "14d");
         List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
@@ -655,8 +663,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
     private void handleH1AndH2Feature(Map<String, Map<String, String>> skuFeature,
                                       Map<String, Map<String, String>> adVerFeature,
                                       Map<String, String> cidFeatureMap) {
-        Map<String, String> h1Feature = adVerFeature.getOrDefault("alg_mid_feature_adver_action", new HashMap<>());
-        Map<String, String> h2Feature = skuFeature.getOrDefault("alg_mid_feature_sku_action", new HashMap<>());
+        Map<String, String> h1Feature = adVerFeature.getOrDefault("alg_mid_feature_adver_action", EMPTY_STRING_MAP);
+        Map<String, String> h2Feature = skuFeature.getOrDefault("alg_mid_feature_sku_action", EMPTY_STRING_MAP);
         List<String> timeList = Arrays.asList("3d", "7d", "30d");
         List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
                 new Tuple2<>(h1Feature, "adverid"),
@@ -695,11 +703,18 @@ public class RankStrategyBy688 extends RankStrategyBasic {
     }
 
     private void handleE1AndE2Feature(Map<String, String> e1Feature, Map<String, String> e2Feature, String title,
-                                      Map<String, String> featureMap, ScoreParam scoreParam) {
+                                      Map<String, String> featureMap, ScoreParam scoreParam,
+                                      Map<String, List<String>> tagWordsCache) {
         if (StringUtils.isEmpty(title)) {
             return;
         }
 
+        // 预先分词 title，在整个方法中复用，避免重复分词
+        List<String> titleWords = null;
+        if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
+            titleWords = SimilarityUtils.segment(title);
+        }
+
         List<Tuple2<Map<String, String>, String>> tuple2List = Arrays.asList(new Tuple2<>(e1Feature, "e1"), new Tuple2<>(e2Feature, "e2"));
 
         List<String> tagsFieldList = Arrays.asList("tags_3d", "tags_7d", "tags_14d");
@@ -713,10 +728,10 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             for (String tagsField : tagsFieldList) {
                 if (StringUtils.isNotEmpty(feature.get(tagsField))) {
                     String tags = feature.get(tagsField);
-                    // Double[] doubles = ExtractorUtils.funcC34567ForTags(tags, title);
                     Double[] doubles;
                     if (scoreParam.getExpCodeSet().contains(word2vecExp)) {
-                        doubles = ExtractorUtils.funcC34567ForTagsNew(tags, title);
+                        // 使用缓存的 title 分词结果和请求级别的 tag 分词缓存
+                        doubles = ExtractorUtils.funcC34567ForTagsNewWithCache(tags, title, titleWords, tagWordsCache);
                     } else {
                         doubles = ExtractorUtils.funcC34567ForTags(tags, title);
                     }
@@ -880,7 +895,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
     }
 
     private Map<String, String> featureBucket(Map<String, String> featureMap) {
-        Map<String, String> newFeatureMap = new ConcurrentHashMap<>(featureMap.size());
+        // 使用 HashMap 替代 ConcurrentHashMap，分桶操作是单线程的
+        Map<String, String> newFeatureMap = new HashMap<>(featureMap.size());
         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
             try {
                 String name = entry.getKey();