Merge branch 'feature_20241122_ad_recall' into pre-master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/dto/AdPlatformCreativeDTO.java

@@ -4,6 +4,8 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.ToString;
 
+import java.util.List;
+
 @Builder
 @Data
 @ToString
@@ -41,4 +43,6 @@ public class AdPlatformCreativeDTO {
      */
     private AdDirectionScore adDirectionScore;
 
+    // 召回源，记录召回用
+    private List<String> recallSources;
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/XGBCalibrationModel.java

@@ -61,7 +61,7 @@ public class XGBCalibrationModel extends Model {
         for (Table.Cell<Double, Double, Double> cell : table.cellSet()) {
             double rowKey = cell.getRowKey();
             double columnKey = cell.getColumnKey();
-            if (rowKey <= score & score < columnKey) {
+            if (rowKey < score & score <= columnKey) {
                 LOGGER.info("score {} in {} - {} , value is {}", score, rowKey, columnKey, cell.getValue());
                 return cell.getValue();
             }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/thread/ThreadPoolFactory.java

@@ -15,14 +15,14 @@ public final class ThreadPoolFactory {
             512,
             512,
             0L, TimeUnit.SECONDS,
-            new LinkedBlockingQueue<>(1000),
+            new LinkedBlockingQueue<>(2000),
             new ThreadFactoryBuilder().setNameFormat("DEFAULT-%d").build(),
             new ThreadPoolExecutor.AbortPolicy());
     private final static ExecutorService FEATURE = new CommonThreadPoolExecutor(
             256,
             256,
             0L, TimeUnit.SECONDS,
-            new LinkedBlockingQueue<>(1000),
+            new LinkedBlockingQueue<>(5000),
             new ThreadFactoryBuilder().setNameFormat("Feature-%d").build(),
             new ThreadPoolExecutor.AbortPolicy());
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -111,6 +111,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                     } else {
                         adRankItem.getExt().put("isApi", "1");
                     }
+                    adRankItem.getExt().put("recallsources", dto.getRecallSources());
 
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
@@ -209,7 +210,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         double cpmCoefficient = weightParam.getOrDefault("cpmCoefficient", 0.9);
         for (AdRankItem item : result) {
 
-            double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getId(), 1d);
+            double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
             item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
 
             item.getScoreMap().put("cpa", item.getCpa());

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -10,6 +10,7 @@ import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
@@ -107,6 +108,8 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                         adRankItem.getExt().put("isApi", "1");
                     }
 
+                    adRankItem.getExt().put("recallsources", dto.getRecallSources());
+
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
@@ -167,14 +170,12 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                 }
             });
         }
-        long time31 = System.currentTimeMillis();
         try {
             cdl2.await(150, TimeUnit.MILLISECONDS);
         } catch (Exception e) {
             log.error("handleE1AndE2Feature and handleD3AndB1Feature wait timeout", e);
         }
 
-        // feature4
         long time3 = System.currentTimeMillis();
         // 分桶
         this.readBucketFile();
@@ -205,7 +206,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
 
         for (AdRankItem item : result) {
 
-            double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getId(), 1d);
+            double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
             item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
 
             item.getScoreMap().put("cpa", item.getCpa());
@@ -219,40 +220,44 @@ public class RankStrategyBy680 extends RankStrategyBasic {
             if (noApiAdVerIds.contains(item.getAdVerId())) {
                 item.setScore(item.getCpm() * cpmCoefficient / 1000);
             }
+        }
+
+
+        result.sort(ComparatorUtil.equalsRandomComparator());
 
+        if (CollectionUtils.isNotEmpty(result)) {
+            AdRankItem top1Item = result.get(0);
             for (Map.Entry<String, Map<String, String>> entry : videoFeature.entrySet()) {
                 if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
 
             for (Map.Entry<String, Map<String, String>> entry : userFeature.entrySet()) {
                 if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
 
-            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(item.getAdVerId(), new HashMap<>());
+            Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(top1Item.getAdVerId(), new HashMap<>());
             for (Map.Entry<String, Map<String, String>> entry : adVerFeature.entrySet()) {
                 if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
 
-            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(item.getAdId()), new HashMap<>());
+            Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(String.valueOf(top1Item.getAdId()), new HashMap<>());
             for (Map.Entry<String, Map<String, String>> entry : cidFeature.entrySet()) {
                 if (MapUtils.isNotEmpty(entry.getValue())) {
-                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
         }
-
-        log.info("cost={}, feature1={}, feature2={}, feature31={}, feature32={}, feature4={}, getScorerPipeline={}, " +
-                        "adIdSize={}, adRankItemsSize={}",
-                time5 - start, time1 - start, time2 - time1, time31 - time2, time3 - time31, time4 - time3,
-                time5 - time4, request.getAdIdList().size(), adRankItems.size());
-
-        result.sort(ComparatorUtil.equalsRandomComparator());
+        long time6 = System.currentTimeMillis();
+        log.info("cost={}, getFeature={}, handleFeature={},  similar={}, bucketFeature={}, getScorerPipeline={}, " +
+                        "other={}, adIdSize={}, adRankItemsSize={}",
+                time6 - start, time1 - start, time2 - time1, time3 - time2, time4 - time3,
+                time5 - time4, time6 - time5, request.getAdIdList().size(), adRankItems.size());
 
         return result;
     }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -89,6 +89,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     } else {
                         adRankItem.getExt().put("isApi", "1");
                     }
+                    adRankItem.getExt().put("recallsources", dto.getRecallSources());
 
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy687.java

@@ -93,6 +93,7 @@ public class RankStrategyBy687 extends RankStrategyBasic {
                 } else {
                     adRankItem.getExt().put("isApi", "1");
                 }
+                adRankItem.getExt().put("recallsources", dto.getRecallSources());
 
 
                 String cidStr = dto.getCreativeId().toString();

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -100,6 +100,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     } else {
                         adRankItem.getExt().put("isApi", "1");
                     }
+                    adRankItem.getExt().put("recallsources", dto.getRecallSources());
 
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyByWeight.java

@@ -49,6 +49,7 @@ public class RankStrategyByWeight extends RankStrategyBasic {
             } else {
                 adRankItem.getExt().put("isApi", "1");
             }
+            adRankItem.getExt().put("recallsources", dto.getRecallSources());
 
 
             adRankItems.add(adRankItem);