feat:添加打分排序的逻辑

ad-engine-commons/pom.xml

@@ -58,37 +58,37 @@
         </dependency>
 
 
-<!--        <dependency>-->
-<!--            <groupId>org.xm</groupId>-->
-<!--            <artifactId>similarity</artifactId>-->
-<!--            <version>1.1</version>-->
-<!--            <exclusions>-->
-<!--                <exclusion>-->
-<!--                    <groupId>ch.qos.logback</groupId>-->
-<!--                    <artifactId>logback-classic</artifactId>-->
-<!--                </exclusion>-->
-<!--                <exclusion>-->
-<!--                    <groupId>ch.qos.logback</groupId>-->
-<!--                    <artifactId>logback-core</artifactId>-->
-<!--                </exclusion>-->
-<!--                <exclusion>-->
-<!--                    <groupId>ch.qos.logback</groupId>-->
-<!--                    <artifactId>logback-access</artifactId>-->
-<!--                </exclusion>-->
-<!--                <exclusion>-->
-<!--                    <artifactId>slf4j-api</artifactId>-->
-<!--                    <groupId>org.slf4j</groupId>-->
-<!--                </exclusion>-->
-<!--                <exclusion>-->
-<!--                    <groupId>com.google.guava</groupId>-->
-<!--                    <artifactId>guava</artifactId>-->
-<!--                </exclusion>-->
-<!--                <exclusion>-->
-<!--                    <groupId>com.google.collections</groupId>-->
-<!--                    <artifactId>google-collections</artifactId>-->
-<!--                </exclusion>-->
-<!--            </exclusions>-->
-<!--        </dependency>-->
+        <dependency>
+            <groupId>org.xm</groupId>
+            <artifactId>similarity</artifactId>
+            <version>1.1</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-classic</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.logback</groupId>
+                    <artifactId>logback-access</artifactId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>slf4j-api</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.collections</groupId>
+                    <artifactId>google-collections</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
 </project>

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/AbstractScorer.java

@@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 
 public abstract class AbstractScorer {
@@ -61,7 +62,12 @@ public abstract class AbstractScorer {
     }
 
     public abstract List<AdRankItem> scoring(final ScoreParam param,
-                                           final UserAdFeature userAdFeature,
-                                           final List<AdRankItem> rankItems);
+                                             final UserAdFeature userAdFeature,
+                                             final List<AdRankItem> rankItems);
 
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerPipeline.java

@@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.*;
 
 
@@ -106,4 +107,80 @@ public class ScorerPipeline {
 
         return items;
     }
+
+
+    public List<AdRankItem> scoring(final Map<String, String> sceneFeatureMap,
+                                    final Map<String, String> userFeatureMap,
+                                    final List<AdRankItem> rankItems) {
+
+        if (CollectionUtils.isEmpty(scorers)) {
+            log.error("scorers is empty");
+            return rankItems;
+        }
+        List<AdRankItem> items = rankItems;
+
+        for (final AbstractScorer scorer : scorers) {
+            if (!scorer.isEnable()) {
+                continue;
+            }
+
+            final int beforeSize = rankItems.size();
+            final long startTime = System.currentTimeMillis();
+
+            String fullScorerName = scorer.getScorerConfigInfo().getScorerName();
+            String[] scorerNames = fullScorerName.split("\\.");
+            final String scorerName = scorerNames.length > 0 ? scorerNames[scorerNames.length - 1] : fullScorerName;
+
+            final List<AdRankItem> scoreRankerItems = items;
+            Callable<List<AdRankItem>> callable = () -> scorer.scoring(sceneFeatureMap, userFeatureMap, scoreRankerItems);
+
+            // execute score use thread to protected score worst time
+            List<AdRankItem> scoredItems = new ArrayList<AdRankItem>();
+            try {
+                List<Future<List<AdRankItem>>> futures = executorService.invokeAll(Arrays.asList(callable), SCORE_TIME_OUT, TimeUnit.MILLISECONDS);
+                for (Future<List<AdRankItem>> future : futures) {
+                    try {
+                        if (future.isDone() && !future.isCancelled() && future.get() != null) {
+                            scoredItems.addAll(future.get());
+                        } else {
+                            LOGGER.error("score task is cancelled, scorename [{}] fail items [{}]",
+                                    new Object[]{scorerName, scoreRankerItems.size()});
+                        }
+                    } catch (Exception e) {
+                        LOGGER.error("thread pool exception scorename [{}], exception [{}]",
+                                new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+                    }
+                }
+            } catch (Exception e) {
+                LOGGER.error("thread pool exception uid [{}] scorename [{}], exception [{}]",
+                        new Object[]{scorerName, ExceptionUtils.getFullStackTrace(e)});
+            }
+
+            //  变更item
+            if (CollectionUtils.isNotEmpty(scoreRankerItems)) {
+                items = scoreRankerItems;
+            } else {
+                items = new ArrayList<>(items);
+            }
+
+            int position = 0;
+            for (AdRankItem item : items) {
+                item.getRankerIndex().put(scorerName, position++);
+                item.getRankerScore().put(scorerName, item.getScore());
+            }
+
+            //
+            long spentTime = System.currentTimeMillis() - startTime;
+            LOGGER.debug("after scorer [{}], spentTime [{}], before size [{}], remaining size [{}]",
+                    new Object[]{scorerName, spentTime, beforeSize, scoreRankerItems.size()});
+        }
+
+        int position = 0;
+        for (AdRankItem item : items) {
+            item.getRankerIndex().put("finalScore", position++);
+            item.getRankerScore().put("finalScore", item.getScore());
+        }
+
+        return items;
+    }
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/ScorerUtils.java

@@ -29,14 +29,18 @@ public final class ScorerUtils {
     public static String CVR_ADJUSTING = "feeds_score_config_cvr_adjusting.conf";
     public static String VIDEO_CREATIVE_THOMPSON = "video_ad_thompson.conf";
 
+    public static String LR_ROV_SCORE_20240626 = "ad_score_config_20240626.conf";
+
     public static void warmUp() {
         log.info("scorer warm up ");
-        ScorerUtils.init(BASE_CONF);
-        ScorerUtils.init(THOMPSON_CONF);
-        ScorerUtils.init(BREAK_CONFIG);
-        ScorerUtils.init(SHARE0_CONFIG);
-        ScorerUtils.init(CVR_ADJUSTING);
-        ScorerUtils.init(VIDEO_CREATIVE_THOMPSON);
+        // ScorerUtils.init(BASE_CONF);
+        // ScorerUtils.init(THOMPSON_CONF);
+        // ScorerUtils.init(BREAK_CONFIG);
+        // ScorerUtils.init(SHARE0_CONFIG);
+        // ScorerUtils.init(CVR_ADJUSTING);
+        // ScorerUtils.init(VIDEO_CREATIVE_THOMPSON);
+
+        ScorerUtils.init(LR_ROV_SCORE_20240626);
     }
 
     private ScorerUtils() {

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/ExtractorUtils.java

@@ -1,6 +1,6 @@
 package com.tzld.piaoquan.ad.engine.commons.util;
 
-// import org.xm.Similarity;
+import org.xm.Similarity;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
@@ -39,29 +39,28 @@ public class ExtractorUtils {
     }
 
     public static Double[] funcC34567ForTags(String tags, String title) {
-        // String[] tagsList = tags.split(",");
-        // int d1 = 0;
-        // List<String> d2 = new ArrayList<>();
-        // double d3 = 0.0;
-        // double d4 = 0.0;
-        //
-        // for (String tag : tagsList) {
-        //     if (title.contains(tag)) {
-        //         d1++;
-        //         d2.add(tag);
-        //     }
-        //     double score = Similarity.conceptSimilarity(tag, title);
-        //     if (score > d3) {
-        //         d3 = score;
-        //     }
-        //     d4 += score;
-        // }
-        //
-        // d4 = (tagsList.length > 0) ? d4 / tagsList.length : d4;
+        String[] tagsList = tags.split(",");
+        int d1 = 0;
+        List<String> d2 = new ArrayList<>();
+        double d3 = 0.0;
+        double d4 = 0.0;
+
+        for (String tag : tagsList) {
+            if (title.contains(tag)) {
+                d1++;
+                d2.add(tag);
+            }
+            double score = Similarity.conceptSimilarity(tag, title);
+            if (score > d3) {
+                d3 = score;
+            }
+            d4 += score;
+        }
+
+        d4 = (tagsList.length > 0) ? d4 / tagsList.length : d4;
 
         // 使用数组来返回多个值
-        // return new Double[]{(double) d1, d3, d4};
-        return new Double[]{(double) 0.0, 0.0, 0.0};
+        return new Double[]{(double) d1, d3, d4};
     }
 
     public static Double calDiv(double a, double b) {

ad-engine-server/src/main/resources/ad_score_config_20240626.conf

@@ -0,0 +1,8 @@
+scorer-config = {
+  lr-rov-score-config = {
+    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogRovLRScorer"
+    scorer-priority = 99
+    model-path = "ad_ctr_model/model_ad_ctr.txt"
+  }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService680.java

@@ -1,6 +1,8 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerPipeline;
+import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.ExtractorUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
@@ -39,6 +41,7 @@ public class RankService680 {
 
         long ts = LocalDateTime.now().toEpochSecond(ZoneOffset.of("+8")) / 1000;
 
+        // 特征处理
         ScoreParam scoreParam = RequestConvert.requestConvert(request);
         Feature feature = this.getFeature(scoreParam, request);
         Map<String, Map<String, String>> userFeature = feature.getUserFeature();
@@ -103,9 +106,13 @@ public class RankService680 {
             adRankItem.setFeatureMap(this.featureBucket(featureMap));
         }
 
+        // 打分排序
+        List<AdRankItem> items = ScorerUtils.getScorerPipeline(ScorerUtils.LR_ROV_SCORE_20240626)
+                .scoring(new HashMap<>(), userFeatureMap, adRankItems);
 
+        Collections.sort(items);
 
-        return null;
+        return items;
     }
 
     private Feature getFeature(ScoreParam param, RankRecommendRequestParam request) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCtrLRScorer.java

@@ -19,6 +19,7 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.*;
 
 
@@ -62,6 +63,11 @@ public class VlogAdCtrLRScorer extends BaseLRModelScorer {
         return result;
     }
 
+    @Override
+    public List<AdRankItem> scoring(Map<String, String> sceneFeatureMap, Map<String, String> userFeatureMap, List<AdRankItem> rankItems) {
+        throw new NoSuchMethodError();
+    }
+
     private List<AdRankItem> rankByJava(final List<AdRankItem> items,
                                         final AdRequestContext requestContext,
                                         final UserAdFeature user) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCvrLRAdjustingScorer.java

@@ -9,13 +9,12 @@ import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.NotImplementedException;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.concurrent.*;
 
 
@@ -52,6 +51,11 @@ public class VlogAdCvrLRAdjustingScorer extends AbstractScorer {
         return result;
     }
 
+    @Override
+    public List<AdRankItem> scoring(Map<String, String> sceneFeatureMap, Map<String, String> userFeatureMap, List<AdRankItem> rankItems) {
+        throw new NotImplementedException();
+    }
+
     private List<AdRankItem> rankByJava(final List<AdRankItem> items,
                                         final AdRequestContext requestContext,
                                         final UserAdFeature user) {