Merge branch 'feature_20240705_zhaohaipeng_model_v2' of algorithm/ad-engine into master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/NumUtil.java

@@ -9,6 +9,14 @@ public class NumUtil {
         return d1 / d2;
     }
 
+
+    public static Double log(double a){
+        if (a <= 0){
+            return 0D;
+        }
+        return Math.log(a + 1.0);
+    }
+
     @SafeVarargs
     public static <T extends Comparable<T>> T min(T... values) {
         if (values == null || values.length == 0) {

ad-engine-server/src/main/resources/ad_score_config_20240626.conf

@@ -1,8 +1,8 @@
 scorer-config = {
   lr-rov-score-config = {
-    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogRovLRScorer"
+    scorer-name = "com.tzld.piaoquan.ad.engine.service.score.VlogRovFMScorer"
     scorer-priority = 99
-    model-path = "zhangbo/model_bkb0.txt"
+    model-path = "zhangbo/model_bkb8_v3.txt"
   }
 
 }

ad-engine-server/src/main/resources/application-dev.yml

@@ -4,7 +4,7 @@ server:
 eureka:
   client:
     serviceUrl:
-      defaultZone: http://deveureka-internal.piaoquantv.com/eureka/
+      defaultZone: http://127.0.0.1:7000/eureka/
 
 datalog: .
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService680.java

@@ -13,6 +13,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.BufferedReader;
@@ -26,6 +27,9 @@ import java.util.stream.Collectors;
 @Service
 public class RankService680 {
 
+    @Value("${id.feature.default.value:0.01}")
+    private String idDefaultValue;
+
     @Autowired
     private FeatureService featureService;
 
@@ -180,19 +184,19 @@ public class RankService680 {
     }
 
     private void handleB1Feature(Map<String, String> b1Feature, Map<String, String> cidFeatureMap, String cid) {
-        cidFeatureMap.put("cid_" + cid, "1");
-        if (StringUtils.isNotBlank(b1Feature.get("adid"))) {
-            String adId = b1Feature.get("adid");
-            cidFeatureMap.put("adid_" + adId, "1");
-        }
+        cidFeatureMap.put("cid_" + cid, "0.01");
+        // if (StringUtils.isNotBlank(b1Feature.get("adid"))) {
+        //     String adId = b1Feature.get("adid");
+        //     cidFeatureMap.put("adid_" + adId, idDefaultValue);
+        // }
         if (StringUtils.isNotBlank(b1Feature.get("adverid"))) {
             String adVerId = b1Feature.get("adverid");
-            cidFeatureMap.put("adverid_" + adVerId, "1");
-        }
-        if (StringUtils.isNotBlank(b1Feature.get("targeting_conversion"))) {
-            String targetingConversion = b1Feature.get("targeting_conversion");
-            cidFeatureMap.put("targeting_conversion_" + targetingConversion, "1");
+            cidFeatureMap.put("adverid_" + adVerId, "0.01");
         }
+        // if (StringUtils.isNotBlank(b1Feature.get("targeting_conversion"))) {
+        //     String targetingConversion = b1Feature.get("targeting_conversion");
+        //     cidFeatureMap.put("targeting_conversion_" + targetingConversion, idDefaultValue);
+        // }
         if (StringUtils.isNotBlank(b1Feature.get("cpa"))) {
             String cpa = b1Feature.get("cpa");
             cidFeatureMap.put("cpa", cpa);
@@ -222,11 +226,16 @@ public class RankService680 {
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
                 double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                double f2 = NumUtil.div(conver, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
                 cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+
+                cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
             }
         }
 
@@ -249,11 +258,16 @@ public class RankService680 {
                 double click = Double.parseDouble(feature.getOrDefault("ad_click_" + time, "0"));
                 double conver = Double.parseDouble(feature.getOrDefault("ad_conversion_" + time, "0"));
                 double income = Double.parseDouble(feature.getOrDefault("ad_income_" + time, "0"));
+                double f2 = NumUtil.div(conver, view);
                 cidFeatureMap.put(prefix + "_" + time + "_ctr", String.valueOf(NumUtil.div(click, view)));
-                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(NumUtil.div(conver, view)));
+                cidFeatureMap.put(prefix + "_" + time + "_ctcvr", String.valueOf(f2));
                 cidFeatureMap.put(prefix + "_" + time + "_cvr", String.valueOf(NumUtil.div(conver, click)));
                 cidFeatureMap.put(prefix + "_" + time + "_conver", String.valueOf(conver));
                 cidFeatureMap.put(prefix + "_" + time + "_ecpm", String.valueOf(NumUtil.div(income * 1000, view)));
+
+                cidFeatureMap.put(prefix + "_" + time + "_click", String.valueOf(click));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*log(view)", String.valueOf(conver * NumUtil.log(view)));
+                cidFeatureMap.put(prefix + "_" + time + "_conver*ctcvr", String.valueOf(conver * f2));
             }
         }
 
@@ -321,9 +335,9 @@ public class RankService680 {
             );
             featureMap.put("actionstatic_ctr", String.valueOf(ctr));
         }
-        if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("timediff_conver_" + cid)) {
+        if (midActionStatic.containsKey("actionstatic_view_" + cid) && midActionStatic.containsKey("actionstatic_conver_" + cid)) {
             double ctcvr = NumUtil.div(
-                    midActionStatic.getOrDefault("timediff_conver_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
                     midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
             );
             featureMap.put("actionstatic_ctcvr", String.valueOf(ctcvr));
@@ -331,7 +345,7 @@ public class RankService680 {
         if (midActionStatic.containsKey("actionstatic_conver_" + cid) && midActionStatic.containsKey("actionstatic_click_" + cid)) {
             double cvr = NumUtil.div(
                     midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
-                    midActionStatic.getOrDefault("timediff_conver_" + cid, 0.0)
+                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0)
             );
             featureMap.put("actionstatic_cvr", String.valueOf(cvr));
         }
@@ -465,7 +479,7 @@ public class RankService680 {
             return;
         }
         synchronized (this) {
-            InputStream resourceStream = RankService680.class.getClassLoader().getResourceAsStream("20240622_ad_bucket_249.txt");
+            InputStream resourceStream = RankService680.class.getClassLoader().getResourceAsStream("20240704_ad_bucket_351.txt");
             if (resourceStream != null) {
                 try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceStream))) {
                     Map<String, double[]> bucketsMap = new HashMap<>();
@@ -490,7 +504,7 @@ public class RankService680 {
                     this.bucketsMap = bucketsMap;
                     this.bucketsLen = bucketsLen;
                 } catch (IOException e) {
-                    log.error("something is wrong in parse bucket file:" + e);
+                    log.error("something is wrong in parse bucket file:", e);
                 }
             } else {
                 log.error("no bucket file");
@@ -505,9 +519,9 @@ public class RankService680 {
             double score = Double.parseDouble(entry.getValue());
             // 注意：0值、不在分桶文件中的特征，会被过滤掉。
             if (score > 1E-8) {
-                if (this.bucketsMap.containsKey(name)) {
+                if (this.bucketsMap.containsKey(name) && this.bucketsLen.containsKey(name)) {
                     double[] buckets = this.bucketsMap.get(name);
-                    int bucketNum = buckets.length + 1;
+                    double bucketNum = this.bucketsLen.get(name);
                     Double scoreNew = 1.0 / bucketNum * (ExtractorUtils.findInsertPosition(buckets, score) + 1.0);
                     newFeatureMap.put(name, String.valueOf(scoreNew));
                 } else {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogRovFMScorer.java

@@ -8,14 +8,12 @@ import com.tzld.piaoquan.ad.engine.commons.score.model.FMModel;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.UserAdFeature;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.*;
 
 
@@ -132,29 +130,28 @@ public class VlogRovFMScorer extends BaseFMModelScorer {
                             final Map<String, String> sceneFeatureMap) {
 
 
-        // Map<String, String> featureMap = new HashMap<>();
-        // if (MapUtils.isNotEmpty(item.getFeatureMap())) {
-        //     featureMap.putAll(item.getFeatureMap());
-        // }
-        // if (MapUtils.isNotEmpty(userFeatureMap)) {
-        //     featureMap.putAll(userFeatureMap);
-        // }
-        // if (MapUtils.isNotEmpty(sceneFeatureMap)) {
-        //     featureMap.putAll(sceneFeatureMap);
-        // }
+        Map<String, String> featureMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(item.getFeatureMap())) {
+            featureMap.putAll(item.getFeatureMap());
+        }
+        if (MapUtils.isNotEmpty(userFeatureMap)) {
+            featureMap.putAll(userFeatureMap);
+        }
+        if (MapUtils.isNotEmpty(sceneFeatureMap)) {
+            featureMap.putAll(sceneFeatureMap);
+        }
 
         double pro = 0.0;
-        // if (MapUtils.isNotEmpty(featureMap)) {
-        //     try {
-        //         pro = model.score(featureMap);
-        //         // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
-        //     } catch (Exception e) {
-        //         LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
-        //     }
-        // }
-        // item.setScoreRov(pro);
-        // item.getScoresMap().put("RovFMScore", pro);
-        // item.setAllFeatureMap(featureMap);
+        if (MapUtils.isNotEmpty(featureMap)) {
+            try {
+                pro = model.score(featureMap);
+                // LOGGER.info("fea : {}, score:{}", JSONUtils.toJson(featureMap), pro);
+            } catch (Exception e) {
+                LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
+            }
+        }
+        item.setLrScore(pro);
+        item.getScoreMap().put("ctcvrScore", pro);
         return pro;
     }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogRovLRScorer.java

@@ -148,7 +148,7 @@ public class VlogRovLRScorer extends BaseLRV2ModelScorer {
                 LOGGER.error("score error for doc={} exception={}", item.getVideoId(), ExceptionUtils.getFullStackTrace(e));
             }
         }
-        item.getScoreMap().put("rovLrScore", pro);
+        item.getScoreMap().put("ctcvrScore", pro);
         item.setLrScore(pro);
         return pro;
     }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/TacticsAndLRModelScoreRankService.java

@@ -1,5 +1,6 @@
 package com.tzld.piaoquan.ad.engine.service.score.impl;
 
+import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.service.score.RankService680;
@@ -27,6 +28,15 @@ public class TacticsAndLRModelScoreRankService {
         // LR模型打分结果
         List<AdRankItem> result = rankService680.adItemRank(requestParam, scoreParam);
 
+        // List<JSONObject> collect = result.stream().map(item -> {
+        //     JSONObject json = new JSONObject();
+        //     json.put("cid", item.getAdId());
+        //     json.put("score", item.getScore());
+        //     json.put("modelScore", item.getLrScore());
+        //     json.put("allfeature", item.getFeatureMap());
+        //     return json;
+        // }).collect(Collectors.toList());
+
         Map<Long, AdDirectionScore> adDirectionScoreMap = requestParam.getAdIdList().stream()
                 .collect(Collectors.toMap(AdPlatformCreativeDTO::getCreativeId, AdPlatformCreativeDTO::getAdDirectionScore));
 
@@ -77,7 +87,7 @@ public class TacticsAndLRModelScoreRankService {
         Map<String, String> scoreDetailMap = new HashMap<>(scoreDetail);
         scoreDetailMap.put("exponent", String.valueOf(exponent));
         scoreDetailMap.put("excludeMin", String.valueOf(excludeMin));
-        adRankItem.getMetaFeatureMap().put("adDirectionScoreDetail", scoreDetailMap);
+        adRankItem.getFeatureMap().putAll(scoreDetailMap);
     }
 
 }