Merge branch 'feature_20241212_chowd_choose_v2' of algorithm/ad-engine into master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/NumUtil.java

@@ -1,5 +1,9 @@
 package com.tzld.piaoquan.ad.engine.commons.util;
 
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.Arrays;
+
 public class NumUtil {
 
     public static double div(double d1, double d2) {
@@ -10,13 +14,41 @@ public class NumUtil {
     }
 
 
-    public static Double log(double a){
-        if (a <= 0){
+    public static double log(double a) {
+        if (a <= 0) {
             return 0D;
         }
         return Math.log(a + 1.0);
     }
 
+    public static double log10(double a) {
+        if (a <= 0) {
+            return 0d;
+        }
+        return Math.log10(a + 1.0);
+    }
+
+    public static double sigmoid(double x) {
+        return 1 / (1 + Math.exp(-x));
+    }
+
+    public static double round(double value, int newScale){
+        return new BigDecimal(value).setScale(newScale, RoundingMode.HALF_UP).doubleValue();
+    }
+
+    public static double[] softmax(double[] inputs) {
+        double max = Arrays.stream(inputs).max().orElse(0);
+        double[] expValues = Arrays.stream(inputs)
+                .map(x -> Math.exp(x - max)) // Subtract max to stabilize
+                .toArray();
+
+        double sum = Arrays.stream(expValues).sum();
+
+        return Arrays.stream(expValues)
+                .map(x -> x / sum)
+                .toArray();
+    }
+
     @SafeVarargs
     public static <T extends Comparable<T>> T min(T... values) {
         if (values == null || values.length == 0) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/FeatureService.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.feature;
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
 import com.tzld.piaoquan.ad.engine.service.remote.FeatureV2RemoteService;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
@@ -82,9 +83,9 @@ public class FeatureService {
         return this.invokeFeatureService(protos);
     }
 
-    public Feature getPredictFeature(String mid) {
+    public Feature getPredictFeature(PredictContext context) {
         List<FeatureKeyProto> protos = new ArrayList<>();
-        protos.add(genWithMid("alg_ad_crowd_choose_feature", mid));
+        protos.add(genWithMidAndAppType("alg_ad_crowd_choose_feature_v2", context.getMid(), context.getAppType()));
         return this.invokeFeatureService(protos);
     }
 
@@ -224,6 +225,15 @@ public class FeatureService {
                 .build();
     }
 
+    private FeatureKeyProto genWithMidAndAppType(String table, String mid, String appType) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(midUkFormat, table, mid))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .putFieldValue("apptype", appType)
+                .build();
+    }
+
     private Map<String, String> featureStrCover(Map<String, String> metaFeatureMap) {
         Map<String, String> newFeatureMap = new HashMap<>();
         for (Map.Entry<String, String> entry : metaFeatureMap.entrySet()) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -136,7 +136,7 @@ public class LogHubServiceImpl implements LogHubService {
                 logMap.put("allfeature", JSON.toJSONString(context.getLogParam().getAllFeature()));
                 logMap.put("metafeature", JSON.toJSONString(context.getLogParam().getMetaFeature()));
                 logMap.put("scoremap", JSON.toJSONString(context.getLogParam().getScoreMap()));
-
+                logMap.put("isNewUser", context.getLogParam().isBIsNewUser());
                 aliyunLogManager.sendLog(project, crowdChooseStatisticsLogStore, "", logMap);
             }
         });

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/impl/PredictModelServiceImpl.java

@@ -9,6 +9,7 @@ import com.tzld.piaoquan.ad.engine.commons.redis.AdRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.util.AbUtil;
 import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
+import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import com.tzld.piaoquan.ad.engine.service.predict.PredictModelService;
 import com.tzld.piaoquan.ad.engine.service.predict.config.AbConfig;
 import com.tzld.piaoquan.ad.engine.service.predict.config.NewExpUserGroupConfig;
@@ -23,6 +24,7 @@ import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelPa
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.RoiPredictModelRequestParam;
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.ThresholdPredictModelRequestParam;
 import com.tzld.piaoquan.ad.engine.service.predict.v2.ConvertUtil;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
 import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictServiceV2;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -61,6 +63,8 @@ public class PredictModelServiceImpl implements PredictModelService {
 
     @Autowired
     private PredictServiceV2 predictServiceV2;
+    @Autowired
+    private LogHubService logHubService;
 
     @Value("${ad.predict.share0.exp.code:000}")
     private String adPredictNoShareUserExpCode;
@@ -157,8 +161,11 @@ public class PredictModelServiceImpl implements PredictModelService {
                 return result;
             }
 
-            if (AbUtil.isInAbExp(expCodes, requestParam.getAppType(), requestParam.getNewExpGroup(), "713")){
-                return predictServiceV2.adPredict(ConvertUtil.predictParam2Context(requestParam));
+            if (AbUtil.isInAbExp(expCodes, requestParam.getAppType(), requestParam.getNewExpGroup(), "713")) {
+                PredictContext context = ConvertUtil.predictParam2Context(requestParam);
+                Map<String, Object> resultMap = predictServiceV2.adPredict(context);
+                logHubService.crowdChooseLogUpload(context);
+                return resultMap;
             }
 
             String abtestId = null;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/PredictLogParam.java

@@ -25,4 +25,6 @@ public class PredictLogParam {
     private Map<String, Double> scoreMap = new HashMap<>();
 
     private boolean aIsShowAd;
+
+    private boolean bIsNewUser;
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/PredictServiceV2.java

@@ -4,7 +4,6 @@ import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.util.NumUtil;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
-import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -19,93 +18,175 @@ public class PredictServiceV2 {
 
     @Autowired
     private FeatureService featureService;
-    @Autowired
-    private LogHubService logHubService;
 
     @ApolloJsonValue("${exp.713.config:{}}")
     private Map<String, Double> exp713Config;
 
     public Map<String, Object> adPredict(PredictContext context) {
-
-        Feature feature = featureService.getPredictFeature(context.getMid());
+        Feature feature = featureService.getPredictFeature(context);
         Map<String, Map<String, String>> userFeature = feature.getUserFeature();
-        Map<String, String> featureMap = userFeature.getOrDefault("alg_ad_crowd_choose_feature", new HashMap<>());
-
-        double score = -1;
-
-        // 没有特征为新用户，随机出广告
-        if (MapUtils.isEmpty(featureMap)) {
-            double newUserShowAdRate = exp713Config.getOrDefault("newUserShowAdRate", 0.8d);
-            double randomRate = Math.random();
-            if (randomRate < newUserShowAdRate) {
-                context.getLogParam().getScoreMap().put("newUserShowAdRate", newUserShowAdRate);
-                context.getLogParam().getScoreMap().put("randomRate", randomRate);
-                score = 1;
-            }
-        } else {
-            context.getLogParam().getMetaFeature().putAll(userFeature);
+        Map<String, String> featureMap = userFeature.getOrDefault("alg_ad_crowd_choose_feature_v2", new HashMap<>());
+        double minScore = exp713Config.getOrDefault("minScore", 0.1d);
+        double maxScore = exp713Config.getOrDefault("maxScore", 0.8d);
+        double score = maxScore;
+
+        context.getLogParam().setBIsNewUser(true);
+
+        if (MapUtils.isNotEmpty(featureMap)) {
+
+            double adViewCnt = Double.parseDouble(featureMap.getOrDefault("ad_view_cnt", "0"));
+            double adClick = Double.parseDouble(featureMap.getOrDefault("ad_click", "0"));
+            double adConver = Double.parseDouble(featureMap.getOrDefault("ad_conver", "0"));
+            double hasAdClick = Double.parseDouble(featureMap.getOrDefault("has_ad_click", "0"));
+            double hasAdShare = Double.parseDouble(featureMap.getOrDefault("has_ad_share", "0"));
+            double hasAdReturnCnt = Double.parseDouble(featureMap.getOrDefault("has_ad_return_cnt", "0"));
+            double noAdClick = Double.parseDouble(featureMap.getOrDefault("no_ad_click", "0"));
+            double noAdShare = Double.parseDouble(featureMap.getOrDefault("no_ad_share", "0"));
+            double noAdReturnCnt = Double.parseDouble(featureMap.getOrDefault("no_ad_return_cnt", "0"));
+
+            // 计算出回流出广告时的收益
+            double adClickValue = NumUtil.div(adClick, adViewCnt) * NumUtil.log10(adClick);
+            double adConverValue = NumUtil.div(adConver, adViewCnt) * NumUtil.log10(adConver);
+            double hasAdShareValue = NumUtil.div(hasAdShare, hasAdClick) * NumUtil.log10(hasAdShare);
+            double hasAdReturnValue = NumUtil.div(hasAdReturnCnt, hasAdClick) * NumUtil.log10(hasAdReturnCnt);
+            double hasAdValue = adClickValue + adConverValue + hasAdShareValue + hasAdReturnValue;
+
+            // 计算回流不出广告时的收益
+            double noAdShareValue = NumUtil.div(noAdShare, noAdClick) * NumUtil.log10(noAdShare);
+            double noAdReturnValue = NumUtil.div(noAdReturnCnt, noAdClick) * NumUtil.log10(noAdReturnCnt + hasAdReturnCnt);
+            double noAdValue = noAdShareValue + noAdReturnValue;
+
+            // 计算最终的收益
+            double hasRate = exp713Config.getOrDefault("hasRate", 1d);
+            double noRate = exp713Config.getOrDefault("noRate", 1d);
+            score = NumUtil.softmax(new double[]{hasAdValue * hasRate, noAdValue * noRate})[0];
+
+
+            context.getLogParam().getMetaFeature().putAll(feature.getUserFeature());
             for (Map.Entry<String, String> entry : featureMap.entrySet()) {
                 context.getLogParam().getAllFeature().put(entry.getKey(), Double.parseDouble(entry.getValue()));
             }
+            context.getLogParam().getScoreMap().put("adClickValue", NumUtil.round(adClickValue, 6));
+            context.getLogParam().getScoreMap().put("adConverValue", NumUtil.round(adConverValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdShareValue", NumUtil.round(hasAdShareValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdReturnValue", NumUtil.round(hasAdReturnValue, 6));
+            context.getLogParam().getScoreMap().put("hasAdValue", NumUtil.round(hasAdValue, 6));
+            context.getLogParam().getScoreMap().put("noAdShareValue", NumUtil.round(noAdShareValue, 6));
+            context.getLogParam().getScoreMap().put("noAdReturnValue", NumUtil.round(noAdReturnValue, 6));
+            context.getLogParam().getScoreMap().put("noAdValue", NumUtil.round(noAdValue, 6));
+            context.getLogParam().getScoreMap().put("originScore", NumUtil.round(score, 6));
+            context.getLogParam().getScoreMap().put("hasRate", NumUtil.round(hasRate, 6));
+            context.getLogParam().getScoreMap().put("noRate", NumUtil.round(noRate, 6));
+
+            context.getLogParam().setBIsNewUser(false);
+        }
 
-            // 获取需要的特征值
-            double showAdClickPv = Double.parseDouble(featureMap.getOrDefault("show_ad_click_pv", "0"));
-            double noShowAdClickPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_click_pv", "0"));
-            double showAdIncome = Double.parseDouble(featureMap.getOrDefault("show_ad_income", "0"));
-            double showAdSharePv = Double.parseDouble(featureMap.getOrDefault("show_ad_share_pv", "0"));
-            double noShowAdSharePv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_share_pv", "0"));
-            double showAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("show_ad_new_return_pv", "0"));
-            double noShowAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_new_return_pv", "0"));
-
-            // 计算中间过程值
-            double singleReturnAdIncome = NumUtil.div(showAdIncome, (showAdClickPv + noShowAdClickPv));
-            double showAdShareRate = NumUtil.div((showAdSharePv + 1), (showAdClickPv + 1));
-            double noShowAdShareRate = NumUtil.div((noShowAdSharePv + 1), (noShowAdClickPv + 1));
-            double returnDivShare = NumUtil.div((showAdNewReturnPv + noShowAdNewReturnPv + 1), (showAdSharePv + noShowAdSharePv + 1));
-
-            double busDauBalanceRate = exp713Config.getOrDefault("busDauBalanceRate", 0.1d);
-            score = singleReturnAdIncome + ((showAdShareRate - noShowAdShareRate) * returnDivShare * busDauBalanceRate);
-
-
-            context.getLogParam().getScoreMap().put("singleReturnAdIncome", singleReturnAdIncome);
-            context.getLogParam().getScoreMap().put("showAdShareRate", showAdShareRate);
-            context.getLogParam().getScoreMap().put("noShowAdShareRate", noShowAdShareRate);
-            context.getLogParam().getScoreMap().put("returnDivShare", returnDivShare);
-            context.getLogParam().getScoreMap().put("busDauBalanceRate", busDauBalanceRate);
-
+        // 分数截断，避免过长或过短
+        if (score < minScore) {
+            score = minScore;
+        } else if (score > maxScore) {
+            score = maxScore;
         }
-        double showAdScoreThreshold = exp713Config.getOrDefault("showAdScoreThreshold", 0d);
-        boolean isShowAd = score >= showAdScoreThreshold;
 
-        context.getLogParam().setAIsShowAd(isShowAd);
+        double random = Math.random();
+        boolean isShowAd = random < score;
         context.getLogParam().setExpId("713");
         context.getLogParam().setScore(score);
-        context.getLogParam().getScoreMap().put("score", score);
-        context.getLogParam().getScoreMap().put("showAdScoreThreshold", showAdScoreThreshold);
-
-
-        logHubService.crowdChooseLogUpload(context);
+        context.getLogParam().getScoreMap().put("score", NumUtil.round(score, 6));
+        context.getLogParam().setAIsShowAd(isShowAd);
+        context.getLogParam().getScoreMap().put("minScore", minScore);
+        context.getLogParam().getScoreMap().put("maxScore", maxScore);
+        context.getLogParam().getScoreMap().put("random", random);
 
 
-        if (isShowAd) {
-            Map<String, Object> rtnMap = rtnAdPredict();
-            rtnMap.putAll(context.getLogParam().getScoreMap());
-            return rtnMap;
-        } else {
-            return rtnNoAdPredict("713_exp");
-        }
+        return isShowAd ? rtnAdPredict(context) : rtnNoAdPredict(context);
     }
 
-    private Map<String, Object> rtnNoAdPredict(String noAdStrategy) {
+
+    private Map<String, Object> rtnNoAdPredict(PredictContext context) {
         Map<String, Object> rtnMap = new HashMap<>();
         rtnMap.put("ad_predict", 1);
-        rtnMap.put("no_ad_strategy", noAdStrategy);
+        rtnMap.put("pqtid", context.getPqtId());
+        rtnMap.put("no_ad_strategy", "713_exp");
         return rtnMap;
     }
 
-    private Map<String, Object> rtnAdPredict() {
+    private Map<String, Object> rtnAdPredict(PredictContext context) {
         Map<String, Object> rtnMap = new HashMap<>();
         rtnMap.put("ad_predict", 2);
+        rtnMap.put("pqtid", context.getPqtId());
+        rtnMap.putAll(context.getLogParam().getScoreMap());
         return rtnMap;
     }
+
+    // public Map<String, Object> adPredictV1(PredictContext context){
+    //
+    //     Feature feature = featureService.getPredictFeature(context);
+    //     Map<String, Map<String, String>> userFeature = feature.getUserFeature();
+    //     Map<String, String> featureMap = userFeature.getOrDefault("alg_ad_crowd_choose_feature", new HashMap<>());
+    //
+    //     double score = -1;
+    //
+    //     // 没有特征为新用户，随机出广告
+    //     if (MapUtils.isEmpty(featureMap)) {
+    //         double newUserShowAdRate = exp713Config.getOrDefault("newUserShowAdRate", 0.8d);
+    //         double randomRate = Math.random();
+    //         if (randomRate < newUserShowAdRate) {
+    //             context.getLogParam().getScoreMap().put("newUserShowAdRate", newUserShowAdRate);
+    //             context.getLogParam().getScoreMap().put("randomRate", randomRate);
+    //             score = 1;
+    //         }
+    //     } else {
+    //         context.getLogParam().getMetaFeature().putAll(userFeature);
+    //         for (Map.Entry<String, String> entry : featureMap.entrySet()) {
+    //             context.getLogParam().getAllFeature().put(entry.getKey(), Double.parseDouble(entry.getValue()));
+    //         }
+    //
+    //         // 获取需要的特征值
+    //         double showAdClickPv = Double.parseDouble(featureMap.getOrDefault("show_ad_click_pv", "0"));
+    //         double noShowAdClickPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_click_pv", "0"));
+    //         double showAdIncome = Double.parseDouble(featureMap.getOrDefault("show_ad_income", "0"));
+    //         double showAdSharePv = Double.parseDouble(featureMap.getOrDefault("show_ad_share_pv", "0"));
+    //         double noShowAdSharePv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_share_pv", "0"));
+    //         double showAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("show_ad_new_return_pv", "0"));
+    //         double noShowAdNewReturnPv = Double.parseDouble(featureMap.getOrDefault("no_show_ad_new_return_pv", "0"));
+    //
+    //         // 计算中间过程值
+    //         double singleReturnAdIncome = NumUtil.div(showAdIncome, (showAdClickPv + noShowAdClickPv));
+    //         double showAdShareRate = NumUtil.div((showAdSharePv + 1), (showAdClickPv + 1));
+    //         double noShowAdShareRate = NumUtil.div((noShowAdSharePv + 1), (noShowAdClickPv + 1));
+    //         double returnDivShare = NumUtil.div((showAdNewReturnPv + noShowAdNewReturnPv + 1), (showAdSharePv + noShowAdSharePv + 1));
+    //
+    //         double busDauBalanceRate = exp713Config.getOrDefault("busDauBalanceRate", 0.1d);
+    //         score = singleReturnAdIncome + ((showAdShareRate - noShowAdShareRate) * returnDivShare * busDauBalanceRate);
+    //
+    //
+    //         context.getLogParam().getScoreMap().put("singleReturnAdIncome", singleReturnAdIncome);
+    //         context.getLogParam().getScoreMap().put("showAdShareRate", showAdShareRate);
+    //         context.getLogParam().getScoreMap().put("noShowAdShareRate", noShowAdShareRate);
+    //         context.getLogParam().getScoreMap().put("returnDivShare", returnDivShare);
+    //         context.getLogParam().getScoreMap().put("busDauBalanceRate", busDauBalanceRate);
+    //
+    //     }
+    //     double showAdScoreThreshold = exp713Config.getOrDefault("showAdScoreThreshold", 0d);
+    //     boolean isShowAd = score >= showAdScoreThreshold;
+    //
+    //     context.getLogParam().setAIsShowAd(isShowAd);
+    //     context.getLogParam().setExpId("713");
+    //     context.getLogParam().setScore(score);
+    //     context.getLogParam().getScoreMap().put("score", score);
+    //     context.getLogParam().getScoreMap().put("showAdScoreThreshold", showAdScoreThreshold);
+    //
+    //
+    //     logHubService.crowdChooseLogUpload(context);
+    //
+    //
+    //     if (isShowAd) {
+    //         Map<String, Object> rtnMap = rtnAdPredict();
+    //         rtnMap.putAll(context.getLogParam().getScoreMap());
+    //         return rtnMap;
+    //     } else {
+    //         return rtnNoAdPredict("713_exp");
+    //     }
+    // }
 }