Merge branch 'feature_20240605_zhaohaipeng_ad_filter_exp' of algorithm/ad-engine into master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/JSONUtils.java

@@ -2,14 +2,49 @@ package com.tzld.piaoquan.ad.engine.commons.util;
 
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.reflect.TypeToken;
+import com.google.gson.ExclusionStrategy;
+import com.google.gson.FieldAttributes;
 import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+
 public class JSONUtils {
 
     private static final Logger log = LoggerFactory.getLogger(JSONUtils.class);
+
+    public static String toJson(Object obj, Collection<String> ignoreFieldName) {
+        if (Objects.isNull(obj)) {
+            return "";
+        }
+        try {
+
+            ExclusionStrategy strategy = new ExclusionStrategy() {
+                @Override
+                public boolean shouldSkipField(FieldAttributes fieldAttributes) {
+                    return CollectionUtils.isNotEmpty(ignoreFieldName) &&
+                            ignoreFieldName.contains(fieldAttributes.getName());
+                }
+
+                @Override
+                public boolean shouldSkipClass(Class<?> aClass) {
+                    return false;
+                }
+            };
+            return new GsonBuilder().setExclusionStrategies(strategy).create().toJson(obj);
+        } catch (Exception e) {
+            log.error("toJsonIgnoreField exception: ", e);
+            return "";
+        }
+    }
+
     public static String toJson(Object obj) {
         if (obj == null) {
             return "";

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/ControllerAspect.java

@@ -1,6 +1,7 @@
 package com.tzld.piaoquan.ad.engine.server.controller;
 
 import com.google.common.base.Stopwatch;
+import com.google.common.collect.Sets;
 import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
 import com.tzld.piaoquan.ad.engine.commons.util.TraceUtils;
 import lombok.extern.slf4j.Slf4j;
@@ -12,6 +13,7 @@ import org.aspectj.lang.annotation.Pointcut;
 import org.aspectj.lang.reflect.MethodSignature;
 import org.springframework.stereotype.Component;
 
+import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -23,12 +25,12 @@ import java.util.concurrent.TimeUnit;
 public class ControllerAspect {
 
     @Pointcut("execution(* com.tzld.piaoquan.ad.engine.server.controller.*Controller.*(..))")
-    public void logPointcut(){
+    public void logPointcut() {
 
     }
 
     @Pointcut("execution(* com.tzld.piaoquan.ad.engine.server.controller.HealthController.*(..))")
-    public void excludePointcut(){
+    public void excludePointcut() {
 
     }
 
@@ -39,13 +41,15 @@ public class ControllerAspect {
         String className = pjp.getTarget().getClass().getSimpleName();
         MethodSignature signature = (MethodSignature) pjp.getSignature();
         Stopwatch stopwatch = Stopwatch.createStarted();
-        log.info("request className=[{}], method=[{}], param=[{}]", className, signature.getName(),
-                JSONUtils.toJson(pjp.getArgs()));
+
+        String param = JSONUtils.toJson(pjp.getArgs(), Sets.newHashSet("statisticsLog"));
+
+        log.info("request className=[{}], method=[{}], param=[{}]", className, signature.getName(), param);
         Object result = pjp.proceed();
         if (result != null && result instanceof String) {
-            log.info("request method=[{}]  param=[{}] result=[{}] cost=[{}]", signature.getName(),JSONUtils.toJson(pjp.getArgs()),result, stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+            log.info("request method=[{}]  param=[{}] result=[{}] cost=[{}]", signature.getName(), param, result, stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
         } else {
-            log.info("request method=[{}]  param=[{}] result=[{}] cost=[{}]", signature.getName(),JSONUtils.toJson(pjp.getArgs()), JSONUtils.toJson(result), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
+            log.info("request method=[{}]  param=[{}] result=[{}] cost=[{}]", signature.getName(), param, JSONUtils.toJson(result), stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
         }
         TraceUtils.removeMDC();
         return result;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -83,6 +83,10 @@ public class LogHubServiceImpl implements LogHubService {
                     logMap.put("earlyAdIds", requestParam.getStatisticsLog().getEarlyAdIds());
                     logMap.put("earlyCidList", requestParam.getStatisticsLog().getEarlyCreativeIds());
                     logMap.put("finalCidList", requestParam.getStatisticsLog().getFinalCreativeIds());
+                    logMap.put("commonFilterAfterAdIds", requestParam.getStatisticsLog().getCommonFilterAfterAdIds());
+                    logMap.put("commonFilterAfterCidList", requestParam.getStatisticsLog().getCommonFilterAfterCreativeIds());
+                    logMap.put("tacticsFilterAfterAdIds", requestParam.getStatisticsLog().getTacticsFilterAfterAdIds());
+                    logMap.put("tacticsFilterAfterCidList", requestParam.getStatisticsLog().getTacticsFilterAfterCreativeIds());
                 }
 
                 aliyunLogManager.sendLog(project, logStore, "", logMap);

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VideoAdThompsonScorerV2.java

@@ -37,8 +37,8 @@ public class VideoAdThompsonScorerV2 {
     private Map<String, Double> exp665Param = new HashMap<>();
     private Map<String, Double> exp666Param = new HashMap<>();
     private Map<String, Double> exp669Param = new HashMap<>();
-    private Map<String, Double> exp670Param = new HashMap<>();
-    private Map<String, String> exp672Param = new HashMap<>();
+    private Map<String, Object> exp670Param = new HashMap<>();
+    private Map<String, Object> exp672Param = new HashMap<>();
 
     Random random = new Random();
     Gson gson = new Gson();
@@ -128,7 +128,6 @@ public class VideoAdThompsonScorerV2 {
                 ext.put("pairOrSingle", "single");
                 ext.put("cidStatistic", JSON.toJSONString(cidStatistics));
                 ext.put("cpa", cpa);
-                ext.put("abCode", 663);
             } catch (Exception e) {
                 log.error("svc=663exp, error: ", e);
             }
@@ -364,7 +363,6 @@ public class VideoAdThompsonScorerV2 {
                 ext.put("cidStatistic", JSON.toJSONString(cidStatistics));
                 ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistics));
                 ext.put("cpa", cpa);
-                ext.put("abCode", 666);
             } catch (Exception e) {
                 log.error("svc=666exp, error: ", e);
             }
@@ -397,30 +395,85 @@ public class VideoAdThompsonScorerV2 {
         for (AdRankItem adRankItem : result) {
             adRankItem.setVideoId(param.getVideoId());
             adRankItem.setScore_type(699);
-            if (MapUtils.isNotEmpty(adRankItem.getExt())) {
-                adRankItem.getExt().put("abCode", 699);
-            }
         }
 
         return result;
     }
 
     public List<AdRankItem> thompsonScorerByExp670(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
-        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(redisCidActionKeyV1, adIdList);
-        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(redisCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
+        Map<Long, CreativeStatistic> cidMap = this.batchFindCreativeRedisCache(redisCidActionKeyV1, adIdList);
+        Map<Long, CreativeStatistic> vidCidMap = this.batchFindCreativeRedisCache(redisCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
+
+        double w1 = Double.parseDouble(exp670Param.getOrDefault("w1", "1").toString());
+        double w2 = Double.parseDouble(exp670Param.getOrDefault("w2", "1").toString());
+
+        double alpha = Double.parseDouble(exp670Param.getOrDefault("alpha", "0").toString());
+        double cidBeta = Double.parseDouble(exp670Param.getOrDefault("cidBeta", "0").toString());
+        double vidCidBeta = Double.parseDouble(exp670Param.getOrDefault("vidCidBeta", "0").toString());
+
 
         List<AdRankItem> result = new ArrayList<>(adIdList.size());
-        this.calcScore(result, adIdList, 0.0, 0.0, 0.0, creativeStatisticsMap, videoCreativeStatisticsMap, exp670Param);
-        result.sort(equalsRandomComparator());
 
-        for (AdRankItem adRankItem : result) {
-            adRankItem.setVideoId(param.getVideoId());
-            adRankItem.setScore_type(670);
-            if (MapUtils.isNotEmpty(adRankItem.getExt())) {
-                adRankItem.getExt().put("abCode", 670);
+        for (AdPlatformCreativeDTO dto : adIdList) {
+
+            // 日志上报扩展参数
+            Map<String, Object> ext = new HashMap<>();
+            double score = 0;
+            double cpa = 0.0;
+            try {
+
+                CreativeStatistic cidStatistic = cidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
+                CreativeStatistic vidCidStatistic = vidCidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
+
+                cpa = cidStatistic.parseCpaToDouble();
+                double cidExp = cidStatistic.parseExpToDouble();
+                double vidCidExp = vidCidStatistic.parseExpToDouble();
+
+                // double cidScore = this.betaSampler(cidStatistic.parseOrderToDouble() + alpha, cidStatistic.parseExpToDouble() + cidBeta);
+                // double vidCidScore = this.betaSampler(vidCidStatistic.parseOrderToDouble() + alpha, vidCidStatistic.parseExpToDouble() + vidCidBeta);
+                double cidScore = this.orderDivideExp(cidStatistic);
+                double vidCidScore = this.orderDivideExp(vidCidStatistic);
+
+
+                if (w1 != 0 || w2 != 0) {
+                    score = ((w1 * vidCidScore * vidCidExp + w2 * cidScore * cidExp) / (w1 * vidCidExp + w2 * cidExp)) * cpa;
+                }
+
+                ext.put("w1", w1);
+                ext.put("w2", w2);
+                ext.put("cidScore", cidScore);
+                ext.put("vidCidScore", vidCidScore);
+                ext.put("cidA", alpha);
+                ext.put("vidCidA", alpha);
+                ext.put("cidB", cidBeta);
+                ext.put("vidCidB", vidCidBeta);
+                ext.put("cidExp", cidExp);
+                ext.put("vidCidExp", vidCidExp);
+                ext.put("bid1", dto.getBid1());
+                ext.put("bid2", dto.getBid2());
+                ext.put("cidStatistic", JSON.toJSONString(cidStatistic));
+                ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistic));
+                ext.put("cpa", cpa);
+            } catch (Exception e) {
+                log.error("svc=670exp, error: ", e);
             }
+
+            AdRankItem item = new AdRankItem();
+            item.setCpa(cpa);
+            item.setAdId(dto.getCreativeId());
+            item.setWeight(dto.getWeight());
+            item.setScore(score);
+            item.setExt(ext);
+            item.setBid1(dto.getBid1());
+            item.setBid2(dto.getBid2());
+            item.setCreativeCode(dto.getCreativeCode());
+            item.setScore_type(670);
+            item.setVideoId(param.getVideoId());
+            result.add(item);
+
         }
 
+        result.sort(equalsRandomComparator());
         return result;
     }
 
@@ -502,6 +555,15 @@ public class VideoAdThompsonScorerV2 {
         return resultMap;
     }
 
+    private double orderDivideExp(CreativeStatistic statistic) {
+        double order = statistic.parseOrderToDouble();
+        double exp = statistic.parseExpToDouble();
+        if (exp != 0) {
+            return order / exp;
+        }
+        return 0.0;
+    }
+
     private double calcThompsonScore(Map<String, Double> expParam, CreativeStatistic creativeStatistic, Double defaultAlpha, Double defaultBeta) {
         Double alpha = expParam.getOrDefault("alpha", defaultAlpha);
         Double beta = expParam.getOrDefault("beta", defaultBeta);
@@ -538,7 +600,6 @@ public class VideoAdThompsonScorerV2 {
 
     private Map<String, Object> extMap(CreativeStatistic statistic, String abCode, Double cpa, Double viewThreshold, Double alpha, Double beta, Double beta_k) {
         Map<String, Object> map = new HashMap<>();
-        map.put("abCode", abCode);
         if (Objects.nonNull(viewThreshold)) {
             map.put("viewThreshold", viewThreshold);
         }
@@ -555,9 +616,9 @@ public class VideoAdThompsonScorerV2 {
     }
 
     public List<AdRankItem> thompsonScorerByExp672(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
-        String cidRedisKey = exp672Param.getOrDefault("cidActionRedisKey", redisCreativeStatisticsPrefix);
-        String vidCidRedisKey = exp672Param.getOrDefault("vidCidActionRedisKey", redisVideoCreativeStatisticsPrefix);
-        double viewThreshold = Double.parseDouble(exp672Param.getOrDefault("viewThreshold", "7000"));
+        String cidRedisKey = exp672Param.getOrDefault("cidActionRedisKey", redisCreativeStatisticsPrefix).toString();
+        String vidCidRedisKey = exp672Param.getOrDefault("vidCidActionRedisKey", redisVideoCreativeStatisticsPrefix).toString();
+        double viewThreshold = Double.parseDouble(exp672Param.getOrDefault("viewThreshold", "7000").toString());
 
         // RedisKey前缀判断
         if (!StringUtils.equals(redisCidActionKeyV1, cidRedisKey) && !StringUtils.equals(redisCreativeStatisticsPrefix, cidRedisKey)) {
@@ -603,7 +664,6 @@ public class VideoAdThompsonScorerV2 {
                 ext.put("cidStatistic", JSON.toJSONString(cidStatistic));
                 ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistic));
                 ext.put("cpa", cpa);
-                ext.put("abCode", 672);
             } catch (Exception e) {
                 log.error("svc=672exp, error: ", e);
             }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/StatisticsLogParam.java

@@ -16,4 +16,11 @@ public class StatisticsLogParam {
 
     private List<Long> finalCreativeIds;
 
+    private List<Long> commonFilterAfterAdIds;
+
+    private List<Long> commonFilterAfterCreativeIds;
+
+    private List<Long> tacticsFilterAfterAdIds;
+
+    private List<Long> tacticsFilterAfterCreativeIds;
 }