Merge branch 'feature_20240605_zhaohaipeng_ad_filter_exp' of algorithm/ad-engine into master

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -18,8 +18,6 @@ import java.util.*;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 
 @Slf4j
 @Service
@@ -61,6 +59,7 @@ public class LogHubServiceImpl implements LogHubService {
                 for (AdRankItem rankItem : rankItems) {
                     JSONObject json = new JSONObject();
                     json.put("adId", rankItem.getAdId());
+                    json.put("cid", rankItem.getAdId());
                     json.put("score", rankItem.getScore());
                     json.put("ext", rankItem.getExt());
                     json.put("weight", rankItem.getWeight());
@@ -70,6 +69,7 @@ public class LogHubServiceImpl implements LogHubService {
 
                 AdRankItem top1 = rankItems.get(0);
                 logMap.put("top1_adId", top1.getAdId());
+                logMap.put("top1_cid", top1.getAdId());
                 logMap.put("top1_score", top1.getScore());
                 logMap.put("top1_ext", JSON.toJSONString(top1.getExt()));
                 logMap.put("top1_weight", top1.getWeight());
@@ -78,6 +78,13 @@ public class LogHubServiceImpl implements LogHubService {
                 logMap.put("adAbGroup", requestParam.getAdAbGroup());
                 logMap.put("scoreStrategy", scoreStrategy);
                 logMap.put("appType", requestParam.getAppType());
+
+                if (Objects.nonNull(requestParam.getStatisticsLog())) {
+                    logMap.put("earlyAdIds", requestParam.getStatisticsLog().getEarlyAdIds());
+                    logMap.put("earlyCidList", requestParam.getStatisticsLog().getEarlyCreativeIds());
+                    logMap.put("finalCidList", requestParam.getStatisticsLog().getFinalCreativeIds());
+                }
+
                 aliyunLogManager.sendLog(project, logStore, "", logMap);
             }
         });

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VideoAdThompsonScorerV2.java

@@ -29,8 +29,8 @@ public class VideoAdThompsonScorerV2 {
     // redis:vid_cid_action:{$vid}_{$cid}
     private String redisVideoCreativeStatisticsPrefix = "redis:vid_cid_action:";
 
-    private String rediCidActionKeyV1 = "redis:cid_action_v1:";
-    private String rediCidVidActionKeyV1 = "redis:vid_cid_action_v1:";
+    private final String redisCidActionKeyV1 = "redis:cid_action_v1:";
+    private final String redisCidVidActionKeyV1 = "redis:vid_cid_action_v1:";
 
     private Map<String, Double> exp663Param = new HashMap<>();
     private Map<String, Double> exp664Param = new HashMap<>();
@@ -38,6 +38,7 @@ public class VideoAdThompsonScorerV2 {
     private Map<String, Double> exp666Param = new HashMap<>();
     private Map<String, Double> exp669Param = new HashMap<>();
     private Map<String, Double> exp670Param = new HashMap<>();
+    private Map<String, String> exp672Param = new HashMap<>();
 
     Random random = new Random();
     Gson gson = new Gson();
@@ -100,8 +101,10 @@ public class VideoAdThompsonScorerV2 {
             // 日志上报扩展参数
             Map<String, Object> ext = new HashMap<>();
             double score = 0.0;
+            double cpa = 0.0;
             try {
                 CreativeStatistic cidStatistics = creativeStatisticsMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
+                cpa = cidStatistics.parseCpaToDouble();
 
                 double order;
                 double exp;
@@ -115,31 +118,27 @@ public class VideoAdThompsonScorerV2 {
                     s1 = a / b;
                 }
 
-                score = s1 * dto.getCpa() * dto.getBid1() * dto.getBid2();
+                score = s1 * cpa;
 
                 ext.put("alpha", alpha);
                 ext.put("beta", beta);
                 ext.put("order", order);
                 ext.put("exp", exp);
                 ext.put("s1", s1);
-                ext.put("bid1", dto.getBid1());
-                ext.put("bid2", dto.getBid2());
                 ext.put("pairOrSingle", "single");
                 ext.put("cidStatistic", JSON.toJSONString(cidStatistics));
-                ext.put("cpa", dto.getCpa());
+                ext.put("cpa", cpa);
                 ext.put("abCode", 663);
             } catch (Exception e) {
                 log.error("svc=663exp, error: ", e);
             }
             AdRankItem item = new AdRankItem();
-            item.setCpa(dto.getCpa());
+            item.setCpa(cpa);
             item.setAdId(dto.getCreativeId());
             item.setScore(score);
             item.setExt(ext);
             item.setVideoId(param.getVideoId());
             item.setScore_type(663);
-            item.setBid1(dto.getBid1());
-            item.setBid2(dto.getBid2());
             item.setWeight(dto.getWeight());
             item.setCreativeCode(dto.getCreativeCode());
             result.add(item);
@@ -336,6 +335,7 @@ public class VideoAdThompsonScorerV2 {
             // 日志上报扩展参数
             Map<String, Object> ext = new HashMap<>();
             double score = 0.0;
+            double cpa = cidStatistics.parseCpaToDouble();
             try {
 
 
@@ -350,7 +350,7 @@ public class VideoAdThompsonScorerV2 {
                 }
 
                 double s1 = this.betaSampler((alpha + order), ((beta + exp) / betaK));
-                score = s1 * dto.getCpa() * dto.getBid1() * dto.getBid2();
+                score = s1 * cpa;
 
 
                 ext.put("alpha", alpha);
@@ -359,26 +359,22 @@ public class VideoAdThompsonScorerV2 {
                 ext.put("order", order);
                 ext.put("exp", exp);
                 ext.put("s1", s1);
-                ext.put("bid1", dto.getBid1());
-                ext.put("bid2", dto.getBid2());
                 ext.put("viewThreshold", viewThreshold);
                 ext.put("pairOrSingle", pairOrSingle);
                 ext.put("cidStatistic", JSON.toJSONString(cidStatistics));
                 ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistics));
-                ext.put("cpa", dto.getCpa());
+                ext.put("cpa", cpa);
                 ext.put("abCode", 666);
             } catch (Exception e) {
                 log.error("svc=666exp, error: ", e);
             }
             AdRankItem item = new AdRankItem();
-            item.setCpa(dto.getCpa());
+            item.setCpa(cpa);
             item.setAdId(dto.getCreativeId());
             item.setScore(score);
             item.setExt(ext);
             item.setVideoId(param.getVideoId());
             item.setScore_type(666);
-            item.setBid2(dto.getBid2());
-            item.setBid1(dto.getBid1());
             item.setWeight(dto.getWeight());
             item.setCreativeCode(dto.getCreativeCode());
             result.add(item);
@@ -389,8 +385,8 @@ public class VideoAdThompsonScorerV2 {
     }
 
     public List<AdRankItem> thompsonScorerByExp669(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
-        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidActionKeyV1, adIdList);
-        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
+        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(redisCidActionKeyV1, adIdList);
+        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(redisCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
         Double creativeExpSum = this.sumCreativeStatisticExp(creativeStatisticsMap.values());
         Double videoCreativeExpSum = this.sumCreativeStatisticExp(videoCreativeStatisticsMap.values());
 
@@ -410,8 +406,8 @@ public class VideoAdThompsonScorerV2 {
     }
 
     public List<AdRankItem> thompsonScorerByExp670(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
-        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidActionKeyV1, adIdList);
-        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
+        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(redisCidActionKeyV1, adIdList);
+        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(redisCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
 
         List<AdRankItem> result = new ArrayList<>(adIdList.size());
         this.calcScore(result, adIdList, 0.0, 0.0, 0.0, creativeStatisticsMap, videoCreativeStatisticsMap, exp670Param);
@@ -434,11 +430,21 @@ public class VideoAdThompsonScorerV2 {
             CreativeStatistic cidStatistic = cidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
             CreativeStatistic vidCidStatistic = vidCidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
 
+            double cidExp = cidStatistic.parseExpToDouble();
+            double vidCidExp = vidCidStatistic.parseExpToDouble();
+
             double cidScore = this.calcThompsonScore(expParam, cidStatistic, alpha, cidBeta);
             double vidCidScore = this.calcThompsonScore(expParam, vidCidStatistic, alpha, vidCidBeta);
+
             double w1 = expParam.getOrDefault("w1", 1d);
             double w2 = expParam.getOrDefault("w2", 1d);
-            double score = ((w1 * vidCidScore + w2 * cidScore) / (w1 + w2)) * dto.getCpa();
+
+            double cpa = cidStatistic.parseCpaToDouble();
+
+            double score = 0;
+            if (w1 != 0 || w2 != 0) {
+                score = ((w1 * vidCidScore * vidCidExp + w2 * cidScore * cidExp) / (w1 * vidCidExp + w2 * cidExp)) * cpa;
+            }
 
 
             // 日志上报扩展参数
@@ -451,17 +457,19 @@ public class VideoAdThompsonScorerV2 {
             ext.put("vidCidA", alpha);
             ext.put("cidB", cidBeta);
             ext.put("vidCidB", vidCidBeta);
+            ext.put("cidExp", cidExp);
+            ext.put("vidCidExp", vidCidExp);
             ext.put("bid1", dto.getBid1());
             ext.put("bid2", dto.getBid2());
             ext.put("cidStatistic", JSON.toJSONString(cidStatistic));
             ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistic));
-            ext.put("cpa", dto.getCpa());
+            ext.put("cpa", cpa);
 
             log.info("svc=699And670Score, score: {}, param: {}", score, JSON.toJSONString(ext));
 
 
             AdRankItem item = new AdRankItem();
-            item.setCpa(dto.getCpa());
+            item.setCpa(cpa);
             item.setAdId(dto.getCreativeId());
             item.setWeight(dto.getWeight());
             item.setScore(score);
@@ -471,6 +479,7 @@ public class VideoAdThompsonScorerV2 {
             item.setCreativeCode(dto.getCreativeCode());
 
             result.add(item);
+
         }
     }
 
@@ -545,6 +554,76 @@ public class VideoAdThompsonScorerV2 {
         return map;
     }
 
+    public List<AdRankItem> thompsonScorerByExp672(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        String cidRedisKey = exp672Param.getOrDefault("cidActionRedisKey", redisCreativeStatisticsPrefix);
+        String vidCidRedisKey = exp672Param.getOrDefault("vidCidActionRedisKey", redisVideoCreativeStatisticsPrefix);
+        double viewThreshold = Double.parseDouble(exp672Param.getOrDefault("viewThreshold", "7000"));
+
+        // RedisKey前缀判断
+        if (!StringUtils.equals(redisCidActionKeyV1, cidRedisKey) && !StringUtils.equals(redisCreativeStatisticsPrefix, cidRedisKey)) {
+            cidRedisKey = redisCreativeStatisticsPrefix;
+        }
+        if (!StringUtils.equals(redisCidVidActionKeyV1, vidCidRedisKey) && !StringUtils.equals(redisVideoCreativeStatisticsPrefix, vidCidRedisKey)) {
+            vidCidRedisKey = redisVideoCreativeStatisticsPrefix;
+        }
+
+        Map<Long, CreativeStatistic> paramCidRedisCache = this.batchFindCreativeRedisCache(cidRedisKey, adIdList);
+        Map<Long, CreativeStatistic> paramVidCidRedisCache = this.batchFindCreativeRedisCache(vidCidRedisKey, adIdList);
+
+        List<AdRankItem> result = new ArrayList<>(adIdList.size());
+        for (AdPlatformCreativeDTO dto : adIdList) {
+            Map<String, Object> ext = new HashMap<>();
+            double score = 0.0;
+            double cpa = 0.0;
+            Long cid = dto.getCreativeId();
+            try {
+                String pairOrSingle = "pair";
+                CreativeStatistic vidCidStatistic = paramVidCidRedisCache.getOrDefault(cid, new CreativeStatistic());
+                CreativeStatistic cidStatistic = paramCidRedisCache.getOrDefault(cid, new CreativeStatistic());
+
+                cpa = cidStatistic.parseCpaToDouble();
+
+                double order = vidCidStatistic.parseOrderToDouble();
+                double exp = vidCidStatistic.parseExpToDouble();
+
+                if (exp < viewThreshold) {
+                    pairOrSingle = "single";
+                    order = cidStatistic.parseOrderToDouble();
+                    exp = cidStatistic.parseExpToDouble();
+                }
+
+                if (exp != 0) {
+                    score = (order / exp) * cpa;
+                }
+
+                ext.put("order", order);
+                ext.put("exp", exp);
+                ext.put("viewThreshold", viewThreshold);
+                ext.put("pairOrSingle", pairOrSingle);
+                ext.put("cidStatistic", JSON.toJSONString(cidStatistic));
+                ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistic));
+                ext.put("cpa", cpa);
+                ext.put("abCode", 672);
+            } catch (Exception e) {
+                log.error("svc=672exp, error: ", e);
+            }
+
+            AdRankItem item = new AdRankItem();
+            item.setCpa(cpa);
+            item.setAdId(dto.getCreativeId());
+            item.setScore(score);
+            item.setExt(ext);
+            item.setVideoId(param.getVideoId());
+            item.setScore_type(672);
+            item.setWeight(dto.getWeight());
+            item.setCreativeCode(dto.getCreativeCode());
+            result.add(item);
+
+        }
+        result.sort(equalsRandomComparator());
+        return result;
+    }
+
     class CreativeStatistic {
 
         private String exp;
@@ -665,4 +744,9 @@ public class VideoAdThompsonScorerV2 {
     public void setExp670Param(String str) {
         this.exp670Param = gson.fromJson(str, Map.class);
     }
+
+    @Value("${ad.engine.new.thompson.exp.V2.672:{}}")
+    public void setExp672Param(String str) {
+        this.exp672Param = gson.fromJson(str, Map.class);
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -285,6 +285,11 @@ public class RankServiceImpl implements RankService {
                         request.getAppType().toString(), request.getNewExpGroup(), "670", modelParam))) {
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp670(param, request.getAdIdList());
             abCode = "670";
+        }else if (expCodes.contains("672") ||
+                (expCodes.contains(NewExpInfoHelper.flagId) && NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
+                        request.getAppType().toString(), request.getNewExpGroup(), "672", modelParam))){
+            rankResult = videoAdThompsonScorerV2.thompsonScorerByExp672(param, request.getAdIdList());
+            abCode = "672";
         }
 
         // 日志上报

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/RecommendRequestParam.java

@@ -22,4 +22,6 @@ public class RecommendRequestParam {
     String pqtId;
     List<Map> adAbExpArr ;
     String adAbGroup;
+
+    private StatisticsLogParam statisticsLog;
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/param/StatisticsLogParam.java

@@ -0,0 +1,19 @@
+package com.tzld.piaoquan.ad.engine.service.score.param;
+
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * 广告分发过程中的统计日志，ad-engine.statistics-log库用
+ */
+@Data
+public class StatisticsLogParam {
+
+    private List<Long> earlyAdIds;
+
+    private List<Long> earlyCreativeIds;
+
+    private List<Long> finalCreativeIds;
+
+}