Merge branch 'feature_20240528_zhaohaipeng_video_ad_thompson_v2' of algorithm/ad-engine into master

.gitignore

@@ -31,3 +31,6 @@ build/
 
 ### VS Code ###
 .vscode/
+apollo-cache-dir
+sentinel
+weblog

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VideoAdThompsonScorerV2.java

@@ -1,10 +1,12 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
+import com.alibaba.fastjson.JSON;
 import com.google.gson.Gson;
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.math3.distribution.BetaDistribution;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -13,6 +15,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 @Component
 public class VideoAdThompsonScorerV2 {
@@ -25,10 +28,16 @@ public class VideoAdThompsonScorerV2 {
     //redis:vid_cid_action:{$vid}_{$cid}
     private String redisVideoCreativeStatisticsPrefix="redis:vid_cid_action:";
 
+    private String rediCidActionKeyV1 = "redis:cid_action_v1:";
+    private String rediCidVidActionKeyV1 = "redis:vid_cid_action_v1:";
+
     private Map<String,Double> exp663Param=new HashMap<>();
     private Map<String,Double> exp664Param=new HashMap<>();
     private Map<String,Double> exp665Param=new HashMap<>();
     private Map<String,Double> exp666Param=new HashMap<>();
+    private Map<String,Double> exp669Param=new HashMap<>();
+    private Map<String,Double> exp670Param=new HashMap<>();
+
     Random random=new Random();
     Gson gson=new Gson();
     public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList){
@@ -231,6 +240,122 @@ public class VideoAdThompsonScorerV2 {
         return result;
     }
 
+    public List<AdRankItem> thompsonScorerByExp669(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidActionKeyV1, adIdList);
+        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
+        Double creativeExpSum = this.sumCreativeStatisticExp(creativeStatisticsMap.values());
+        Double videoCreativeExpSum = this.sumCreativeStatisticExp(videoCreativeStatisticsMap.values());
+
+        List<AdRankItem> result = new ArrayList<>(adIdList.size());
+        this.calcScore(result, adIdList,1d, creativeExpSum, videoCreativeExpSum, creativeStatisticsMap, videoCreativeStatisticsMap, exp669Param);
+        result.sort(equalsRandomComparator());
+
+        for (AdRankItem adRankItem : result) {
+            adRankItem.setVideoId(param.getVideoId());
+            adRankItem.setScore_type(699);
+        }
+
+        return result;
+    }
+
+    public List<AdRankItem> thompsonScorerByExp670(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidActionKeyV1, adIdList);
+        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
+
+        List<AdRankItem> result = new ArrayList<>(adIdList.size());
+        this.calcScore(result, adIdList, 0.0, 0.0, 0.0, creativeStatisticsMap, videoCreativeStatisticsMap, exp670Param);
+        result.sort(equalsRandomComparator());
+
+        for (AdRankItem adRankItem : result) {
+            adRankItem.setVideoId(param.getVideoId());
+            adRankItem.setScore_type(670);
+        }
+
+        return result;
+    }
+
+    private void calcScore(List<AdRankItem> result, List<AdPlatformCreativeDTO> adIdList,Double alpha, Double cidBeta, Double vidCidBeta, Map<Long, CreativeStatistic> cidMap, Map<Long, CreativeStatistic> vidCidMap, Map<String, Double> expParam) {
+        for (AdPlatformCreativeDTO dto : adIdList) {
+
+            CreativeStatistic cidStatistic = cidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
+            CreativeStatistic vidCidStatistic = vidCidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
+
+            log.info("cid: {} statistics: {}", dto.getCreativeId(), JSON.toJSONString(cidStatistic));
+            log.info("cid: {} vidCidStatistic: {}", dto.getCreativeId(), JSON.toJSONString(vidCidStatistic));
+
+            double cidScore = this.calcThompsonScore(expParam, cidStatistic, alpha, cidBeta);
+            double vidCidScore = this.calcThompsonScore(expParam, vidCidStatistic, alpha, vidCidBeta);
+            double w1 = expParam.getOrDefault("w1", 1d);
+            double w2 = expParam.getOrDefault("w2", 1d);
+            double score = ((w1 * vidCidScore + w2 * cidScore) / (w1 + w2)) * dto.getCpa();
+
+            log.info("calcScore w1: {}, w1: {}, cidScore: {}, vidCidScore: {}", w1, w2, cidScore, vidCidScore);
+
+            AdRankItem item = new AdRankItem();
+            item.setCpa(dto.getCpa());
+            item.setAdId(dto.getCreativeId());
+            item.setScore(score);
+
+            result.add(item);
+        }
+    }
+
+
+    private Map<Long, CreativeStatistic> batchFindCreativeRedisCache(String keyPrefix, List<AdPlatformCreativeDTO> adIdList) {
+        Map<Long, CreativeStatistic> resultMap = new HashMap<>();
+        List<String> redisKeys = adIdList.stream()
+                .map(dto -> keyPrefix + dto.getCreativeId())
+                .collect(Collectors.toList());
+
+        List<String> values = redisHelper.getValues(redisKeys);
+        for (int i = 0; i < adIdList.size(); i++) {
+            Long cid = adIdList.get(i).getCreativeId();
+            String value = values.get(i);
+            if (StringUtils.isNotBlank(value)) {
+                resultMap.put(cid, gson.fromJson(value, CreativeStatistic.class));
+            }
+        }
+
+        return resultMap;
+    }
+
+    private double calcThompsonScore(Map<String, Double> expParam, CreativeStatistic creativeStatistic, Double defaultAlpha, Double defaultBeta) {
+        Double alpha = expParam.getOrDefault("alpha", defaultAlpha);
+        Double beta = expParam.getOrDefault("beta", defaultBeta);
+
+        double order = creativeStatistic.parseOrderToDouble() + alpha;
+        double exp = creativeStatistic.parseExpToDouble() + beta;
+
+        if (order == 0 || exp == 0) {
+            return 0.0;
+        }
+        log.info("calcThompsonScore.order: {}, exp: {}", order, exp);
+        return this.betaSampler(order, exp);
+    }
+
+    private Double sumCreativeStatisticExp(Collection<CreativeStatistic> creativeStatistics) {
+        return creativeStatistics.stream()
+                .map(CreativeStatistic::getExp)
+                .filter(StringUtils::isNotBlank)
+                .collect(Collectors.summarizingDouble(Double::parseDouble))
+                .getSum();
+    }
+
+    private Comparator<AdRankItem> equalsRandomComparator() {
+        return new Comparator<AdRankItem>() {
+            @Override
+            public int compare(AdRankItem o1, AdRankItem o2) {
+                int comparison = o1.compareTo(o2);
+                if (comparison != 0) {
+                    return comparison;
+                }
+                // 数字相等时随机排列
+                return random.nextInt(3) - 1; // 产生-1、0或1的随机值
+            }
+        };
+    }
+
+
     private Map<String, Object> extMap(CreativeStatistic statistic, String abCode, Double cpa, Double viewThreshold, Double alpha, Double beta, Double beta_k) {
         Map<String, Object> map = new HashMap<>();
         map.put("abCode", abCode);
@@ -250,6 +375,12 @@ public class VideoAdThompsonScorerV2 {
     }
 
     class CreativeStatistic{
+
+        private String exp;
+        private String click;
+        private String order;
+        private String cpa;
+
         public CreativeStatistic() {
         }
 
@@ -257,11 +388,6 @@ public class VideoAdThompsonScorerV2 {
             this.cpa = cpa;
         }
 
-        private String exp;
-        private String click;
-        private String order;
-        private String cpa;
-
         public String getExp() {
             if (exp == null || "".equals(exp)) {
                 return "0";
@@ -269,6 +395,13 @@ public class VideoAdThompsonScorerV2 {
             return exp;
         }
 
+        public Double parseExpToDouble() {
+            if (StringUtils.isBlank(exp)) {
+                return 0.0;
+            }
+            return Double.parseDouble(exp);
+        }
+
         public void setExp(String exp) {
             this.exp = exp;
         }
@@ -280,6 +413,13 @@ public class VideoAdThompsonScorerV2 {
             return click;
         }
 
+        public Double parseClickToDouble(){
+            if (StringUtils.isBlank(click)){
+                return 0.0;
+            }
+            return Double.parseDouble(click);
+        }
+
         public void setClick(String click) {
             this.click = click;
         }
@@ -291,6 +431,13 @@ public class VideoAdThompsonScorerV2 {
             return order;
         }
 
+        public Double parseOrderToDouble(){
+            if (StringUtils.isBlank(order)){
+                return 0.0;
+            }
+            return Double.parseDouble(order);
+        }
+
         public void setOrder(String order) {
             this.order = order;
         }
@@ -301,6 +448,13 @@ public class VideoAdThompsonScorerV2 {
             }
             return cpa;
         }
+
+        public Double parseCpaToDouble() {
+            if (StringUtils.isBlank(cpa)) {
+                return 0.0;
+            }
+            return Double.parseDouble(cpa);
+        }
     }
     double betaSampler(double alpha, double beta) {
         BetaDistribution betaSample = new BetaDistribution(alpha, beta);
@@ -322,4 +476,13 @@ public class VideoAdThompsonScorerV2 {
     public void setExp666Param(String str){
         this.exp663Param=gson.fromJson(str,Map.class);
     }
+
+    @Value("${ad.engine.new.thompson.exp.V2.669:{}}")
+    public void setExp669Param(String str){
+        this.exp669Param=gson.fromJson(str,Map.class);
+    }
+    @Value("${ad.engine.new.thompson.exp.V2.670:{}}")
+    public void setExp670Param(String str){
+        this.exp670Param=gson.fromJson(str,Map.class);
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/RequestConvert.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score.convert;
 import com.tzld.piaoquan.ad.engine.service.score.param.RecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
@@ -25,6 +26,7 @@ public class RequestConvert {
         scoreParam.setVideoId(request.getVideoId());
         scoreParam.setNewExpGroup(request.getNewExpGroup());
         scoreParam.setPqtId(request.getPqtId());
+        scoreParam.setMid(request.getMid());
         return scoreParam;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -261,6 +261,14 @@ public class RankServiceImpl implements RankService {
                 (expCodes.contains(NewExpInfoHelper.flagId)&&NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
                         request.getAppType().toString(),request.getNewExpGroup(),"666",modelParam))){
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp666(param, request.getAdIdList());
+        } else if (expCodes.contains("669") ||
+                (expCodes.contains(NewExpInfoHelper.flagId) && NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
+                        request.getAppType().toString(), request.getNewExpGroup(), "669", modelParam))) {
+            rankResult = videoAdThompsonScorerV2.thompsonScorerByExp669(param, request.getAdIdList());
+        } else if (expCodes.contains("670") ||
+                (expCodes.contains(NewExpInfoHelper.flagId) && NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
+                        request.getAppType().toString(), request.getNewExpGroup(), "670", modelParam))) {
+            rankResult = videoAdThompsonScorerV2.thompsonScorerByExp670(param, request.getAdIdList());
         }
 
         // 日志上报