Merge branch 'feature_20240531_zhaohaipeng_log' into pre-master

.gitignore

@@ -31,3 +31,6 @@ build/
 
 ### VS Code ###
 .vscode/
+apollo-cache-dir
+sentinel
+weblog

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/ModelManager.java

@@ -202,7 +202,7 @@ public class ModelManager {
             }
             ossObj.close();
         } catch (Exception e) {
-            log.error("update model fail", e);
+            log.error("update model fail: {}", loadTask.path, e);
         } finally {
             loadTask.isLoading = false;
             if (ossObj != null) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VideoAdThompsonScorerV2.java

@@ -1,10 +1,13 @@
 package com.tzld.piaoquan.ad.engine.service.score;
 
+import com.alibaba.fastjson.JSON;
 import com.google.gson.Gson;
 import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.math3.distribution.BetaDistribution;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -13,6 +16,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 @Component
 public class VideoAdThompsonScorerV2 {
@@ -25,10 +29,16 @@ public class VideoAdThompsonScorerV2 {
     //redis:vid_cid_action:{$vid}_{$cid}
     private String redisVideoCreativeStatisticsPrefix="redis:vid_cid_action:";
 
+    private String rediCidActionKeyV1 = "redis:cid_action_v1:";
+    private String rediCidVidActionKeyV1 = "redis:vid_cid_action_v1:";
+
     private Map<String,Double> exp663Param=new HashMap<>();
     private Map<String,Double> exp664Param=new HashMap<>();
     private Map<String,Double> exp665Param=new HashMap<>();
     private Map<String,Double> exp666Param=new HashMap<>();
+    private Map<String,Double> exp669Param=new HashMap<>();
+    private Map<String,Double> exp670Param=new HashMap<>();
+
     Random random=new Random();
     Gson gson=new Gson();
     public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList){
@@ -58,8 +68,11 @@ public class VideoAdThompsonScorerV2 {
                     score = (exp663Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()))
                             / (exp663Param.getOrDefault("beta",10000d)+Long.parseLong(statistic.getExp())) ;
                 }
+                double s1 = score;
                 score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
+
                 Map<String, Object> ext = this.extMap(statistic, "663", dto.getCpa(), null, exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
+                ext.put("s1", s1);
                 item.setExt(ext);
                 item.setScore(score);
                 item.setScore_type(663);
@@ -100,10 +113,12 @@ public class VideoAdThompsonScorerV2 {
                     score = (exp664Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()))
                             / (exp664Param.getOrDefault("beta",10000d)+Long.parseLong(statistic.getExp())) ;
                 }
+                double s1 = score;
                 score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
                 item.setScore(score);
 
                 Map<String, Object> ext = this.extMap(statistic, "664", dto.getCpa(), null, exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
+                ext.put("s1", s1);
                 item.setExt(ext);
                 item.setScore_type(664);
             }catch (Exception e){
@@ -142,11 +157,13 @@ public class VideoAdThompsonScorerV2 {
                     score = betaSampler(exp665Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
                             (exp665Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp())))/ (1+exp665Param.getOrDefault("beta_k",9d));
                 }
+                double s1 = score;
                 score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
                 item.setScore(score);
                 item.setScore_type(665);
 
                 Map<String, Object> ext = this.extMap(statistic, "665", dto.getCpa(), null, exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
+                ext.put("s1", s1);
                 item.setExt(ext);
             }catch (Exception e){
                 log.error("svc=thompsonScorerByExp665 {}",gson.toJson(e.getStackTrace()));
@@ -204,12 +221,14 @@ public class VideoAdThompsonScorerV2 {
                         score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
                     }
                 }
+                double s1 = score;
                 score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
                 item.setScore(score);
                 item.setScore_type(666);
 
-                Map<String, Object> ext = this.extMap(statistic, "665", dto.getCpa(), exp666Param.getOrDefault("viewThreshold",5000d), exp663Param.getOrDefault("alpha", 1d),
+                Map<String, Object> ext = this.extMap(statistic, "666", dto.getCpa(), exp666Param.getOrDefault("viewThreshold",5000d), exp663Param.getOrDefault("alpha", 1d),
                         exp663Param.getOrDefault("beta", 10000d), exp666Param.getOrDefault("beta_k",9d));
+                ext.put("s1", s1);
                 item.setExt(ext);
 
             }catch (Exception e){
@@ -222,6 +241,142 @@ public class VideoAdThompsonScorerV2 {
         return result;
     }
 
+    public List<AdRankItem> thompsonScorerByExp669(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidActionKeyV1, adIdList);
+        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
+        Double creativeExpSum = this.sumCreativeStatisticExp(creativeStatisticsMap.values());
+        Double videoCreativeExpSum = this.sumCreativeStatisticExp(videoCreativeStatisticsMap.values());
+
+        List<AdRankItem> result = new ArrayList<>(adIdList.size());
+        this.calcScore(result, adIdList,1d, creativeExpSum, videoCreativeExpSum, creativeStatisticsMap, videoCreativeStatisticsMap, exp669Param);
+        result.sort(equalsRandomComparator());
+
+        for (AdRankItem adRankItem : result) {
+            adRankItem.setVideoId(param.getVideoId());
+            adRankItem.setScore_type(699);
+            if (MapUtils.isNotEmpty(adRankItem.getExt())) {
+                adRankItem.getExt().put("abCode", 699);
+            }
+        }
+
+        return result;
+    }
+
+    public List<AdRankItem> thompsonScorerByExp670(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidActionKeyV1, adIdList);
+        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
+
+        List<AdRankItem> result = new ArrayList<>(adIdList.size());
+        this.calcScore(result, adIdList, 0.0, 0.0, 0.0, creativeStatisticsMap, videoCreativeStatisticsMap, exp670Param);
+        result.sort(equalsRandomComparator());
+
+        for (AdRankItem adRankItem : result) {
+            adRankItem.setVideoId(param.getVideoId());
+            adRankItem.setScore_type(670);
+            if (MapUtils.isNotEmpty(adRankItem.getExt())){
+                adRankItem.getExt().put("abCode", 670);
+            }
+        }
+
+        return result;
+    }
+
+    private void calcScore(List<AdRankItem> result, List<AdPlatformCreativeDTO> adIdList,Double alpha, Double cidBeta, Double vidCidBeta, Map<Long, CreativeStatistic> cidMap, Map<Long, CreativeStatistic> vidCidMap, Map<String, Double> expParam) {
+        for (AdPlatformCreativeDTO dto : adIdList) {
+
+            CreativeStatistic cidStatistic = cidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
+            CreativeStatistic vidCidStatistic = vidCidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
+
+            double cidScore = this.calcThompsonScore(expParam, cidStatistic, alpha, cidBeta);
+            double vidCidScore = this.calcThompsonScore(expParam, vidCidStatistic, alpha, vidCidBeta);
+            double w1 = expParam.getOrDefault("w1", 1d);
+            double w2 = expParam.getOrDefault("w2", 1d);
+            double score = ((w1 * vidCidScore + w2 * cidScore) / (w1 + w2)) * dto.getCpa();
+
+
+            // 日志上报扩展参数
+            Map<String, Object> ext = new HashMap<>();
+            ext.put("w1", w1);
+            ext.put("w2", w2);
+            ext.put("cidScore", cidScore);
+            ext.put("vidCidScore", vidCidScore);
+            ext.put("cidA", alpha);
+            ext.put("vidCidA", alpha);
+            ext.put("cidB", cidBeta);
+            ext.put("vidCidB", vidCidBeta);
+            ext.put("cidStatistic", JSON.toJSONString(cidStatistic));
+            ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistic));
+            ext.put("cpa", dto.getCpa());
+
+            log.info("svc=699And670Score, score: {}, param: {}", score, JSON.toJSONString(expParam));
+
+
+            AdRankItem item = new AdRankItem();
+            item.setCpa(dto.getCpa());
+            item.setAdId(dto.getCreativeId());
+            item.setScore(score);
+            item.setExt(ext);
+
+            result.add(item);
+        }
+    }
+
+
+    private Map<Long, CreativeStatistic> batchFindCreativeRedisCache(String keyPrefix, List<AdPlatformCreativeDTO> adIdList) {
+        Map<Long, CreativeStatistic> resultMap = new HashMap<>();
+        List<String> redisKeys = adIdList.stream()
+                .map(dto -> keyPrefix + dto.getCreativeId())
+                .collect(Collectors.toList());
+
+        List<String> values = redisHelper.getValues(redisKeys);
+        for (int i = 0; i < adIdList.size(); i++) {
+            Long cid = adIdList.get(i).getCreativeId();
+            String value = values.get(i);
+            if (StringUtils.isNotBlank(value)) {
+                resultMap.put(cid, gson.fromJson(value, CreativeStatistic.class));
+            }
+        }
+
+        return resultMap;
+    }
+
+    private double calcThompsonScore(Map<String, Double> expParam, CreativeStatistic creativeStatistic, Double defaultAlpha, Double defaultBeta) {
+        Double alpha = expParam.getOrDefault("alpha", defaultAlpha);
+        Double beta = expParam.getOrDefault("beta", defaultBeta);
+
+        double order = creativeStatistic.parseOrderToDouble() + alpha;
+        double exp = creativeStatistic.parseExpToDouble() + beta;
+
+        if (order == 0 || exp == 0) {
+            return 0.0;
+        }
+        log.info("calcThompsonScore.order: {}, exp: {}", order, exp);
+        return this.betaSampler(order, exp);
+    }
+
+    private Double sumCreativeStatisticExp(Collection<CreativeStatistic> creativeStatistics) {
+        return creativeStatistics.stream()
+                .map(CreativeStatistic::getExp)
+                .filter(StringUtils::isNotBlank)
+                .collect(Collectors.summarizingDouble(Double::parseDouble))
+                .getSum();
+    }
+
+    private Comparator<AdRankItem> equalsRandomComparator() {
+        return new Comparator<AdRankItem>() {
+            @Override
+            public int compare(AdRankItem o1, AdRankItem o2) {
+                int comparison = o1.compareTo(o2);
+                if (comparison != 0) {
+                    return comparison;
+                }
+                // 数字相等时随机排列
+                return random.nextInt(3) - 1; // 产生-1、0或1的随机值
+            }
+        };
+    }
+
+
     private Map<String, Object> extMap(CreativeStatistic statistic, String abCode, Double cpa, Double viewThreshold, Double alpha, Double beta, Double beta_k) {
         Map<String, Object> map = new HashMap<>();
         map.put("abCode", abCode);
@@ -241,6 +396,12 @@ public class VideoAdThompsonScorerV2 {
     }
 
     class CreativeStatistic{
+
+        private String exp;
+        private String click;
+        private String order;
+        private String cpa;
+
         public CreativeStatistic() {
         }
 
@@ -248,11 +409,6 @@ public class VideoAdThompsonScorerV2 {
             this.cpa = cpa;
         }
 
-        private String exp;
-        private String click;
-        private String order;
-        private String cpa;
-
         public String getExp() {
             if (exp == null || "".equals(exp)) {
                 return "0";
@@ -260,6 +416,13 @@ public class VideoAdThompsonScorerV2 {
             return exp;
         }
 
+        public Double parseExpToDouble() {
+            if (StringUtils.isBlank(exp)) {
+                return 0.0;
+            }
+            return Double.parseDouble(exp);
+        }
+
         public void setExp(String exp) {
             this.exp = exp;
         }
@@ -271,6 +434,13 @@ public class VideoAdThompsonScorerV2 {
             return click;
         }
 
+        public Double parseClickToDouble(){
+            if (StringUtils.isBlank(click)){
+                return 0.0;
+            }
+            return Double.parseDouble(click);
+        }
+
         public void setClick(String click) {
             this.click = click;
         }
@@ -282,6 +452,13 @@ public class VideoAdThompsonScorerV2 {
             return order;
         }
 
+        public Double parseOrderToDouble(){
+            if (StringUtils.isBlank(order)){
+                return 0.0;
+            }
+            return Double.parseDouble(order);
+        }
+
         public void setOrder(String order) {
             this.order = order;
         }
@@ -292,6 +469,13 @@ public class VideoAdThompsonScorerV2 {
             }
             return cpa;
         }
+
+        public Double parseCpaToDouble() {
+            if (StringUtils.isBlank(cpa)) {
+                return 0.0;
+            }
+            return Double.parseDouble(cpa);
+        }
     }
     double betaSampler(double alpha, double beta) {
         BetaDistribution betaSample = new BetaDistribution(alpha, beta);
@@ -313,4 +497,13 @@ public class VideoAdThompsonScorerV2 {
     public void setExp666Param(String str){
         this.exp663Param=gson.fromJson(str,Map.class);
     }
+
+    @Value("${ad.engine.new.thompson.exp.V2.669:{}}")
+    public void setExp669Param(String str){
+        this.exp669Param=gson.fromJson(str,Map.class);
+    }
+    @Value("${ad.engine.new.thompson.exp.V2.670:{}}")
+    public void setExp670Param(String str){
+        this.exp670Param=gson.fromJson(str,Map.class);
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCvrLRScorer.java

@@ -45,6 +45,7 @@ public class VlogAdCvrLRScorer extends BaseLRModelScorer {
         }
 
         long startTime = System.currentTimeMillis();
+        LOGGER.info("VlogAdCvrLRScorer.scorerConfigInfo.getModelPath: {}: {}", scorerConfigInfo.getModelPath(), this.getModel());
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/RequestConvert.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score.convert;
 import com.tzld.piaoquan.ad.engine.service.score.param.RecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
@@ -25,6 +26,7 @@ public class RequestConvert {
         scoreParam.setVideoId(request.getVideoId());
         scoreParam.setNewExpGroup(request.getNewExpGroup());
         scoreParam.setPqtId(request.getPqtId());
+        scoreParam.setMid(request.getMid());
         return scoreParam;
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -177,6 +177,13 @@ public class RankServiceImpl implements RankService {
             }else {
                 log.info("svc=pid_log obj={}", JSONObject.toJSONString(object));
             }
+
+            for (AdRankItem adRankItem : rankResult) {
+                adRankItem.getExt().put("userAdFeature", JSON.toJSONString(userAdFeature));
+            }
+            // 日志上报
+            logHubService.scoreLogUpload(param, request.getAdIdList(), rankResult, request, inCvrAdjustingExp ? "cvr_adjusting" : "base_line");
+
             return rankResult.get(0);
         }else {
             //空返回值
@@ -262,6 +269,14 @@ public class RankServiceImpl implements RankService {
                 (expCodes.contains(NewExpInfoHelper.flagId)&&NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
                         request.getAppType().toString(),request.getNewExpGroup(),"666",modelParam))){
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp666(param, request.getAdIdList());
+        } else if (expCodes.contains("669") ||
+                (expCodes.contains(NewExpInfoHelper.flagId) && NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
+                        request.getAppType().toString(), request.getNewExpGroup(), "669", modelParam))) {
+            rankResult = videoAdThompsonScorerV2.thompsonScorerByExp669(param, request.getAdIdList());
+        } else if (expCodes.contains("670") ||
+                (expCodes.contains(NewExpInfoHelper.flagId) && NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
+                        request.getAppType().toString(), request.getNewExpGroup(), "670", modelParam))) {
+            rankResult = videoAdThompsonScorerV2.thompsonScorerByExp670(param, request.getAdIdList());
         }
         log.info("RankServiceImpl.adItemRankWithVideoAdThompson.adIdList: {}, result: {}", JSON.toJSONString(request.getAdIdList()), JSON.toJSONString(rankResult));