feat:修改过滤

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/ModelManager.java

@@ -202,7 +202,7 @@ public class ModelManager {
             }
             ossObj.close();
         } catch (Exception e) {
-            log.error("update model fail", e);
+            log.error("update model fail: {}", loadTask.path, e);
         } finally {
             loadTask.isLoading = false;
             if (ossObj != null) {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/LogHubService.java

@@ -9,5 +9,5 @@ import java.util.List;
 
 public interface LogHubService {
 
-    void scoreLogUpload(ScoreParam param, List<AdPlatformCreativeDTO> adIdList, List<AdRankItem> rankItems, RecommendRequestParam requestParam, String scoreStrategy);
+    void scoreLogUpload(ScoreParam param, List<AdPlatformCreativeDTO> adIdList, List<AdRankItem> rankItems, RecommendRequestParam requestParam, String scoreStrategy, String abCode);
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/log/impl/LogHubServiceImpl.java

@@ -14,10 +14,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -40,7 +37,7 @@ public class LogHubServiceImpl implements LogHubService {
     private AliyunLogManager aliyunLogManager;
 
     @Override
-    public void scoreLogUpload(ScoreParam param, List<AdPlatformCreativeDTO> adIdList, List<AdRankItem> rankItems, RecommendRequestParam requestParam, String scoreStrategy) {
+    public void scoreLogUpload(ScoreParam param, List<AdPlatformCreativeDTO> adIdList, List<AdRankItem> rankItems, RecommendRequestParam requestParam, String scoreStrategy, String abCode) {
         logUploadThreadPool.execute(new Runnable() {
             @Override
             public void run() {
@@ -48,6 +45,7 @@ public class LogHubServiceImpl implements LogHubService {
                 logMap.put("pqtId", param.getPqtId());
                 logMap.put("mid", param.getMid());
                 logMap.put("videoId", param.getVideoId());
+                logMap.put("abCode", abCode);
                 // 获取AB实验列表
                 Set<String> abExpCode = new HashSet<>();
                 if (CollectionUtils.isNotEmpty(requestParam.getAdAbExpArr())) {
@@ -57,11 +55,28 @@ public class LogHubServiceImpl implements LogHubService {
                         }
                     }
                 }
+                logMap.put("abExpCode", abExpCode);
 
-                logMap.put("scoreResult", JSON.toJSONString(rankItems));
-                logMap.put("creativeList", JSON.toJSONString(adIdList));
+                List<JSONObject> scoreResult = new ArrayList<>();
+                for (AdRankItem rankItem : rankItems) {
+                    JSONObject json = new JSONObject();
+                    json.put("adId", rankItem.getAdId());
+                    json.put("score", rankItem.getScore());
+                    json.put("ext", rankItem.getExt());
+                    json.put("weight", rankItem.getWeight());
+                    json.put("creativeCode", rankItem.getCreativeCode());
+                    scoreResult.add(json);
+                }
+                logMap.put("scoreResult", JSON.toJSONString(scoreResult));
 
-                logMap.put("abExpCode", abExpCode);
+                AdRankItem top1 = rankItems.get(0);
+                logMap.put("top1_adId", top1.getAdId());
+                logMap.put("top1_score", top1.getScore());
+                logMap.put("top1_ext", JSON.toJSONString(top1.getExt()));
+                logMap.put("top1_weight", top1.getWeight());
+                logMap.put("top1_creativeCode", top1.getCreativeCode());
+
+                logMap.put("creativeList", JSON.toJSONString(adIdList));
                 logMap.put("adAbGroup", requestParam.getAdAbGroup());
                 logMap.put("scoreStrategy", scoreStrategy);
                 logMap.put("appType", requestParam.getAppType());

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VideoAdThompsonScorerV2.java

@@ -6,6 +6,7 @@ import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.math3.distribution.BetaDistribution;
 import org.slf4j.Logger;
@@ -24,99 +25,171 @@ public class VideoAdThompsonScorerV2 {
     @Autowired
     private AlgorithmRedisHelper redisHelper;
     // redis:cid_action:{$cid}
-    private String redisCreativeStatisticsPrefix="redis:cid_action:";
-    //redis:vid_cid_action:{$vid}_{$cid}
-    private String redisVideoCreativeStatisticsPrefix="redis:vid_cid_action:";
+    private String redisCreativeStatisticsPrefix = "redis:cid_action:";
+    // redis:vid_cid_action:{$vid}_{$cid}
+    private String redisVideoCreativeStatisticsPrefix = "redis:vid_cid_action:";
 
     private String rediCidActionKeyV1 = "redis:cid_action_v1:";
     private String rediCidVidActionKeyV1 = "redis:vid_cid_action_v1:";
 
-    private Map<String,Double> exp663Param=new HashMap<>();
-    private Map<String,Double> exp664Param=new HashMap<>();
-    private Map<String,Double> exp665Param=new HashMap<>();
-    private Map<String,Double> exp666Param=new HashMap<>();
-    private Map<String,Double> exp669Param=new HashMap<>();
-    private Map<String,Double> exp670Param=new HashMap<>();
-
-    Random random=new Random();
-    Gson gson=new Gson();
-    public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList){
-        List<AdRankItem> result=new LinkedList<>();
-        String jsonStr;
-        CreativeStatistic statistic = null;
-        List<String> redisKey=new LinkedList<>();
-        adIdList.forEach(creativeDTO -> {
-            redisKey.add(redisCreativeStatisticsPrefix+creativeDTO.getCreativeId());
-        });
-        List<String> values=redisHelper.getValues(redisKey);
-        double score=0d;
-        int i=0;
-        for(AdPlatformCreativeDTO dto:adIdList){
-            AdRankItem item=new AdRankItem();
-            item.setVideoId(param.getVideoId());
-            item.setCpa(dto.getCpa());
-            item.setAdId(dto.getCreativeId());
+    private Map<String, Double> exp663Param = new HashMap<>();
+    private Map<String, Double> exp664Param = new HashMap<>();
+    private Map<String, Double> exp665Param = new HashMap<>();
+    private Map<String, Double> exp666Param = new HashMap<>();
+    private Map<String, Double> exp669Param = new HashMap<>();
+    private Map<String, Double> exp670Param = new HashMap<>();
+
+    Random random = new Random();
+    Gson gson = new Gson();
+
+    // public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+    //     List<AdRankItem> result = new LinkedList<>();
+    //     String jsonStr;
+    //     CreativeStatistic statistic = null;
+    //     List<String> redisKey = new LinkedList<>();
+    //     adIdList.forEach(creativeDTO -> {
+    //         redisKey.add(redisCreativeStatisticsPrefix + creativeDTO.getCreativeId());
+    //     });
+    //     List<String> values = redisHelper.getValues(redisKey);
+    //     double score = 0d;
+    //     int i = 0;
+    //     for (AdPlatformCreativeDTO dto : adIdList) {
+    //         AdRankItem item = new AdRankItem();
+    //         item.setVideoId(param.getVideoId());
+    //         item.setCpa(dto.getCpa());
+    //         item.setAdId(dto.getCreativeId());
+    //         try {
+    //             jsonStr = values.get(i);
+    //             if (jsonStr == null) {
+    //                 score = exp663Param.getOrDefault("randomMin", 0.000001)
+    //                         + (random.nextDouble() *
+    //                         (exp663Param.getOrDefault("randomMax", 0.00001) - exp663Param.getOrDefault("randomMin", 0.000001)));
+    //             } else {
+    //                 statistic = gson.fromJson(jsonStr, CreativeStatistic.class);
+    //                 score = (exp663Param.getOrDefault("alpha", 1d) + Long.parseLong(statistic.getOrder()))
+    //                         / (exp663Param.getOrDefault("beta", 10000d) + Long.parseLong(statistic.getExp()));
+    //             }
+    //             double s1 = score;
+    //             score = score * dto.getCpa() * dto.getBid1() * dto.getBid2();
+    //
+    //             Map<String, Object> ext = this.extMap(statistic, "663", dto.getCpa(), null,
+    //                     exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
+    //             ext.put("s1", s1);
+    //             item.setExt(ext);
+    //             item.setScore(score);
+    //             item.setScore_type(663);
+    //         } catch (Exception e) {
+    //             log.error("svc=thompsonScorerByExp663 {}", gson.toJson(e.getStackTrace()));
+    //         }
+    //         result.add(item);
+    //         i++;
+    //     }
+    //     Collections.sort(result);
+    //     return result;
+    // }
+
+    public List<AdRankItem> thompsonScorerByExp663(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        List<AdRankItem> result = new LinkedList<>();
+        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(redisCreativeStatisticsPrefix, adIdList);
+
+        double alpha = exp663Param.getOrDefault("alpha", 0d);
+        double beta = exp663Param.getOrDefault("beta", 0d);
+
+        for (AdPlatformCreativeDTO dto : adIdList) {
+
+            // 日志上报扩展参数
+            Map<String, Object> ext = new HashMap<>();
+            double score = 0.0;
             try {
-                jsonStr=values.get(i);
-                if(jsonStr==null){
-                    score = exp663Param.getOrDefault("randomMin",0.000001)
-                            + (random.nextDouble() *
-                            (exp663Param.getOrDefault("randomMax",0.00001 ) - exp663Param.getOrDefault("randomMin",0.000001 )));
-                }else {
-                    statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-                    score = (exp663Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()))
-                            / (exp663Param.getOrDefault("beta",10000d)+Long.parseLong(statistic.getExp())) ;
+                CreativeStatistic cidStatistics = creativeStatisticsMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
+
+                double order;
+                double exp;
+                order = cidStatistics.parseOrderToDouble();
+                exp = cidStatistics.parseExpToDouble();
+
+                double s1 = 0.0;
+                double a = order + alpha;
+                double b = exp + beta;
+                if (a != 0 && b != 0) {
+                    s1 = a / b;
                 }
-                score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
-                Map<String, Object> ext = this.extMap(statistic, "663", dto.getCpa(), null, exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
-                item.setExt(ext);
-                item.setScore(score);
-                item.setScore_type(663);
-            }catch (Exception e){
-                log.error("svc=thompsonScorerByExp663 {}",gson.toJson(e.getStackTrace()));
+
+                score = s1 * dto.getCpa() * dto.getBid1() * dto.getBid2();
+
+                ext.put("alpha", alpha);
+                ext.put("beta", beta);
+                ext.put("order", order);
+                ext.put("exp", exp);
+                ext.put("s1", s1);
+                ext.put("bid1", dto.getBid1());
+                ext.put("bid2", dto.getBid2());
+                ext.put("pairOrSingle", "single");
+                ext.put("cidStatistic", JSON.toJSONString(cidStatistics));
+                ext.put("cpa", dto.getCpa());
+                ext.put("abCode", 663);
+            } catch (Exception e) {
+                log.error("svc=663exp, error: ", e);
             }
+            AdRankItem item = new AdRankItem();
+            item.setCpa(dto.getCpa());
+            item.setAdId(dto.getCreativeId());
+            item.setScore(score);
+            item.setExt(ext);
+            item.setVideoId(param.getVideoId());
+            item.setScore_type(663);
+            item.setBid1(dto.getBid1());
+            item.setBid2(dto.getBid2());
+            item.setWeight(dto.getWeight());
+            item.setCreativeCode(dto.getCreativeCode());
             result.add(item);
-            i++;
         }
-        Collections.sort(result);
+
+        result.sort(equalsRandomComparator());
         return result;
     }
-    public List<AdRankItem> thompsonScorerByExp664(ScoreParam param,List<AdPlatformCreativeDTO> adIdList){
-        List<AdRankItem> result=new LinkedList<>();
+
+    public List<AdRankItem> thompsonScorerByExp664(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        List<AdRankItem> result = new LinkedList<>();
         String jsonStr;
-        CreativeStatistic statistic=null;
-        List<String> redisKey=new LinkedList<>();
+        CreativeStatistic statistic = null;
+        List<String> redisKey = new LinkedList<>();
         adIdList.forEach(creativeDTO -> {
-            redisKey.add(redisVideoCreativeStatisticsPrefix+param.getVideoId()+"_"+creativeDTO.getCreativeId());
+            redisKey.add(redisVideoCreativeStatisticsPrefix + param.getVideoId() + "_" + creativeDTO.getCreativeId());
         });
-        List<String> values=redisHelper.getValues(redisKey);
-        int i=0;
-        double score=0d;
-        for(AdPlatformCreativeDTO dto:adIdList){
-            AdRankItem item=new AdRankItem();
+        List<String> values = redisHelper.getValues(redisKey);
+        int i = 0;
+        double score = 0d;
+        for (AdPlatformCreativeDTO dto : adIdList) {
+            AdRankItem item = new AdRankItem();
             item.setVideoId(param.getVideoId());
             item.setCpa(dto.getCpa());
             item.setAdId(dto.getCreativeId());
             try {
 
-                jsonStr=values.get(i);
-                if(jsonStr==null){
-                    score = exp664Param.getOrDefault("randomMin",0.000001)
+                jsonStr = values.get(i);
+                if (jsonStr == null) {
+                    score = exp664Param.getOrDefault("randomMin", 0.000001)
                             + (random.nextDouble() *
-                            (exp664Param.getOrDefault("randomMax",0.00001 ) - exp664Param.getOrDefault("randomMin",0.000001 )));
-                }else {
-                    statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-                    score = (exp664Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()))
-                            / (exp664Param.getOrDefault("beta",10000d)+Long.parseLong(statistic.getExp())) ;
+                            (exp664Param.getOrDefault("randomMax", 0.00001) - exp664Param.getOrDefault("randomMin", 0.000001)));
+                } else {
+                    statistic = gson.fromJson(jsonStr, CreativeStatistic.class);
+                    score = (exp664Param.getOrDefault("alpha", 1d) + Long.parseLong(statistic.getOrder()))
+                            / (exp664Param.getOrDefault("beta", 10000d) + Long.parseLong(statistic.getExp()));
                 }
-                score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
+                double s1 = score;
+                score = score * dto.getCpa() * dto.getBid1() * dto.getBid2();
                 item.setScore(score);
-                item.setScore_type(664);
 
-                Map<String, Object> ext = this.extMap(statistic, "664", dto.getCpa(), null, exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
+                Map<String, Object> ext = this.extMap(statistic, "664", dto.getCpa(), null,
+                        exp664Param.getOrDefault("alpha", 1d), exp664Param.getOrDefault("beta", 10000d), null);
+                ext.put("s1", s1);
+                item.setCreativeCode(dto.getCreativeCode());
+                item.setWeight(dto.getWeight());
                 item.setExt(ext);
-            }catch (Exception e){
-                log.error("svc=thompsonScorerByExp664 {}",gson.toJson(e.getStackTrace()));
+                item.setScore_type(664);
+            } catch (Exception e) {
+                log.error("svc=thompsonScorerByExp664 {}", gson.toJson(e.getStackTrace()));
             }
             result.add(item);
             i++;
@@ -125,40 +198,46 @@ public class VideoAdThompsonScorerV2 {
         Collections.sort(result);
         return result;
     }
-    public List<AdRankItem> thompsonScorerByExp665(ScoreParam param,List<AdPlatformCreativeDTO> adIdList){
-        List<AdRankItem> result=new LinkedList<>();
+
+    public List<AdRankItem> thompsonScorerByExp665(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        List<AdRankItem> result = new LinkedList<>();
         String jsonStr;
         CreativeStatistic statistic = null;
-        List<String> redisKey=new LinkedList<>();
+        List<String> redisKey = new LinkedList<>();
         adIdList.forEach(creativeDTO -> {
-            redisKey.add(redisCreativeStatisticsPrefix+creativeDTO.getCreativeId());
+            redisKey.add(redisCreativeStatisticsPrefix + creativeDTO.getCreativeId());
         });
-        //view/sum（view）
-        List<String> values=redisHelper.getValues(redisKey);
-        int i=0;
-        double score=0d;
-        for(AdPlatformCreativeDTO dto:adIdList){
-            AdRankItem item=new AdRankItem();
+        // view/sum（view）
+        List<String> values = redisHelper.getValues(redisKey);
+        int i = 0;
+        double score = 0d;
+        for (AdPlatformCreativeDTO dto : adIdList) {
+            AdRankItem item = new AdRankItem();
             item.setVideoId(param.getVideoId());
             item.setCpa(dto.getCpa());
             item.setAdId(dto.getCreativeId());
             try {
-                jsonStr=values.get(i);
-                if(jsonStr==null){
-                    score = betaSampler(exp665Param.getOrDefault("alpha",1d),exp665Param.getOrDefault("beta",100000d));
-                }else {
-                    statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-                    score = betaSampler(exp665Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-                            (exp665Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp())))/ (1+exp665Param.getOrDefault("beta_k",9d));
+                jsonStr = values.get(i);
+                if (jsonStr == null) {
+                    score = betaSampler(exp665Param.getOrDefault("alpha", 1d), exp665Param.getOrDefault("beta", 100000d));
+                } else {
+                    statistic = gson.fromJson(jsonStr, CreativeStatistic.class);
+                    score = betaSampler(exp665Param.getOrDefault("alpha", 1d) + Long.parseLong(statistic.getOrder()),
+                            (exp665Param.getOrDefault("beta", 100000d) + Long.parseLong(statistic.getExp()))) / (1 + exp665Param.getOrDefault("beta_k", 9d));
                 }
-                score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
+                double s1 = score;
+                score = score * dto.getCpa() * dto.getBid1() * dto.getBid2();
                 item.setScore(score);
                 item.setScore_type(665);
+                item.setWeight(dto.getWeight());
 
-                Map<String, Object> ext = this.extMap(statistic, "665", dto.getCpa(), null, exp663Param.getOrDefault("alpha", 1d), exp663Param.getOrDefault("beta", 10000d), null);
+                Map<String, Object> ext = this.extMap(statistic, "665", dto.getCpa(), null,
+                        exp665Param.getOrDefault("alpha", 1d), exp665Param.getOrDefault("beta", 10000d), null);
+                ext.put("s1", s1);
+                item.setCreativeCode(dto.getCreativeCode());
                 item.setExt(ext);
-            }catch (Exception e){
-                log.error("svc=thompsonScorerByExp665 {}",gson.toJson(e.getStackTrace()));
+            } catch (Exception e) {
+                log.error("svc=thompsonScorerByExp665 {}", gson.toJson(e.getStackTrace()));
             }
             result.add(item);
             i++;
@@ -167,89 +246,148 @@ public class VideoAdThompsonScorerV2 {
         Collections.sort(result);
         return result;
     }
-    public List<AdRankItem> thompsonScorerByExp666(ScoreParam param,List<AdPlatformCreativeDTO> adIdList){
-        List<AdRankItem> result=new LinkedList<>();
-        String jsonStr;
-        CreativeStatistic statistic = null;
+    // public List<AdRankItem> thompsonScorerByExp666(ScoreParam param,List<AdPlatformCreativeDTO> adIdList){
+    //     List<AdRankItem> result=new LinkedList<>();
+    //     String jsonStr;
+    //     CreativeStatistic statistic = null;
+    //
+    //     List<String> redisKey=new LinkedList<>();
+    //     adIdList.forEach(creativeDTO -> {
+    //         redisKey.add(redisCreativeStatisticsPrefix+creativeDTO.getCreativeId());
+    //     });
+    //     List<String> values=redisHelper.getValues(redisKey);
+    //     List<String> combineKeys=new LinkedList<>();
+    //     adIdList.forEach(creativeDTO -> {
+    //         combineKeys.add(redisVideoCreativeStatisticsPrefix+param.getVideoId()+"_"+creativeDTO.getCreativeId());
+    //     });
+    //     List<String> combineValues=redisHelper.getValues(combineKeys);
+    //     int i=0;
+    //     double score=0d;
+    //     for(AdPlatformCreativeDTO dto:adIdList){
+    //         String statisticType = "vid+cid";
+    //         AdRankItem item=new AdRankItem();
+    //         item.setVideoId(param.getVideoId());
+    //         item.setCpa(dto.getCpa());
+    //         item.setAdId(dto.getCreativeId());
+    //         try {
+    //             jsonStr=combineValues.get(i);
+    //             if(jsonStr==null){
+    //                 jsonStr=values.get(i);
+    //                 if(jsonStr==null){
+    //                     statisticType = "";
+    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
+    //                 }else {
+    //                     statisticType = "cid";
+    //                     statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
+    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
+    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
+    //                 }
+    //             }else {
+    //                 statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
+    //                 if(Double.parseDouble(statistic.getExp())>exp666Param.getOrDefault("viewThreshold",5000d)){
+    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
+    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
+    //                 } else if( values.get(i)!=null) {
+    //                     statistic =gson.fromJson(values.get(i),CreativeStatistic.class);
+    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
+    //                             (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
+    //                 }else {
+    //                     score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
+    //                 }
+    //             }
+    //             double s1 = score;
+    //             score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
+    //             item.setScore(score);
+    //             item.setScore_type(666);
+    //
+    //             Map<String, Object> ext = this.extMap(statistic, "666", dto.getCpa(), exp666Param.getOrDefault("viewThreshold",5000d), exp663Param.getOrDefault("alpha", 1d),
+    //                     exp663Param.getOrDefault("beta", 10000d), exp666Param.getOrDefault("beta_k",9d));
+    //             ext.put("s1", s1);
+    //             ext.put("group_field", statisticType);
+    //             item.setExt(ext);
+    //
+    //         }catch (Exception e){
+    //             log.error("svc=thompsonScorerByExp666 {}",gson.toJson(e.getStackTrace()));
+    //         }
+    //         result.add(item);
+    //         i++;
+    //     }
+    //     Collections.sort(result);
+    //     return result;
+    // }
+
+    public List<AdRankItem> thompsonScorerByExp666(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
+        List<AdRankItem> result = new LinkedList<>();
+        Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(redisCreativeStatisticsPrefix, adIdList);
+        Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(redisVideoCreativeStatisticsPrefix + param.getVideoId() + "_", adIdList);
+
+        double alpha = exp666Param.getOrDefault("alpha", 0d);
+        double beta = exp666Param.getOrDefault("beta", 0d);
+        double betaK = exp666Param.getOrDefault("beta_k", 0d) + 1;
+        double viewThreshold = exp666Param.getOrDefault("viewThreshold", 7000d);
 
-        List<String> redisKey=new LinkedList<>();
-        adIdList.forEach(creativeDTO -> {
-            redisKey.add(redisCreativeStatisticsPrefix+creativeDTO.getCreativeId());
-        });
-        List<String> values=redisHelper.getValues(redisKey);
-        List<String> combineKeys=new LinkedList<>();
-        adIdList.forEach(creativeDTO -> {
-            combineKeys.add(redisVideoCreativeStatisticsPrefix+param.getVideoId()+"_"+creativeDTO.getCreativeId());
-        });
-        List<String> combineValues=redisHelper.getValues(combineKeys);
-        int i=0;
-        double score=0d;
-        for(AdPlatformCreativeDTO dto:adIdList){
-            AdRankItem item=new AdRankItem();
-            item.setVideoId(param.getVideoId());
-            item.setCpa(dto.getCpa());
-            item.setAdId(dto.getCreativeId());
+        for (AdPlatformCreativeDTO dto : adIdList) {
+            Long cid = dto.getCreativeId();
+            String pairOrSingle;
+            double order;
+            double exp;
+            CreativeStatistic vidCidStatistics = videoCreativeStatisticsMap.getOrDefault(cid, new CreativeStatistic());
+            CreativeStatistic cidStatistics = creativeStatisticsMap.getOrDefault(cid, new CreativeStatistic());
+            // 日志上报扩展参数
+            Map<String, Object> ext = new HashMap<>();
+            double score = 0.0;
             try {
-                jsonStr=combineValues.get(i);
-                if(jsonStr==null){
-                    jsonStr=values.get(i);
-                    if(jsonStr==null){
-                        score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
-                    }else {
-                        statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-                        score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-                                (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-                    }
-                }else {
-                    statistic =gson.fromJson(jsonStr,CreativeStatistic.class);
-                    if(Double.parseDouble(statistic.getExp())>exp666Param.getOrDefault("viewThreshold",5000d)){
-                        score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-                                (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-                    } else if( values.get(i)!=null) {
-                        statistic =gson.fromJson(values.get(i),CreativeStatistic.class);
-                        score = betaSampler(exp666Param.getOrDefault("alpha",1d)+Long.parseLong(statistic.getOrder()) ,
-                                (exp666Param.getOrDefault("beta",100000d)+Long.parseLong(statistic.getExp()))/(1+exp666Param.getOrDefault("beta_k",9d))) ;
-                    }else {
-                        score = betaSampler(exp666Param.getOrDefault("alpha",1d),exp666Param.getOrDefault("beta",100000d));
-                    }
-                }
-                score=score*dto.getCpa()*dto.getBid1()*dto.getBid2();
-                item.setScore(score);
 
-                Map<String, Object> ext = this.extMap(statistic, "665", dto.getCpa(), exp666Param.getOrDefault("viewThreshold",5000d), exp663Param.getOrDefault("alpha", 1d),
-                        exp663Param.getOrDefault("beta", 10000d), exp666Param.getOrDefault("beta_k",9d));
-                item.setExt(ext);
 
-                item.setScore_type(666);
-            }catch (Exception e){
-                log.error("svc=thompsonScorerByExp666 {}",gson.toJson(e.getStackTrace()));
+                if (vidCidStatistics.parseExpToDouble() >= viewThreshold) {
+                    pairOrSingle = "pair";
+                    order = vidCidStatistics.parseOrderToDouble();
+                    exp = vidCidStatistics.parseExpToDouble();
+                } else {
+                    pairOrSingle = "single";
+                    order = cidStatistics.parseOrderToDouble();
+                    exp = cidStatistics.parseExpToDouble();
+                }
+
+                double s1 = this.betaSampler((alpha + order), ((beta + exp) / betaK));
+                score = s1 * dto.getCpa() * dto.getBid1() * dto.getBid2();
+
+
+                ext.put("alpha", alpha);
+                ext.put("beta", beta);
+                ext.put("beta_k", betaK);
+                ext.put("order", order);
+                ext.put("exp", exp);
+                ext.put("s1", s1);
+                ext.put("bid1", dto.getBid1());
+                ext.put("bid2", dto.getBid2());
+                ext.put("viewThreshold", viewThreshold);
+                ext.put("pairOrSingle", pairOrSingle);
+                ext.put("cidStatistic", JSON.toJSONString(cidStatistics));
+                ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistics));
+                ext.put("cpa", dto.getCpa());
+                ext.put("abCode", 666);
+            } catch (Exception e) {
+                log.error("svc=666exp, error: ", e);
             }
+            AdRankItem item = new AdRankItem();
+            item.setCpa(dto.getCpa());
+            item.setAdId(dto.getCreativeId());
+            item.setScore(score);
+            item.setExt(ext);
+            item.setVideoId(param.getVideoId());
+            item.setScore_type(666);
+            item.setBid2(dto.getBid2());
+            item.setBid1(dto.getBid1());
+            item.setWeight(dto.getWeight());
+            item.setCreativeCode(dto.getCreativeCode());
             result.add(item);
-            i++;
         }
-        Collections.sort(result);
-        return result;
-    }
 
-    private Map<String, Object> extMap(CreativeStatistic statistic, String abCode, Double cpa, Double viewThreshold, Double alpha, Double beta, Double beta_k) {
-        Map<String, Object> map = new HashMap<>();
-        map.put("abCode", abCode);
-        if (Objects.nonNull(viewThreshold)) {
-            map.put("viewThreshold", viewThreshold);
-        }
-        map.put("alpha", alpha);
-        map.put("beta", beta);
-        map.put("beta_k", beta_k);
-        map.put("cpa", cpa);
-        if (Objects.nonNull(statistic)) {
-            map.put("click", statistic.click);
-            map.put("order", statistic.getOrder());
-            map.put("exp", statistic.getExp());
-        }
-        return map;
+        result.sort(equalsRandomComparator());
+        return result;
     }
 
-
     public List<AdRankItem> thompsonScorerByExp669(ScoreParam param, List<AdPlatformCreativeDTO> adIdList) {
         Map<Long, CreativeStatistic> creativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidActionKeyV1, adIdList);
         Map<Long, CreativeStatistic> videoCreativeStatisticsMap = this.batchFindCreativeRedisCache(rediCidVidActionKeyV1 + param.getVideoId() + "_", adIdList);
@@ -257,12 +395,15 @@ public class VideoAdThompsonScorerV2 {
         Double videoCreativeExpSum = this.sumCreativeStatisticExp(videoCreativeStatisticsMap.values());
 
         List<AdRankItem> result = new ArrayList<>(adIdList.size());
-        this.calcScore(result, adIdList,1d, creativeExpSum, videoCreativeExpSum, creativeStatisticsMap, videoCreativeStatisticsMap, exp669Param);
+        this.calcScore(result, adIdList, 1d, creativeExpSum, videoCreativeExpSum, creativeStatisticsMap, videoCreativeStatisticsMap, exp669Param);
         result.sort(equalsRandomComparator());
 
         for (AdRankItem adRankItem : result) {
             adRankItem.setVideoId(param.getVideoId());
             adRankItem.setScore_type(699);
+            if (MapUtils.isNotEmpty(adRankItem.getExt())) {
+                adRankItem.getExt().put("abCode", 699);
+            }
         }
 
         return result;
@@ -279,32 +420,55 @@ public class VideoAdThompsonScorerV2 {
         for (AdRankItem adRankItem : result) {
             adRankItem.setVideoId(param.getVideoId());
             adRankItem.setScore_type(670);
+            if (MapUtils.isNotEmpty(adRankItem.getExt())) {
+                adRankItem.getExt().put("abCode", 670);
+            }
         }
 
         return result;
     }
 
-    private void calcScore(List<AdRankItem> result, List<AdPlatformCreativeDTO> adIdList,Double alpha, Double cidBeta, Double vidCidBeta, Map<Long, CreativeStatistic> cidMap, Map<Long, CreativeStatistic> vidCidMap, Map<String, Double> expParam) {
+    private void calcScore(List<AdRankItem> result, List<AdPlatformCreativeDTO> adIdList, Double alpha, Double cidBeta, Double vidCidBeta, Map<Long, CreativeStatistic> cidMap, Map<Long, CreativeStatistic> vidCidMap, Map<String, Double> expParam) {
         for (AdPlatformCreativeDTO dto : adIdList) {
 
             CreativeStatistic cidStatistic = cidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
             CreativeStatistic vidCidStatistic = vidCidMap.getOrDefault(dto.getCreativeId(), new CreativeStatistic());
 
-            log.info("cid: {} statistics: {}", dto.getCreativeId(), JSON.toJSONString(cidStatistic));
-            log.info("cid: {} vidCidStatistic: {}", dto.getCreativeId(), JSON.toJSONString(vidCidStatistic));
-
             double cidScore = this.calcThompsonScore(expParam, cidStatistic, alpha, cidBeta);
             double vidCidScore = this.calcThompsonScore(expParam, vidCidStatistic, alpha, vidCidBeta);
             double w1 = expParam.getOrDefault("w1", 1d);
             double w2 = expParam.getOrDefault("w2", 1d);
-            double score = w1 * vidCidScore + w2 * cidScore;
+            double score = ((w1 * vidCidScore + w2 * cidScore) / (w1 + w2)) * dto.getCpa();
+
+
+            // 日志上报扩展参数
+            Map<String, Object> ext = new HashMap<>();
+            ext.put("w1", w1);
+            ext.put("w2", w2);
+            ext.put("cidScore", cidScore);
+            ext.put("vidCidScore", vidCidScore);
+            ext.put("cidA", alpha);
+            ext.put("vidCidA", alpha);
+            ext.put("cidB", cidBeta);
+            ext.put("vidCidB", vidCidBeta);
+            ext.put("bid1", dto.getBid1());
+            ext.put("bid2", dto.getBid2());
+            ext.put("cidStatistic", JSON.toJSONString(cidStatistic));
+            ext.put("vidCidStatistic", JSON.toJSONString(vidCidStatistic));
+            ext.put("cpa", dto.getCpa());
+
+            log.info("svc=699And670Score, score: {}, param: {}", score, JSON.toJSONString(ext));
 
-            log.info("calcScore w1: {}, w1: {}, cidScore: {}, vidCidScore: {}", w1, w2, cidScore, vidCidScore);
 
             AdRankItem item = new AdRankItem();
             item.setCpa(dto.getCpa());
             item.setAdId(dto.getCreativeId());
+            item.setWeight(dto.getWeight());
             item.setScore(score);
+            item.setExt(ext);
+            item.setBid1(dto.getBid1());
+            item.setBid2(dto.getBid2());
+            item.setCreativeCode(dto.getCreativeCode());
 
             result.add(item);
         }
@@ -336,9 +500,6 @@ public class VideoAdThompsonScorerV2 {
         double order = creativeStatistic.parseOrderToDouble() + alpha;
         double exp = creativeStatistic.parseExpToDouble() + beta;
 
-        if (order == 0 || exp == 0) {
-            return 0.0;
-        }
         log.info("calcThompsonScore.order: {}, exp: {}", order, exp);
         return this.betaSampler(order, exp);
     }
@@ -365,7 +526,26 @@ public class VideoAdThompsonScorerV2 {
         };
     }
 
-    class CreativeStatistic{
+
+    private Map<String, Object> extMap(CreativeStatistic statistic, String abCode, Double cpa, Double viewThreshold, Double alpha, Double beta, Double beta_k) {
+        Map<String, Object> map = new HashMap<>();
+        map.put("abCode", abCode);
+        if (Objects.nonNull(viewThreshold)) {
+            map.put("viewThreshold", viewThreshold);
+        }
+        map.put("alpha", alpha);
+        map.put("beta", beta);
+        map.put("beta_k", beta_k);
+        map.put("cpa", cpa);
+        if (Objects.nonNull(statistic)) {
+            map.put("click", statistic.click);
+            map.put("order", statistic.getOrder());
+            map.put("exp", statistic.getExp());
+        }
+        return map;
+    }
+
+    class CreativeStatistic {
 
         private String exp;
         private String click;
@@ -386,7 +566,7 @@ public class VideoAdThompsonScorerV2 {
             return exp;
         }
 
-        public Double parseExpToDouble() {
+        public double parseExpToDouble() {
             if (StringUtils.isBlank(exp)) {
                 return 0.0;
             }
@@ -404,8 +584,8 @@ public class VideoAdThompsonScorerV2 {
             return click;
         }
 
-        public Double parseClickToDouble(){
-            if (StringUtils.isBlank(click)){
+        public double parseClickToDouble() {
+            if (StringUtils.isBlank(click)) {
                 return 0.0;
             }
             return Double.parseDouble(click);
@@ -422,8 +602,8 @@ public class VideoAdThompsonScorerV2 {
             return order;
         }
 
-        public Double parseOrderToDouble(){
-            if (StringUtils.isBlank(order)){
+        public double parseOrderToDouble() {
+            if (StringUtils.isBlank(order)) {
                 return 0.0;
             }
             return Double.parseDouble(order);
@@ -440,40 +620,49 @@ public class VideoAdThompsonScorerV2 {
             return cpa;
         }
 
-        public Double parseCpaToDouble() {
+        public double parseCpaToDouble() {
             if (StringUtils.isBlank(cpa)) {
                 return 0.0;
             }
             return Double.parseDouble(cpa);
         }
     }
+
     double betaSampler(double alpha, double beta) {
+        if (alpha == 0 || beta == 0) {
+            return 0.0d;
+        }
         BetaDistribution betaSample = new BetaDistribution(alpha, beta);
         return betaSample.sample();
     }
+
     @Value("${ad.engine.new.thompson.exp.V2.663:{\"randomMin\":0.000001,\"randomMax\":0.00001,\"alpha\":1,\"beta\":10000}}")
-    public void setExp663Param(String str){
-        this.exp663Param=gson.fromJson(str,Map.class);
+    public void setExp663Param(String str) {
+        this.exp663Param = gson.fromJson(str, Map.class);
     }
+
     @Value("${ad.engine.new.thompson.exp.V2.664:{}}")
-    public void setExp664Param(String str){
-        this.exp663Param=gson.fromJson(str,Map.class);
+    public void setExp664Param(String str) {
+        this.exp664Param = gson.fromJson(str, Map.class);
     }
+
     @Value("${ad.engine.new.thompson.exp.V2.665:{}}")
-    public void setExp665Param(String str){
-        this.exp663Param=gson.fromJson(str,Map.class);
+    public void setExp665Param(String str) {
+        this.exp665Param = gson.fromJson(str, Map.class);
     }
+
     @Value("${ad.engine.new.thompson.exp.V2.666:{}}")
-    public void setExp666Param(String str){
-        this.exp663Param=gson.fromJson(str,Map.class);
+    public void setExp666Param(String str) {
+        this.exp666Param = gson.fromJson(str, Map.class);
     }
 
     @Value("${ad.engine.new.thompson.exp.V2.669:{}}")
-    public void setExp669Param(String str){
-        this.exp669Param=gson.fromJson(str,Map.class);
+    public void setExp669Param(String str) {
+        this.exp669Param = gson.fromJson(str, Map.class);
     }
+
     @Value("${ad.engine.new.thompson.exp.V2.670:{}}")
-    public void setExp670Param(String str){
-        this.exp670Param=gson.fromJson(str,Map.class);
+    public void setExp670Param(String str) {
+        this.exp670Param = gson.fromJson(str, Map.class);
     }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/VlogAdCvrLRScorer.java

@@ -45,6 +45,7 @@ public class VlogAdCvrLRScorer extends BaseLRModelScorer {
         }
 
         long startTime = System.currentTimeMillis();
+        LOGGER.info("VlogAdCvrLRScorer.scorerConfigInfo.getModelPath: {}: {}", scorerConfigInfo.getModelPath(), this.getModel());
         LRModel model = (LRModel) this.getModel();
         LOGGER.debug("model size: [{}]", model.getModelSize());
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/convert/RequestConvert.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score.convert;
 import com.tzld.piaoquan.ad.engine.service.score.param.RecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRequestContext;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
+import com.tzld.piaoquan.ad.engine.service.score.param.RankRecommendRequestParam;
 
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/dto/AdPlatformCreativeDTO.java

@@ -30,6 +30,8 @@ public class AdPlatformCreativeDTO {
 
     private Double ecpm2;
 
+    private double weight;
+
     public static void main(String[] args) {
         System.out.println(JSON.toJSONString(AdPlatformCreativeDTO.builder()
                 .creativeId(3366L)

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -1,7 +1,7 @@
 package com.tzld.piaoquan.ad.engine.service.score.impl;
 
+import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
-import com.aliyun.openservices.shade.com.alibaba.fastjson.JSON;
 import com.tzld.piaoquan.ad.engine.service.predict.helper.NewExpInfoHelper;
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
 import com.tzld.piaoquan.ad.engine.service.log.LogHubService;
@@ -178,6 +178,14 @@ public class RankServiceImpl implements RankService {
             }else {
                 log.info("svc=pid_log obj={}", JSONObject.toJSONString(object));
             }
+
+            for (AdRankItem adRankItem : rankResult) {
+                adRankItem.getExt().put("userAdFeature", JSON.toJSONString(userAdFeature));
+            }
+            // 日志上报
+            logHubService.scoreLogUpload(param, request.getAdIdList(), rankResult, request,
+                    inCvrAdjustingExp ? "cvr_adjusting" : "base_line", "531");
+
             return rankResult.get(0);
         }else {
             //空返回值
@@ -246,36 +254,43 @@ public class RankServiceImpl implements RankService {
         ThresholdPredictModelParam modelParam = ThresholdPredictModelParam.builder()
                 .build();
         // 兜底方案
+        String abCode = "";
         List<AdRankItem> rankResult=null;
         if(expCodes.contains("663")||
                 (expCodes.contains(NewExpInfoHelper.flagId)&&NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
                         request.getAppType().toString(),request.getNewExpGroup(),"663",modelParam))){
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp663(param, request.getAdIdList());
+            abCode = "663";
         }else if(expCodes.contains("664")||
                 (expCodes.contains(NewExpInfoHelper.flagId)&&NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
                         request.getAppType().toString(),request.getNewExpGroup(),"664",modelParam))){
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp664(param, request.getAdIdList());
+            abCode = "664";
         }else if(expCodes.contains("665")||
                 (expCodes.contains(NewExpInfoHelper.flagId)&&NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
                         request.getAppType().toString(),request.getNewExpGroup(),"665",modelParam))){
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp665(param, request.getAdIdList());
+            abCode = "665";
         }else if(expCodes.contains("666")||
                 (expCodes.contains(NewExpInfoHelper.flagId)&&NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
                         request.getAppType().toString(),request.getNewExpGroup(),"666",modelParam))){
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp666(param, request.getAdIdList());
+            abCode = "666";
         } else if (expCodes.contains("669") ||
                 (expCodes.contains(NewExpInfoHelper.flagId) && NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
                         request.getAppType().toString(), request.getNewExpGroup(), "669", modelParam))) {
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp669(param, request.getAdIdList());
+            abCode = "669";
         } else if (expCodes.contains("670") ||
                 (expCodes.contains(NewExpInfoHelper.flagId) && NewExpInfoHelper.checkInNewExpGroupAndSetParamIfIn(
                         request.getAppType().toString(), request.getNewExpGroup(), "670", modelParam))) {
             rankResult = videoAdThompsonScorerV2.thompsonScorerByExp670(param, request.getAdIdList());
+            abCode = "670";
         }
         log.info("RankServiceImpl.adItemRankWithVideoAdThompson.adIdList: {}, result: {}", JSON.toJSONString(request.getAdIdList()), JSON.toJSONString(rankResult));
 
         // 日志上报
-        logHubService.scoreLogUpload(param, request.getAdIdList(), rankResult, request, "video_ad_thompson");
+        logHubService.scoreLogUpload(param, request.getAdIdList(), rankResult, request, "video_ad_thompson", abCode);
 
         if (!CollectionUtils.isEmpty(rankResult)) {
             JSONObject object=new JSONObject();