Merge branch 'master' into dev-xym-add-feature

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/redis/AdRedisHelper.java

@@ -7,6 +7,7 @@ import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -51,4 +52,8 @@ public class AdRedisHelper {
         return Boolean.TRUE.equals(adRedisTemplate.hasKey(key));
     }
 
+    public List<String> mget(List<String> keys) {
+        return adRedisTemplate.opsForValue().multiGet(keys);
+    }
+
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/util/DateUtils.java

@@ -1,10 +1,8 @@
 package com.tzld.piaoquan.ad.engine.commons.util;
 
 import java.text.SimpleDateFormat;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
+import java.time.*;
+import java.time.format.DateTimeFormatter;
 import java.util.Calendar;
 import java.util.Date;
 
@@ -52,4 +50,10 @@ public final class DateUtils {
         int minute = zonedDateTime.getMinute();
         return minute / 15 + hour * 4;
     }
+
+    public static String getThatDayDateString() {
+        DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern("yyyyMMdd");
+        LocalDate today = LocalDate.now();
+        return dateFormat.format(today);
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/entity/GuaranteeView.java

@@ -0,0 +1,19 @@
+package com.tzld.piaoquan.ad.engine.service.entity;
+
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class GuaranteeView {
+
+    private String adrId;
+
+    private Integer adrAlgoViewNum;
+
+    private Integer allAlgoViewNum;
+
+    private Integer guaranteeNum;
+
+    private Double guaranteeRate;
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

@@ -1,7 +1,10 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
+import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
+import com.tzld.piaoquan.ad.engine.commons.util.DateUtils;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.commons.enums.RedisPrefixEnum;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
 import com.tzld.piaoquan.ad.engine.commons.redis.AdRedisHelper;
@@ -10,8 +13,10 @@ import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.service.feature.FeatureService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 
 import java.util.*;
 import java.util.stream.Collectors;
@@ -25,11 +30,18 @@ public abstract class RankStrategyBasic implements RankStrategy {
     @ApolloJsonValue("${creative.model.score.coefficient:{}}")
     private Map<Long, Double> creativeScoreCoefficient;
 
+    @Value("${guarantee.exp:742}")
+    protected String guaranteeExp;
+    @ApolloJsonValue("${alpha:1.0}")
+    protected Double alpha;
     @Autowired
     private FeatureService featureService;
     @Autowired
     protected AdRedisHelper adRedisHelper;
 
+    String adPlatformGuaranteeKey = "ad:platform:guarantee:data:{date}:{adverId}";
+
+
     protected Feature getFeature(ScoreParam param, RankRecommendRequestParam request) {
         List<AdPlatformCreativeDTO> adIdList = request.getAdIdList();
         List<String> cidList = adIdList.stream()
@@ -98,4 +110,84 @@ public abstract class RankStrategyBasic implements RankStrategy {
         }
 
     }
+
+    protected Map<String, GuaranteeView> getGuaranteeViewMap(RankRecommendRequestParam request, ScoreParam scoreParam) {
+        Map<String, GuaranteeView> map = new HashMap<>();
+        try {
+            if (scoreParam.getExpCodeSet().contains(guaranteeExp)) {
+                String thatDayDateString = DateUtils.getThatDayDateString();
+                String redisKey = adPlatformGuaranteeKey.replace("{date}", thatDayDateString);
+                List<String> adVerIds = request.getAdIdList().stream().map(AdPlatformCreativeDTO::getAdVerId).distinct()
+                        .filter(Objects::nonNull).collect(Collectors.toList());
+                List<String> redisKeys = adVerIds.stream().map(e -> redisKey.replace("{adverId}", e)).collect(Collectors.toList());
+                List<String> values = adRedisHelper.mget(redisKeys);
+                if (CollectionUtils.isNotEmpty(values)) {
+                    for (int i = 0; i < redisKeys.size(); i++) {
+                        String value = values.get(i);
+                        if (value != null) {
+                            GuaranteeView guaranteeView = JSONObject.parseObject(value, GuaranteeView.class);
+                            map.put(adVerIds.get(i), guaranteeView);
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.error("getGuaranteeViewMap error", e);
+        }
+
+        return map;
+    }
+
+    protected void setGuaranteeWeight(Map<String, GuaranteeView> map, String adVerId, Map<String, Object> ext) {
+        if (MapUtils.isNotEmpty(map)) {
+            GuaranteeView guaranteeView = map.get(adVerId);
+            if (guaranteeView != null) {
+                double guaranteeWeight = calculateGuaranteedWeight(guaranteeView);
+                ext.put("guaranteeView", guaranteeView.toString());
+                ext.put("guaranteeWeight", guaranteeWeight);
+            }
+        }
+    }
+
+    protected double calculateGuaranteedWeight(GuaranteeView guaranteeView) {
+        if (guaranteeView == null) {
+            return 1.0;
+        }
+        double guaranteeWeight;
+        if (guaranteeView.getGuaranteeNum() == null || guaranteeView.getGuaranteeNum() == 0
+                || guaranteeView.getGuaranteeRate() == null || guaranteeView.getGuaranteeRate() == 0.0) {
+            guaranteeWeight = 1.0;
+        } else {
+            if (guaranteeView.getAdrAlgoViewNum() != null &&
+                    guaranteeView.getGuaranteeNum() <= guaranteeView.getAdrAlgoViewNum()) {
+                guaranteeWeight = 0.0;
+            } else {
+                int allViewNum = guaranteeView.getAllAlgoViewNum() == null || guaranteeView.getAllAlgoViewNum() == 0 ?
+                        1 : guaranteeView.getAllAlgoViewNum();
+                int adrAlogViewNum = guaranteeView.getAdrAlgoViewNum() == null || guaranteeView.getAdrAlgoViewNum() == 0 ?
+                        1 : guaranteeView.getAdrAlgoViewNum();
+                //guaranteeView.getGuaranteeRate() 是百分之几  要乘0.01
+                guaranteeWeight = guaranteeView.getGuaranteeRate() * 0.01 * allViewNum / adrAlogViewNum;
+                if (guaranteeWeight < 0.5) {
+                    guaranteeWeight = 0.5;
+                }
+                if (guaranteeWeight > 2) {
+                    guaranteeWeight = 2;
+                }
+            }
+        }
+        return guaranteeWeight;
+    }
+
+    protected double getGuaranteeScoreCoefficient(ScoreParam scoreParam, Map<String, Object> ext) {
+        if (scoreParam.getExpCodeSet().contains(guaranteeExp)) {
+            if (ext.get("guaranteeWeight") == null) {
+                return 1.0;
+            } else {
+                return Math.pow((double) ext.get("guaranteeWeight"), alpha);
+            }
+        } else {
+            return 1.0;
+        }
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -7,6 +7,7 @@ import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdRankItem;
 import lombok.extern.slf4j.Slf4j;
@@ -88,7 +89,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
 
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
-
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, scoreParam);
         List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
@@ -112,7 +113,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                         adRankItem.getExt().put("isApi", "1");
                     }
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-
+                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt());
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
@@ -211,8 +212,9 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         for (AdRankItem item : result) {
 
             double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
-            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
-
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(scoreParam, item.getExt());
+            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa() * guaranteeScoreCoefficient);
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
             item.getScoreMap().put("cpmCoefficient", cpmCoefficient);

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -5,6 +5,7 @@ import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
@@ -97,6 +98,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, scoreParam);
         List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
@@ -121,7 +123,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                     }
 
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-
+                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt());
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
@@ -228,8 +230,9 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         for (AdRankItem item : result) {
 
             double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
-            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
-
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(scoreParam, item.getExt());
+            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa() * guaranteeScoreCoefficient);
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
             item.getScoreMap().put("cpmCoefficient", cpmCoefficient);

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -5,6 +5,7 @@ import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
@@ -98,6 +99,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, scoreParam);
         List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
@@ -122,7 +124,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     }
 
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-
+                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt());
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
@@ -229,8 +231,9 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         for (AdRankItem item : result) {
 
             double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
-            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
-
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(scoreParam, item.getExt());
+            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa() * guaranteeScoreCoefficient);
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
             item.getScoreMap().put("cpmCoefficient", cpmCoefficient);

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy687.java

@@ -4,6 +4,7 @@ import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
@@ -74,6 +75,7 @@ public class RankStrategyBy687 extends RankStrategyBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, scoreParam);
         List<AdRankItem> adRankItems = new ArrayList<>(request.getAdIdList().size());
         Random random = new Random();
         if (similarityConcurrent) {
@@ -94,7 +96,7 @@ public class RankStrategyBy687 extends RankStrategyBasic {
                     adRankItem.getExt().put("isApi", "1");
                 }
                 adRankItem.getExt().put("recallsources", dto.getRecallSources());
-
+                setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt());
 
                 String cidStr = dto.getCreativeId().toString();
                 Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
@@ -218,7 +220,9 @@ public class RankStrategyBy687 extends RankStrategyBasic {
 
         long time3 = System.currentTimeMillis();
         for (AdRankItem item : result) {
-            item.setScore(item.getLrScore() * item.getCpa());
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(scoreParam, item.getExt());
+            item.setScore(item.getLrScore() * item.getCpa() * guaranteeScoreCoefficient);
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
             item.getFeatureMap().putAll(userFeatureMap);

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -1,10 +1,13 @@
 package com.tzld.piaoquan.ad.engine.service.score.strategy;
 
+import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.ad.engine.commons.redis.AdRedisHelper;
 import com.tzld.piaoquan.ad.engine.commons.score.ScoreParam;
 import com.tzld.piaoquan.ad.engine.commons.score.ScorerUtils;
 import com.tzld.piaoquan.ad.engine.commons.thread.ThreadPoolFactory;
 import com.tzld.piaoquan.ad.engine.commons.util.*;
+import com.tzld.piaoquan.ad.engine.service.entity.GuaranteeView;
 import com.tzld.piaoquan.ad.engine.service.feature.Feature;
 import com.tzld.piaoquan.ad.engine.commons.dto.AdPlatformCreativeDTO;
 import com.tzld.piaoquan.ad.engine.commons.param.RankRecommendRequestParam;
@@ -13,6 +16,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.xm.Similarity;
@@ -42,6 +46,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
     @Value("${word2vec.exp:694}")
     private String word2vecExp;
 
+
     // FIXME(zhoutian): 可能需要独立配置
     @ApolloJsonValue("${rank.score.weight.680:{}}")
     private Map<String, Double> weightMap;
@@ -51,6 +56,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
 
     Set<String> sparseFeatureSet;
 
+
     @PostConstruct
     public void afterInit() {
         this.readBucketFile();
@@ -164,6 +170,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
+        Map<String, GuaranteeView> map = getGuaranteeViewMap(request, scoreParam);
         List<AdRankItem> adRankItems = new ArrayList<>();
         Random random = new Random();
         List<Future<AdRankItem>> futures = new ArrayList<>();
@@ -186,9 +193,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     } else {
                         adRankItem.getExt().put("isApi", "1");
                     }
-
                     adRankItem.getExt().put("recallsources", dto.getRecallSources());
-
+                    setGuaranteeWeight(map, dto.getAdVerId(), adRankItem.getExt());
                     String cidStr = dto.getCreativeId().toString();
                     Map<String, String> cidFeatureMap = adRankItem.getFeatureMap();
                     Map<String, Map<String, String>> cidFeature = allCidFeature.getOrDefault(cidStr, new HashMap<>());
@@ -300,8 +306,9 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         for (AdRankItem item : result) {
 
             double scoreCoefficient = creativeScoreCoefficient.getOrDefault(item.getAdId(), 1d);
-            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa());
-
+            double guaranteeScoreCoefficient = getGuaranteeScoreCoefficient(scoreParam, item.getExt());
+            item.setScore(item.getLrScore() * scoreCoefficient * item.getCpa() * guaranteeScoreCoefficient);
+            item.getScoreMap().put("guaranteeScoreCoefficient", guaranteeScoreCoefficient);
             item.getScoreMap().put("cpa", item.getCpa());
             item.getScoreMap().put("cpm", item.getCpm());
             item.getScoreMap().put("cpmCoefficient", cpmCoefficient);