Merge branch 'refs/heads/dev-xym-add-category' into pre-master

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/dto/AdPlatformCreativeDTO.java

@@ -52,4 +52,6 @@ public class AdPlatformCreativeDTO {
     private String customer;
 
     private Long customerId;
+
+    private String categoryName;
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/PAIModelV1.java

@@ -43,7 +43,7 @@ public class PAIModelV1 {
             "user_vid_return_tags_2h", "user_vid_return_tags_1d", "user_vid_return_tags_3d", "user_vid_return_tags_7d",
             "user_vid_return_tags_14d", "root_source_scene", "root_source_channel", "title_split", "user_vid_share_tags_1d",
             "user_vid_share_tags_14d", "user_vid_return_cate1_14d", "user_vid_return_cate2_14d", "user_vid_share_cate1_14d",
-            "user_vid_share_cate2_14d", "user_conver_ad_class", "user_layer_class"
+            "user_vid_share_cate2_14d", "user_conver_ad_class"
     };
 
     private static final String[] sparseUserLongFeatures = {
@@ -65,7 +65,7 @@ public class PAIModelV1 {
     };
 
     private static final String[] sparseAdStrFeatures = {
-            "profession"
+            "profession", "category_name"
     };
 
     private final String[] userFeatures = {

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/PAIModelV2.java

@@ -43,7 +43,7 @@ public class PAIModelV2 {
             "user_vid_return_tags_2h", "user_vid_return_tags_1d", "user_vid_return_tags_3d", "user_vid_return_tags_7d",
             "user_vid_return_tags_14d", "root_source_scene", "root_source_channel", "title_split", "user_vid_share_tags_1d",
             "user_vid_share_tags_14d", "user_vid_return_cate1_14d", "user_vid_return_cate2_14d", "user_vid_share_cate1_14d",
-            "user_vid_share_cate2_14d", "user_conver_ad_class", "user_layer_class"
+            "user_vid_share_cate2_14d", "user_conver_ad_class"
     };
 
     private static final String[] sparseUserLongFeatures = {
@@ -65,7 +65,7 @@ public class PAIModelV2 {
     };
 
     private static final String[] sparseAdStrFeatures = {
-            "profession"
+            "profession", "category_name"
     };
 
     private final String[] userFeatures = {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/impl/PredictModelServiceImpl.java

@@ -23,10 +23,8 @@ import com.tzld.piaoquan.ad.engine.service.predict.param.RoiThresholdPredictMode
 import com.tzld.piaoquan.ad.engine.service.predict.param.ThresholdPredictModelParam;
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.RoiPredictModelRequestParam;
 import com.tzld.piaoquan.ad.engine.service.predict.param.request.ThresholdPredictModelRequestParam;
-import com.tzld.piaoquan.ad.engine.service.predict.v2.ConvertUtil;
-import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictContext;
-import com.tzld.piaoquan.ad.engine.service.predict.v2.PredictServiceV2;
-import com.tzld.piaoquan.ad.engine.service.predict.v2.RootSessionIdPredict;
+import com.tzld.piaoquan.ad.engine.service.predict.v2.*;
+import com.tzld.piaoquan.ad.engine.service.user.UserService;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -67,6 +65,11 @@ public class PredictModelServiceImpl implements PredictModelService {
     private PredictServiceV2 predictServiceV2;
     @Autowired
     private RootSessionIdPredict rootSessionIdPredict;
+    @Autowired
+    private UserLayerRootSessionIdPredict userLayerRootSessionIdPredict;
+
+    @Autowired
+    private UserService userService;
 
     @Autowired
     private LogHubService logHubService;
@@ -271,14 +274,21 @@ public class PredictModelServiceImpl implements PredictModelService {
             modelParam.addUserExtraFuture("shareType", shareType);
             setExtraParam(modelParam);
 
+            requestParam.setShareType(shareType);
+            String userLayer = userService.getUserLayerByMid(requestParam.getMid());
+            requestParam.setUserLayer(userLayer);
+
             // 先走rootSessionId 实验
-            PredictContext ctx = ConvertUtil.predictParam2Context(requestParam);
-            ctx.setShareType(shareType);
-            Map<String, Object> predict = rootSessionIdPredict.predict(ctx);
+            Map<String, Object> predict = rootSessionIdPredict.predict(ConvertUtil.predictParam2Context(requestParam));
             if (MapUtils.isNotEmpty(predict)) {
                 return predict;
             }
 
+            Map<String, Object> userLayerPredict = userLayerRootSessionIdPredict.predict(ConvertUtil.predictParam2Context(requestParam));
+            if (MapUtils.isNotEmpty(userLayerPredict)) {
+                return userLayerPredict;
+            }
+
             String appTypeStr = requestParam.getAppType().toString();
 
             List<String> userSourceAndLayerAdRateExpIds = Arrays.asList(userSourceLayerAdRateExpIds.split(","));

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/param/request/ThresholdPredictModelRequestParam.java

@@ -41,4 +41,8 @@ public class ThresholdPredictModelRequestParam {
     private String shareLayer;
 
     private String rootSessionId;
+
+    private String shareType;
+
+    private String userLayer;
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/BasicPredict.java

@@ -1,7 +1,12 @@
 package com.tzld.piaoquan.ad.engine.service.predict.v2;
 
+import com.tzld.piaoquan.ad.engine.service.predict.container.RandWContainer;
+import lombok.Data;
+
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 public abstract class BasicPredict {
 
@@ -22,4 +27,17 @@ public abstract class BasicPredict {
         rtnMap.put("pqtId", ctx.getPqtId());
         return rtnMap;
     }
+
+    protected double calcScoreByMid(String mid) {
+        int hash = mid.hashCode();
+        hash = hash < 0 ? -hash : hash;
+        return (hash + RandWContainer.getRandW()) % 100 / 100d;
+    }
+
+    @Data
+    protected static class RootSessionIdTailConfigItem {
+        private Set<String> appType = new HashSet<>();
+        private Set<String> tail = new HashSet<>();
+        private Map<String, Double> config = new HashMap<>();
+    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/ConvertUtil.java

@@ -29,6 +29,8 @@ public class ConvertUtil {
 
         context.setRootSessionId(param.getRootSessionId());
         context.setAdPlatformType(param.getAdPlatformType());
+        context.setShareType(param.getShareType());
+        context.setUserLayer(param.getUserLayer());
 
         JSONObject abExpInfo = param.getAbExpInfo();
         Map<String, JSONObject> expConfigMap = ConvertUtil.parseAbTest002Info(abExpInfo);

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/PredictContext.java

@@ -73,5 +73,7 @@ public class PredictContext {
 
     private String adPlatformType;
 
+    private String userLayer;
+
     private PredictLogParam logParam = new PredictLogParam();
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/RootSessionIdPredict.java

@@ -3,21 +3,22 @@ package com.tzld.piaoquan.ad.engine.service.predict.v2;
 import com.alibaba.fastjson.JSONObject;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
-import com.tzld.piaoquan.ad.engine.service.predict.container.RandWContainer;
-import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 @Slf4j
 @Component
 public class RootSessionIdPredict extends BasicPredict {
 
     @ApolloJsonValue("${root.session.id.tail.predict.config:[]}")
-    private List<ConfigItem> configItems;
+    private List<RootSessionIdTailConfigItem> configItems;
 
     @Override
     public Map<String, Object> predict(PredictContext ctx) {
@@ -26,9 +27,8 @@ public class RootSessionIdPredict extends BasicPredict {
             return Collections.emptyMap();
         }
 
-        int hash = ctx.getMid().hashCode();
-        hash = hash < 0 ? -hash : hash;
-        double score = (hash + RandWContainer.getRandW()) % 100 / 100d;
+
+        double score = this.calcScoreByMid(ctx.getMid());
 
         String appType = ctx.getAppType();
         String tail = rootSessionId.substring(rootSessionId.length() - 1);
@@ -37,7 +37,7 @@ public class RootSessionIdPredict extends BasicPredict {
                 .replace("mids", "");
 
 
-        for (ConfigItem item : configItems) {
+        for (RootSessionIdTailConfigItem item : configItems) {
             if (item.getAppType().contains(appType) && item.getTail().contains(tail)) {
                 double threshold;
                 if (item.getConfig().containsKey(shareType)) {
@@ -83,11 +83,4 @@ public class RootSessionIdPredict extends BasicPredict {
 
         return Collections.emptyMap();
     }
-
-    @Data
-    private static class ConfigItem {
-        private Set<String> appType = new HashSet<>();
-        private Set<String> tail = new HashSet<>();
-        private Map<String, Double> config = new HashMap<>();
-    }
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/predict/v2/UserLayerRootSessionIdPredict.java

@@ -0,0 +1,96 @@
+package com.tzld.piaoquan.ad.engine.service.predict.v2;
+
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
+import com.tzld.piaoquan.ad.engine.commons.util.JSONUtils;
+import com.tzld.piaoquan.ad.engine.service.user.UserService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+@Slf4j
+@Component
+public class UserLayerRootSessionIdPredict extends BasicPredict {
+
+    @Autowired
+    private UserService userService;
+
+    @ApolloJsonValue("${user.layer.root.session.id.tail.predict.config:[]}")
+    private List<RootSessionIdTailConfigItem> configItems;
+
+    @Override
+    public Map<String, Object> predict(PredictContext ctx) {
+
+        String rootSessionId = ctx.getRootSessionId();
+        String userLayer = ctx.getUserLayer();
+        String shareType = ctx.getShareType();
+        if (CollectionUtils.isEmpty(configItems) || StringUtils.isAnyBlank(rootSessionId, userLayer, shareType)) {
+            return Collections.emptyMap();
+        }
+
+        double score = this.calcScoreByMid(ctx.getMid());
+        String appType = ctx.getAppType();
+        String tail = rootSessionId.substring(rootSessionId.length() - 1);
+        shareType = ctx.getShareType()
+                .replace("return", "")
+                .replace("mids", "");
+
+        for (RootSessionIdTailConfigItem item : configItems) {
+            if (item.getAppType().contains(appType) && item.getTail().contains(tail)) {
+                Map<String, Object> returnMap = new HashMap<>();
+                double threshold;
+                String thresholdKey = "";
+
+                // 生成可用的Key列表
+                List<String> keys = Arrays.asList(userLayer + "_" + shareType, "default_" + shareType, "default_threshold");
+                for (String key : keys) {
+                    if (item.getConfig().containsKey(key)) {
+                        thresholdKey = key;
+                        break;
+                    }
+                }
+
+
+                threshold = item.getConfig().getOrDefault(thresholdKey, 0.0);
+
+                if (score < threshold) {
+                    returnMap.putAll(rtnAdPredict(ctx));
+                    returnMap.put("model", "userLayerRootSessionIdTailModel");
+                } else {
+                    returnMap.putAll(rtnNoAdPredict(ctx));
+                    returnMap.put("no_ad_strategy", "userLayerRootSessionIdTailModel");
+                }
+
+                returnMap.put("score", score);
+                returnMap.put("threshold", threshold);
+                returnMap.put("userLayer", userLayer);
+
+                JSONObject logJson = new JSONObject();
+                logJson.putAll(returnMap);
+                logJson.put("mid", ctx.getMid());
+                logJson.put("appType", appType);
+                logJson.put("rootSessionIdTail", tail);
+                logJson.put("shareType", shareType);
+
+                logJson.put("expId", "userLayerRootSessionIdTailExp");
+                logJson.put("thresholdParamKey", thresholdKey);
+                logJson.put("adPlatformType", ctx.getAdPlatformType());
+                logJson.put("abCode", ctx.getAdAbCode());
+                logJson.put("configItem", item);
+                logJson.put("rootSessionId", ctx.getRootSessionId());
+
+                log.info("广告跳出选择 -- userLayerRootSessionId尾号实验结果: {}, 参数: {}",
+                        JSONUtils.toJson(returnMap), logJson.toJSONString());
+
+                return returnMap;
+            }
+        }
+
+
+        return Collections.emptyMap();
+    }
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

@@ -853,6 +853,7 @@ public abstract class RankStrategyBasic implements RankStrategy {
             reqFeature.put("adid", String.valueOf(adPlatformCreativeDTO.getAdId()));
             reqFeature.put("adverid", String.valueOf(adPlatformCreativeDTO.getAdVerId()));
             reqFeature.put("profession", adPlatformCreativeDTO.getProfession());
+            reqFeature.put("category_name", adPlatformCreativeDTO.getCategoryName());
         }
         adRankItem.getMetaFeatureMap().put("reqFeature", reqFeature);
         adRankItem.getMetaFeatureMap().put("sceneFeature", sceneFeatureMap);

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -145,8 +145,6 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         userFeatureMap.put("user_vid_share_cate1_14d", g2Feature.getOrDefault("cate1_14d", null));
         userFeatureMap.put("user_vid_share_cate2_14d", g2Feature.getOrDefault("cate2_14d", null));
 
-        userFeatureMap.put("user_layer_class", reqFeature.getOrDefault("user_layer_class", null));
-
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
@@ -202,6 +200,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     cidFeatureMap.put("adid", dto.getAdId() != null ? String.valueOf(dto.getAdId()) : "");
                     cidFeatureMap.put("adverid", dto.getAdVerId() != null ? dto.getAdVerId() : "");
                     cidFeatureMap.put("profession", dto.getProfession() != null ? dto.getProfession() : "");
+                    cidFeatureMap.put("category_name", dto.getCategoryName() != null ? dto.getCategoryName() : "");
                     //DNN模型没训练过的cid才不传入广告相关的稀疏特征
                     if (CollectionUtils.isNotEmpty(DnnCidDataHelper.getCidSetV2()) && !DnnCidDataHelper.getCidSetV2().contains(adRankItem.getAdId())) {
                         cidFeatureMap.put("cid", "");
@@ -843,7 +842,6 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             add("user_vid_share_cate1_14d");
             add("user_vid_share_cate2_14d");
             add("user_has_conver_1y");
-            add("user_conver_ad_class");
             add("user_adverid_view_3d");
             add("user_adverid_click_3d");
             add("user_adverid_conver_3d");
@@ -862,7 +860,8 @@ public class RankStrategyBy683 extends RankStrategyBasic {
             add("user_skuid_view_30d");
             add("user_skuid_click_30d");
             add("user_skuid_conver_30d");
-            add("user_layer_class");
+            add("user_conver_ad_class");
+            add("category_name");
         }};
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -145,8 +145,6 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         userFeatureMap.put("user_vid_share_cate1_14d", g2Feature.getOrDefault("cate1_14d", null));
         userFeatureMap.put("user_vid_share_cate2_14d", g2Feature.getOrDefault("cate2_14d", null));
 
-        userFeatureMap.put("user_layer_class", reqFeature.getOrDefault("user_layer_class", null));
-
         Map<String, String> sceneFeatureMap = this.handleSceneFeature(ts);
         long time1 = System.currentTimeMillis();
 
@@ -202,6 +200,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     cidFeatureMap.put("adid", dto.getAdId() != null ? String.valueOf(dto.getAdId()) : "");
                     cidFeatureMap.put("adverid", dto.getAdVerId() != null ? dto.getAdVerId() : "");
                     cidFeatureMap.put("profession", dto.getProfession() != null ? dto.getProfession() : "");
+                    cidFeatureMap.put("category_name", dto.getCategoryName() != null ? dto.getCategoryName() : "");
                     //DNN模型没训练过的cid才不传入广告相关的稀疏特征
                     if (CollectionUtils.isNotEmpty(DnnCidDataHelper.getCidSet()) && !DnnCidDataHelper.getCidSet().contains(adRankItem.getAdId())) {
                         cidFeatureMap.put("cid", "");
@@ -855,7 +854,6 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             add("user_vid_share_cate1_14d");
             add("user_vid_share_cate2_14d");
             add("user_has_conver_1y");
-            add("user_conver_ad_class");
             add("user_adverid_view_3d");
             add("user_adverid_click_3d");
             add("user_adverid_conver_3d");
@@ -874,7 +872,8 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             add("user_skuid_view_30d");
             add("user_skuid_click_30d");
             add("user_skuid_conver_30d");
-            add("user_layer_class");
+            add("user_conver_ad_class");
+            add("category_name");
         }};
     }
 

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/user/UserService.java

@@ -0,0 +1,5 @@
+package com.tzld.piaoquan.ad.engine.service.user;
+
+public interface UserService {
+    String getUserLayerByMid(String mid);
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/user/impl/UserServiceImpl.java

@@ -0,0 +1,42 @@
+package com.tzld.piaoquan.ad.engine.service.user.impl;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import com.tzld.piaoquan.ad.engine.commons.redis.AlgorithmRedisHelper;
+import com.tzld.piaoquan.ad.engine.service.user.UserService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.Map;
+
+@Slf4j
+@Service
+public class UserServiceImpl implements UserService {
+
+    @Autowired
+    protected AlgorithmRedisHelper algRedisHelper;
+
+    @Override
+    public String getUserLayerByMid(String mid) {
+        String key = String.format("ad:engine:mid:layer:%s", mid);
+        String value = algRedisHelper.get(key);
+        if (StringUtils.isEmpty(value)) {
+            return "无曝光";
+        }
+        try {
+            Map<String, String> layerMap = JSON.parseObject(value, new TypeReference<Map<String, String>>() {
+            });
+            String layer = layerMap.getOrDefault("layer", "无曝光");
+            if (StringUtils.equals(layer, "已转化")) {
+                layer = "有转化";
+            }
+            return layer;
+        } catch (Exception e) {
+            log.error("UserServiceImpl getUserLayerByMi error: ", e);
+        }
+
+        return "无曝光";
+    }
+}