Merge branch 'dev-xym-add-feature' of algorithm/ad-engine into master

ad-engine-commons/pom.xml

@@ -11,7 +11,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>ad-engine-commons</artifactId>
-    <version>1.1.0</version>
+    <version>1.1.1</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
@@ -26,7 +26,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.1.21</version>
+            <version>1.1.22</version>
         </dependency>
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/dto/AdPlatformCreativeDTO.java

@@ -44,6 +44,8 @@ public class AdPlatformCreativeDTO {
     // 召回源，记录召回用
     private Object recallSources;
 
-    //所属行业
+    //广告主行业
     private String profession;
+
+    private Long skuId;
 }

ad-engine-commons/src/main/java/com/tzld/piaoquan/ad/engine/commons/score/model/PAIModelV1.java

@@ -55,7 +55,13 @@ public class PAIModelV1 {
     };
 
     private static final String[] sparseAdLongFeatures = {
-            "cid", "adid", "adverid"
+            "cid", "adid", "adverid",
+            "user_adverid_view_3d", "user_adverid_view_7d", "user_adverid_view_30d",
+            "user_adverid_click_3d", "user_adverid_click_7d", "user_adverid_click_30d",
+            "user_adverid_conver_3d", "user_adverid_conver_7d", "user_adverid_conver_30d",
+            "user_skuid_view_3d", "user_skuid_view_7d", "user_skuid_view_30d",
+            "user_skuid_click_3d", "user_skuid_click_7d", "user_skuid_click_30d",
+            "user_skuid_conver_3d", "user_skuid_conver_7d", "user_skuid_conver_30d"
     };
 
     private static final String[] sparseAdStrFeatures = {
@@ -124,27 +130,23 @@ public class PAIModelV1 {
             TFRequest request = new TFRequest();
 
             for (String feature : sparseUserStrFeatures) {
-                String key = feature.replace("_x_", "*").replace("_view", "(view)");
-                String v = userFeatureMap.getOrDefault(key, "");
+                String v = userFeatureMap.getOrDefault(feature, "");
                 request.addFeed(feature, TFDataType.DT_STRING, new long[]{1}, new String[]{v});
             }
 
             for (String feature : sparseUserLongFeatures) {
-                String key = feature.replace("_x_", "*").replace("_view", "(view)");
-                long v = NumberUtils.toLong(userFeatureMap.getOrDefault(key, "0"), 0);
+                long v = NumberUtils.toLong(userFeatureMap.getOrDefault(feature, "0"), 0);
                 request.addFeed(feature, TFDataType.DT_INT64, new long[]{1}, new long[]{v});
             }
 
             for (String feature : sparseSceneLongFeatures) {
-                String key = feature.replace("_x_", "*").replace("_view", "(view)");
-                long v = NumberUtils.toLong(sceneFeatureMap.getOrDefault(key, "0"), 0);
+                long v = NumberUtils.toLong(sceneFeatureMap.getOrDefault(feature, "0"), 0);
                 request.addFeed(feature, TFDataType.DT_INT64, new long[]{1}, new long[]{v});
             }
 
 
             for (String feature : userFeatures) {
-                String key = feature.replace("_x_", "*").replace("_view", "(view)");
-                double v = NumberUtils.toDouble(userFeatureMap.getOrDefault(key, "0.0"), 0.0);
+                double v = NumberUtils.toDouble(userFeatureMap.getOrDefault(feature, "0.0"), 0.0);
                 request.addFeed(feature.toLowerCase(), TFDataType.DT_DOUBLE, new long[]{1}, new double[]{v});
             }
             Map<String, double[]> doubleFeed = new HashMap<>();
@@ -163,24 +165,22 @@ public class PAIModelV1 {
                 }
 
                 for (String feature : sparseAdLongFeatures) {
-                    String key = feature.replace("_x_", "*").replace("_view", "(view)");
                     long[] longs = longFeed.computeIfAbsent(feature, k -> new long[items.size()]);
                     if (MapUtils.isEmpty(items.get(i).getFeatureMap())) {
                         longs[i] = 0L;
                         continue;
                     }
-                    long v = NumberUtils.toLong(items.get(i).getFeatureMap().getOrDefault(key, "0"), 0L);
+                    long v = NumberUtils.toLong(items.get(i).getFeatureMap().getOrDefault(feature, "0"), 0L);
                     longs[i] = v;
                 }
 
                 for (String feature : sparseAdStrFeatures) {
-                    String key = feature.replace("_x_", "*").replace("_view", "(view)");
                     String[] strs = strFeed.computeIfAbsent(feature, k -> new String[items.size()]);
                     if (MapUtils.isEmpty(items.get(i).getFeatureMap())) {
                         strs[i] = "";
                         continue;
                     }
-                    String v = items.get(i).getFeatureMap().getOrDefault(key, "");
+                    String v = items.get(i).getFeatureMap().getOrDefault(feature, "");
                     strs[i] = v;
                 }
             }

ad-engine-service/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>ad-engine-commons</artifactId>
-            <version>1.1.0</version>
+            <version>1.1.1</version>
         </dependency>
         <dependency>
             <groupId>com.aliyun.odps</groupId>

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/Feature.java

@@ -20,4 +20,7 @@ public class Feature {
     // k1:mid、k2:表、v:特征值
     private Map<String, Map<String, String>> userFeature = new HashMap<>();
 
+    // k1:skuid、k2:表、v:特征值
+    private Map<String, Map<String, Map<String, String>>> skuFeature = new HashMap<>();
+
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/feature/FeatureService.java

@@ -26,8 +26,9 @@ public class FeatureService {
     private static final String avUkFormat = "av:%s:%s";
     private static final String vidUkFormat = "v:%s:%s";
     private static final String midUkFormat = "m:%s:%s";
+    private static final String skuUkFormat = "s:%s:%s";
 
-    public Feature getFeature(Collection<String> cidList, Collection<String> adVerIdList, ScoreParam param) {
+    public Feature getFeature(Collection<String> cidList, Collection<String> adVerIdList, List<Long> skuIdList, ScoreParam param) {
         AdRequestContext context = param.getRequestContext();
 
         List<FeatureKeyProto> protos = new ArrayList<>();
@@ -63,9 +64,19 @@ public class FeatureService {
             }
         }
 
+        //skuid
+        for (Long skuId : skuIdList) {
+            if (StringUtils.isNotEmpty(param.getMid())) {
+                protos.add(genWithMidAndSkuId("alg_mid_feature_sku_action", param.getMid(), String.valueOf(skuId)));
+            }
+        }
+
         for (String adVerId : adVerIdList) {
             // adverid
             protos.add(genWithAdVerId("alg_cid_feature_adver_action", adVerId));
+            if (StringUtils.isNotEmpty(param.getMid())) {
+                protos.add(genWithMidAndAdVerId("alg_mid_feature_adver_action", param.getMid(), adVerId));
+            }
         }
 
         // vid
@@ -126,6 +137,12 @@ public class FeatureService {
                     feature.getVideoFeature().put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
                     }, Collections.emptyMap()));
                     break;
+                case "s":
+                    Map<String, Map<String, String>> skuFeatureMap = feature.getSkuFeature().getOrDefault(id, new HashMap<>());
+                    skuFeatureMap.put(tableName, JSONUtils.fromJson(value, new TypeToken<Map<String, String>>() {
+                    }, Collections.emptyMap()));
+                    feature.getSkuFeature().put(id, skuFeatureMap);
+                    break;
             }
         }
 
@@ -148,6 +165,24 @@ public class FeatureService {
                 .build();
     }
 
+    private FeatureKeyProto genWithMidAndAdVerId(String table, String mid, String adVerId) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(avUkFormat, table, adVerId))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .putFieldValue("adverid", adVerId)
+                .build();
+    }
+
+    private FeatureKeyProto genWithMidAndSkuId(String table, String mid, String skuId) {
+        return FeatureKeyProto.newBuilder()
+                .setUniqueKey(String.format(skuUkFormat, table, skuId))
+                .setTableName(table)
+                .putFieldValue("mid", mid)
+                .putFieldValue("skuid", skuId)
+                .build();
+    }
+
     private FeatureKeyProto genWithCidAndRegion(String table, String cid, String region) {
         return FeatureKeyProto.newBuilder()
                 .setUniqueKey(String.format(cidUkDoubleFieldFormat, table, cid, region))

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBasic.java

@@ -68,7 +68,13 @@ public abstract class RankStrategyBasic implements RankStrategy {
                 .filter(StringUtils::isNotBlank)
                 .distinct()
                 .collect(Collectors.toList());
-        return featureService.getFeature(cidList, adVerIdList, param);
+
+        List<Long> skuIdList = adIdList.stream()
+                .map(AdPlatformCreativeDTO::getSkuId)
+                .filter(Objects::nonNull)
+                .distinct()
+                .collect(Collectors.toList());
+        return featureService.getFeature(cidList, adVerIdList, skuIdList, param);
     }
 
     protected Set<String> getNoApiAdVerIds() {

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy679.java

@@ -71,6 +71,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
         Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
 
 
@@ -108,6 +109,7 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                     adRankItem.setId(dto.getAdId());
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -254,6 +256,13 @@ public class RankStrategyBy679 extends RankStrategyBasic {
                     item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
+
+            Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(item.getSkuId()), new HashMap<>());
+            for (Map.Entry<String, Map<String, String>> entry : skuFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
             item.getMetaFeatureMap().put("reqFeature", reqFeature);
             item.getMetaFeatureMap().put("sceneFeature", sceneFeatureMap);
         }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy680.java

@@ -79,6 +79,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
         Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
 
 
@@ -117,6 +118,7 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                     adRankItem.setId(dto.getAdId());
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -278,6 +280,13 @@ public class RankStrategyBy680 extends RankStrategyBasic {
                     top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
+
+            Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(top1Item.getSkuId()), new HashMap<>());
+            for (Map.Entry<String, Map<String, String>> entry : skuFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
             top1Item.getMetaFeatureMap().put("reqFeature", reqFeature);
             top1Item.getMetaFeatureMap().put("sceneFeature", sceneFeatureMap);
         }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy683.java

@@ -80,6 +80,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
         Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
 
 
@@ -118,6 +119,7 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     adRankItem.setId(dto.getAdId());
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -279,6 +281,13 @@ public class RankStrategyBy683 extends RankStrategyBasic {
                     top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
+
+            Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(top1Item.getSkuId()), new HashMap<>());
+            for (Map.Entry<String, Map<String, String>> entry : skuFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
             top1Item.getMetaFeatureMap().put("reqFeature", reqFeature);
             top1Item.getMetaFeatureMap().put("sceneFeature", sceneFeatureMap);
         }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy687.java

@@ -56,6 +56,7 @@ public class RankStrategyBy687 extends RankStrategyBasic {
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
         Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
 
 
@@ -165,6 +166,7 @@ public class RankStrategyBy687 extends RankStrategyBasic {
                 adRankItem.setId(dto.getAdId());
                 adRankItem.setCampaignId(dto.getCampaignId());
                 adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                adRankItem.setSkuId(dto.getSkuId());
                 adRankItem.setRandom(random.nextInt(1000));
 
                 String cidStr = dto.getCreativeId().toString();
@@ -260,6 +262,13 @@ public class RankStrategyBy687 extends RankStrategyBasic {
                     item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
+
+            Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(item.getSkuId()), new HashMap<>());
+            for (Map.Entry<String, Map<String, String>> entry : skuFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
             item.getMetaFeatureMap().put("reqFeature", reqFeature);
             item.getMetaFeatureMap().put("sceneFeature", sceneFeatureMap);
         }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/strategy/RankStrategyBy688.java

@@ -89,6 +89,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         Map<String, Map<String, String>> videoFeature = feature.getVideoFeature();
         Map<String, Map<String, Map<String, String>>> allAdVerFeature = feature.getAdVerFeature();
         Map<String, Map<String, Map<String, String>>> allCidFeature = feature.getCidFeature();
+        Map<String, Map<String, Map<String, String>>> allSkuFeature = feature.getSkuFeature();
         Map<String, String> reqFeature = this.getReqFeature(scoreParam, request);
 
         Map<String, String> userFeatureMap = new HashMap<>();
@@ -156,6 +157,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     adRankItem.setId(dto.getAdId());
                     adRankItem.setCampaignId(dto.getCampaignId());
                     adRankItem.setCpm(ObjUtil.nullOrDefault(dto.getCpm(), 90).doubleValue());
+                    adRankItem.setSkuId(dto.getSkuId());
                     adRankItem.setRandom(random.nextInt(1000));
                     if (noApiAdVerIds.contains(dto.getAdVerId())) {
                         adRankItem.getExt().put("isApi", "0");
@@ -170,7 +172,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     Map<String, String> b1Feature = cidFeature.getOrDefault("alg_cid_feature_basic_info", new HashMap<>());
 
                     Map<String, Map<String, String>> adVerFeature = allAdVerFeature.getOrDefault(dto.getAdVerId(), new HashMap<>());
-
+                    Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(dto.getSkuId()), new HashMap<>());
                     Map<String, String> d1Feature = cidFeature.getOrDefault("alg_cid_feature_vid_cf", new HashMap<>());
 
                     this.handleB1Feature(b1Feature, cidFeatureMap, cidStr);
@@ -179,6 +181,7 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     this.handleC1UIFeature(midTimeDiffMap, actionStaticMap, cidFeatureMap, cidStr);
                     this.handleD1Feature(d1Feature, cidFeatureMap);
                     this.handleD2Feature(vidRankMaps, cidFeatureMap, cidStr);
+                    this.handleH1AndH2Feature(skuFeature, adVerFeature, cidFeatureMap);
 
                     cidFeatureMap.put("cid", dto.getCreativeId() != null ? String.valueOf(dto.getCreativeId()) : "");
                     cidFeatureMap.put("adid", dto.getAdId() != null ? String.valueOf(dto.getAdId()) : "");
@@ -321,6 +324,13 @@ public class RankStrategyBy688 extends RankStrategyBasic {
                     top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
                 }
             }
+
+            Map<String, Map<String, String>> skuFeature = allSkuFeature.getOrDefault(String.valueOf(top1Item.getSkuId()), new HashMap<>());
+            for (Map.Entry<String, Map<String, String>> entry : skuFeature.entrySet()) {
+                if (MapUtils.isNotEmpty(entry.getValue())) {
+                    top1Item.getMetaFeatureMap().put(entry.getKey(), entry.getValue());
+                }
+            }
             top1Item.getMetaFeatureMap().put("reqFeature", reqFeature);
             top1Item.getMetaFeatureMap().put("sceneFeature", sceneFeatureMap);
         }
@@ -557,6 +567,33 @@ public class RankStrategyBy688 extends RankStrategyBasic {
         }
     }
 
+    private void handleH1AndH2Feature(Map<String, Map<String, String>> skuFeature,
+                                      Map<String, Map<String, String>> adVerFeature,
+                                      Map<String, String> cidFeatureMap) {
+        Map<String, String> h1Feature = adVerFeature.getOrDefault("alg_mid_feature_adver_action", new HashMap<>());
+        Map<String, String> h2Feature = skuFeature.getOrDefault("alg_mid_feature_sku_action", new HashMap<>());
+        List<String> timeList = Arrays.asList("3d", "7d", "30d");
+        List<Tuple2<Map<String, String>, String>> featureList = Arrays.asList(
+                new Tuple2<>(h1Feature, "adverid"),
+                new Tuple2<>(h2Feature, "skuid")
+        );
+        for (Tuple2<Map<String, String>, String> tuple2 : featureList) {
+            Map<String, String> feature = tuple2.f1;
+            String prefix = tuple2.f2;
+            for (String time : timeList) {
+                String timeValue = feature.getOrDefault(time, "");
+                if (StringUtils.isNotEmpty(timeValue)) {
+                    String[] split = timeValue.split(",");
+                    cidFeatureMap.put("user" + "_" + prefix + "_" + "view" + "_" + time, split[0]);
+                    cidFeatureMap.put("user" + "_" + prefix + "_" + "click" + "_" + time, split[1]);
+                    cidFeatureMap.put("user" + "_" + prefix + "_" + "conver" + "_" + time, split[2]);
+                }
+            }
+        }
+
+
+    }
+
     private void handleD3AndB1Feature(Map<String, String> d3Feature, String cTitle, Map<String, String> featureMap,
                                       ScoreParam scoreParam) {
         if (MapUtils.isEmpty(d3Feature) || !d3Feature.containsKey("title") || StringUtils.isEmpty(cTitle)) {
@@ -733,6 +770,24 @@ public class RankStrategyBy688 extends RankStrategyBasic {
             add("user_vid_share_cate1_14d");
             add("user_vid_share_cate2_14d");
             add("user_has_conver_1y");
+            add("user_adverid_view_3d");
+            add("user_adverid_click_3d");
+            add("user_adverid_conver_3d");
+            add("user_adverid_view_7d");
+            add("user_adverid_click_7d");
+            add("user_adverid_conver_7d");
+            add("user_adverid_view_30d");
+            add("user_adverid_click_30d");
+            add("user_adverid_conver_30d");
+            add("user_skuid_view_3d");
+            add("user_skuid_click_3d");
+            add("user_skuid_conver_3d");
+            add("user_skuid_view_7d");
+            add("user_skuid_click_7d");
+            add("user_skuid_conver_7d");
+            add("user_skuid_view_30d");
+            add("user_skuid_click_30d");
+            add("user_skuid_conver_30d");
         }};
     }