Merge branch 'main' into feature/zhangbo_makedata_v1

src/main/java/examples/dataloader/AdRedisFeatureConstructor.java

@@ -117,7 +117,79 @@ public class AdRedisFeatureConstructor {
         adIdActionFeature3month.setOriginCvr(record.getString("cvr_ad_3month"));
         itemFeature.setMonth3_cnt_features(adIdActionFeature3month);
 
-        // TODO campaignId等维度特征
+        // TODO creativeId等维度特征
+        // creative 维度特征
+        AdActionFeature creativeActionFeature1day = new AdActionFeature();
+        creativeActionFeature1day.setOriginAdView(record.getString("view_creative_1day"));
+        creativeActionFeature1day.setOriginAdClick(record.getString("click_creative_1day"));
+        creativeActionFeature1day.setOriginAdConversion(record.getString("conversion_creative_1day"));
+        creativeActionFeature1day.setOriginCtr(record.getString("ctr_creative_1day"));
+        creativeActionFeature1day.setOriginCvr(record.getString("cvr_creative_1day"));
+        itemFeature.setCreative_1day_cnt_features(creativeActionFeature1day);
+
+        AdActionFeature creativeActionFeature3day = new AdActionFeature();
+        creativeActionFeature3day.setOriginAdView(record.getString("view_creative_3day"));
+        creativeActionFeature3day.setOriginAdClick(record.getString("click_creative_3day"));
+        creativeActionFeature3day.setOriginAdConversion(record.getString("conversion_creative_3day"));
+        creativeActionFeature3day.setOriginCtr(record.getString("ctr_creative_3day"));
+        creativeActionFeature3day.setOriginCvr(record.getString("cvr_creative_3day"));
+        itemFeature.setCreative_3day_cnt_features(creativeActionFeature3day);
+
+        AdActionFeature creativeActionFeature7day = new AdActionFeature();
+        creativeActionFeature7day.setOriginAdView(record.getString("view_creative_7day"));
+        creativeActionFeature7day.setOriginAdClick(record.getString("click_creative_7day"));
+        creativeActionFeature7day.setOriginAdConversion(record.getString("conversion_creative_7day"));
+        creativeActionFeature7day.setOriginCtr(record.getString("ctr_creative_7day"));
+        creativeActionFeature7day.setOriginCvr(record.getString("cvr_creative_7day"));
+        itemFeature.setCreative_7day_cnt_features(creativeActionFeature7day);
+
+
+        AdActionFeature creativeActionFeature3month = new AdActionFeature();
+        creativeActionFeature3month.setOriginAdView(record.getString("view_creative_3month"));
+        creativeActionFeature3month.setOriginAdClick(record.getString("click_creative_3month"));
+        creativeActionFeature3month.setOriginAdConversion(record.getString("conversion_creative_3month"));
+        creativeActionFeature3month.setOriginCtr(record.getString("ctr_creative_3month"));
+        creativeActionFeature3month.setOriginCvr(record.getString("cvr_creative_3month"));
+        itemFeature.setCreative_3month_cnt_features(creativeActionFeature3month);
+
+
+
+        // TODO advertiser维度
+        // advertiser 维度特征
+        AdActionFeature advidActionFeature1day = new AdActionFeature();
+        advidActionFeature1day.setOriginAdView(record.getString("view_advertiser_1day"));
+        advidActionFeature1day.setOriginAdClick(record.getString("click_advertiser_1day"));
+        advidActionFeature1day.setOriginAdConversion(record.getString("conversion_advertiser_1day"));
+        advidActionFeature1day.setOriginCtr(record.getString("ctr_advertiser_1day"));
+        advidActionFeature1day.setOriginCvr(record.getString("cvr_advertiser_1day"));
+        itemFeature.setAdvertiser_1day_cnt_features(advidActionFeature1day);
+
+        AdActionFeature advidActionFeature3day = new AdActionFeature();
+        advidActionFeature3day.setOriginAdView(record.getString("view_advertiser_3day"));
+        advidActionFeature3day.setOriginAdClick(record.getString("click_advertiser_3day"));
+        advidActionFeature3day.setOriginAdConversion(record.getString("conversion_advertiser_3day"));
+        advidActionFeature3day.setOriginCtr(record.getString("ctr_advertiser_3day"));
+        advidActionFeature3day.setOriginCvr(record.getString("cvr_advertiser_3day"));
+        itemFeature.setAdvertiser_3day_cnt_features(advidActionFeature3day);
+
+        AdActionFeature advidActionFeature7day = new AdActionFeature();
+        advidActionFeature7day.setOriginAdView(record.getString("view_advertiser_7day"));
+        advidActionFeature7day.setOriginAdClick(record.getString("click_advertiser_7day"));
+        advidActionFeature7day.setOriginAdConversion(record.getString("conversion_advertiser_7day"));
+        advidActionFeature7day.setOriginCtr(record.getString("ctr_advertiser_7day"));
+        advidActionFeature7day.setOriginCvr(record.getString("cvr_advertiser_7day"));
+        itemFeature.setAdvertiser_7day_cnt_features(advidActionFeature7day);
+
+
+        AdActionFeature advidActionFeature3month = new AdActionFeature();
+        advidActionFeature3month.setOriginAdView(record.getString("view_advertiser_3month"));
+        advidActionFeature3month.setOriginAdClick(record.getString("click_advertiser_3month"));
+        advidActionFeature3month.setOriginAdConversion(record.getString("conversion_advertiser_3month"));
+        advidActionFeature3month.setOriginCtr(record.getString("ctr_advertiser_3month"));
+        advidActionFeature3month.setOriginCvr(record.getString("cvr_advertiser_3month"));
+        itemFeature.setAdvertiser_3month_cnt_features(advidActionFeature3month);
+
+
 
         return itemFeature;
     }

src/main/java/examples/dataloader/AdSampleConstructor.java

@@ -30,6 +30,12 @@ public class AdSampleConstructor {
     public static AdRequestContext constructRequestContext(Record record) {
         AdRequestContext requestContext = new AdRequestContext();
         requestContext.setApptype(record.getString("apptype"));
+        requestContext.setMachineinfoBrand(record.getString("machineinfo_brand"));
+        requestContext.setMachineinfoModel(record.getString("machineinfo_model"));
+        requestContext.setMachineinfoSdkversion(record.getString("machineinfo_sdkversion"));
+        requestContext.setMachineinfoWchatversion(record.getString("machineinfo_wechatversion"));
+
+
         requestContext.setDay(record.getString("ctx_day"));
         requestContext.setWeek(record.getString("ctx_week"));
         requestContext.setHour(record.getString("ctx_hour"));
@@ -42,11 +48,6 @@ public class AdSampleConstructor {
     public static UserAdFeature constructUserFeature(Record record) {
         UserAdFeature userFeature = new UserAdFeature();
         userFeature.setMid(record.get("machinecode").toString());
-        userFeature.setMachineinfo_brand(record.getString("machineinfo_brand"));
-        userFeature.setMachineinfo_model(record.getString("machineinfo_model"));
-        userFeature.setMachineinfo_sdkversion(record.getString("machineinfo_sdkversion"));
-        userFeature.setMachineinfo_brand(record.getString("machineinfo_wechatversion"));
-
 
         // 1day features
         AdActionFeature user1dayActionFeature = new AdActionFeature();
@@ -137,42 +138,41 @@ public class AdSampleConstructor {
         itemFeature.setMonth3_cnt_features(user3MonthActionFeature);
 
 
-        // CREATIVE 维度
-        // 1day features
+        //TODO  CREATIVE 维度  需要在样本中补齐
         AdActionFeature creative1dayFeature = new AdActionFeature();
-        creative1dayFeature.setAdView(record.getString("ad_view_1day"));
-        creative1dayFeature.setAdClick(record.getString("ad_click_1day"));
-        creative1dayFeature.setAdConversion(record.getString("ad_conversion_1day"));
-        creative1dayFeature.setCtr(record.getString("ad_ctr_1day"));
-        creative1dayFeature.setCvr(record.getString("ad_cvr_1day"));
+        creative1dayFeature.setAdView(record.getString("view_creative_1day"));
+        creative1dayFeature.setAdClick(record.getString("click_creative_1day"));
+        creative1dayFeature.setAdConversion(record.getString("conversion_creative_1day"));
+        creative1dayFeature.setCtr(record.getString("ctr_creative_1day"));
+        creative1dayFeature.setCvr(record.getString("cvr_creative_1day"));
         itemFeature.setCreative_1day_cnt_features(creative1dayFeature);
 
         // 3day features
         AdActionFeature creative3dayFeature = new AdActionFeature();
-        creative3dayFeature.setAdView(record.getString("ad_view_3day"));
-        creative3dayFeature.setAdClick(record.getString("ad_click_3day"));
-        creative3dayFeature.setAdConversion(record.getString("ad_conversion_3day"));
-        creative3dayFeature.setCtr(record.getString("ad_ctr_3day"));
-        creative3dayFeature.setCvr(record.getString("ad_cvr_3day"));
+        creative3dayFeature.setAdView(record.getString("view_creative_3day"));
+        creative3dayFeature.setAdClick(record.getString("click_creative_3day"));
+        creative3dayFeature.setAdConversion(record.getString("conversion_creative_3day"));
+        creative3dayFeature.setCtr(record.getString("ctr_creative_3day"));
+        creative3dayFeature.setCvr(record.getString("cvr_creative_3day"));
         itemFeature.setCreative_3day_cnt_features(creative3dayFeature);
 
 
         // 7day features
         AdActionFeature creative7dayFeature = new AdActionFeature();
-        creative7dayFeature.setAdView(record.getString("ad_view_7day"));
-        creative7dayFeature.setAdClick(record.getString("ad_click_7day"));
-        creative7dayFeature.setAdConversion(record.getString("ad_conversion_7day"));
-        creative7dayFeature.setCtr(record.getString("ad_ctr_7day"));
-        creative7dayFeature.setCvr(record.getString("ad_cvr_7day"));
+        creative7dayFeature.setAdView(record.getString("view_creative_7day"));
+        creative7dayFeature.setAdClick(record.getString("click_creative_7day"));
+        creative7dayFeature.setAdConversion(record.getString("conversion_creative_7day"));
+        creative7dayFeature.setCtr(record.getString("ctr_creative_7day"));
+        creative7dayFeature.setCvr(record.getString("cvr_creative_7day"));
         itemFeature.setCreative_7day_cnt_features(creative7dayFeature);
 
         // 3month features
         AdActionFeature creative3MonthFeature = new AdActionFeature();
-        creative3MonthFeature.setAdView(record.getString("ad_view_3month"));
-        creative3MonthFeature.setAdClick(record.getString("ad_click_3month"));
-        creative3MonthFeature.setAdConversion(record.getString("ad_conversion_3month"));
-        creative3MonthFeature.setCtr(record.getString("ad_ctr_3month"));
-        creative3MonthFeature.setCvr(record.getString("ad_cvr_3month"));
+        creative3MonthFeature.setAdView(record.getString("view_creative_3month"));
+        creative3MonthFeature.setAdClick(record.getString("click_creative_3month"));
+        creative3MonthFeature.setAdConversion(record.getString("conversion_creative_3month"));
+        creative3MonthFeature.setCtr(record.getString("ctr_creative_3month"));
+        creative3MonthFeature.setCvr(record.getString("cvr_creative_3month"));
         itemFeature.setCreative_3month_cnt_features(creative3MonthFeature);
 
 

src/main/java/examples/sparksql/SparkAdCTRSampleLoader.java

@@ -70,7 +70,7 @@ public class SparkAdCTRSampleLoader {
         // 转化成bytes
         AdRequestContextBytesFeature adRequestContextBytesFeature = new AdRequestContextBytesFeature(requestContext);
         UserAdBytesFeature userBytesFeature = new UserAdBytesFeature(userFeature);
-        AdItemBytesFeature  adItemBytesFeature = new AdItemBytesFeature(itemFeature);
+        AdItemBytesFeature adItemBytesFeature = new AdItemBytesFeature(itemFeature);
 
         // 特征抽取
         VlogAdCtrLRFeatureExtractor bytesFeatureExtractor;
@@ -79,6 +79,7 @@ public class SparkAdCTRSampleLoader {
         bytesFeatureExtractor.getUserFeatures(userBytesFeature);
         bytesFeatureExtractor.getItemFeature(adItemBytesFeature);
         bytesFeatureExtractor.getContextFeatures(adRequestContextBytesFeature);
+        bytesFeatureExtractor.getCrossFeature(adItemBytesFeature, adRequestContextBytesFeature, userBytesFeature);
 
         ListMultimap<FeatureGroup, BaseFeature> featureMap = bytesFeatureExtractor.getFeatures();
         return parseSamplesToString(label, featureMap);

src/main/java/examples/sparksql/SparkAdFeaToRedisLoader.java

@@ -53,7 +53,6 @@ public class SparkAdFeaToRedisLoader {
     }
 
 
-
     static class RecordsToAdRedisKV implements Function2<Record, TableSchema, List<String>> {
         @Override
         public List<String> call(Record record, TableSchema schema) throws Exception {
@@ -73,9 +72,10 @@ public class SparkAdFeaToRedisLoader {
         @Override
         public List<String> call(Record record, TableSchema schema) throws Exception {
             UserAdFeature userFeature = AdRedisFeatureConstructor.constructUserFeature(record);
+            List<String> kv = new ArrayList<String>();
             String key = String.format(userKeyFormat, userFeature.getKey());
+
             String value = userFeature.getValue();
-            List<String> kv = new ArrayList<String>();
             kv.add(key);
             kv.add(value);
             return kv;