增加onlinefeature训练0116

pom.xml

@@ -48,14 +48,14 @@
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>recommend-feature-client</artifactId>
-            <version>1.0.9</version>
+            <version>1.1.15</version>
         </dependency>
 
 
         <dependency>
             <groupId>com.tzld.piaoquan</groupId>
             <artifactId>ad-engine-commons</artifactId>
-            <version>1.0.3</version>
+            <version>1.1.0</version>
         </dependency>
 
 

src/main/java/examples/dataloader/AdSampleConstructor.java

@@ -51,39 +51,39 @@ public class AdSampleConstructor {
 
         // 1day features
         AdActionFeature user1dayActionFeature = new AdActionFeature();
-        user1dayActionFeature.setOriginAdView(record.getString("user_view_1day"));
-        user1dayActionFeature.setOriginAdClick(record.getString("user_click_1day"));
-        user1dayActionFeature.setOriginAdConversion(record.getString("user_conversion_1day"));
-        user1dayActionFeature.setOriginCtr(record.getString("user_ctr_1day"));
-        user1dayActionFeature.setOriginCvr(record.getString("user_cvr_1day"));
+        user1dayActionFeature.setAdView(record.getString("user_view_1day"));
+        user1dayActionFeature.setAdClick(record.getString("user_click_1day"));
+        user1dayActionFeature.setAdConversion(record.getString("user_conversion_1day"));
+        user1dayActionFeature.setCtr(record.getString("user_ctr_1day"));
+        user1dayActionFeature.setCvr(record.getString("user_cvr_1day"));
         userFeature.setDay1_cnt_features(user1dayActionFeature);
 
         // 3day features
         AdActionFeature user3dayActionFeature = new AdActionFeature();
-        user3dayActionFeature.setOriginAdView(record.getString("user_view_3day"));
-        user3dayActionFeature.setOriginAdClick(record.getString("user_click_3day"));
-        user3dayActionFeature.setOriginAdConversion(record.getString("user_conversion_3day"));
-        user3dayActionFeature.setOriginCtr(record.getString("user_ctr_3day"));
-        user3dayActionFeature.setOriginCvr(record.getString("user_cvr_3day"));
+        user3dayActionFeature.setAdView(record.getString("user_view_3day"));
+        user3dayActionFeature.setAdClick(record.getString("user_click_3day"));
+        user3dayActionFeature.setAdConversion(record.getString("user_conversion_3day"));
+        user3dayActionFeature.setCtr(record.getString("user_ctr_3day"));
+        user3dayActionFeature.setCvr(record.getString("user_cvr_3day"));
         userFeature.setDay3_cnt_features(user3dayActionFeature);
 
 
         // 7day features
         AdActionFeature user7dayActionFeature = new AdActionFeature();
-        user7dayActionFeature.setOriginAdView(record.getString("user_view_7day"));
-        user7dayActionFeature.setOriginAdClick(record.getString("user_click7day"));
-        user7dayActionFeature.setOriginAdConversion(record.getString("user_conversion_7day"));
-        user7dayActionFeature.setOriginCtr(record.getString("user_ctr_7day"));
-        user7dayActionFeature.setOriginCvr(record.getString("user_cvr_7day"));
+        user7dayActionFeature.setAdView(record.getString("user_view_7day"));
+        user7dayActionFeature.setAdClick(record.getString("user_click7day"));
+        user7dayActionFeature.setAdConversion(record.getString("user_conversion_7day"));
+        user7dayActionFeature.setCtr(record.getString("user_ctr_7day"));
+        user7dayActionFeature.setCvr(record.getString("user_cvr_7day"));
         userFeature.setDay7_cnt_features(user7dayActionFeature);
 
         // 3month features
         AdActionFeature user3MonthActionFeature = new AdActionFeature();
-        user3MonthActionFeature.setOriginAdView(record.getString("user_view_3month"));
-        user3MonthActionFeature.setOriginAdClick(record.getString("user_click_3month"));
-        user3MonthActionFeature.setOriginAdConversion(record.getString("user_conversion_3month"));
-        user3MonthActionFeature.setOriginCtr(record.getString("user_ctr_3month"));
-        user3MonthActionFeature.setOriginCvr(record.getString("user_cvr_3month"));
+        user3MonthActionFeature.setAdView(record.getString("user_view_3month"));
+        user3MonthActionFeature.setAdClick(record.getString("user_click_3month"));
+        user3MonthActionFeature.setAdConversion(record.getString("user_conversion_3month"));
+        user3MonthActionFeature.setCtr(record.getString("user_ctr_3month"));
+        user3MonthActionFeature.setCvr(record.getString("user_cvr_3month"));
         userFeature.setMonth3_cnt_features(user3MonthActionFeature);
 
         return userFeature;
@@ -102,116 +102,116 @@ public class AdSampleConstructor {
 
         // 1day features
         AdActionFeature user1dayActionFeature = new AdActionFeature();
-        user1dayActionFeature.setOriginAdView(record.getString("ad_view_1day"));
-        user1dayActionFeature.setOriginAdClick(record.getString("ad_click_1day"));
-        user1dayActionFeature.setOriginAdConversion(record.getString("ad_conversion_1day"));
-        user1dayActionFeature.setOriginCtr(record.getString("ad_ctr_1day"));
-        user1dayActionFeature.setOriginCvr(record.getString("ad_cvr_1day"));
+        user1dayActionFeature.setAdView(record.getString("ad_view_1day"));
+        user1dayActionFeature.setAdClick(record.getString("ad_click_1day"));
+        user1dayActionFeature.setAdConversion(record.getString("ad_conversion_1day"));
+        user1dayActionFeature.setCtr(record.getString("ad_ctr_1day"));
+        user1dayActionFeature.setCvr(record.getString("ad_cvr_1day"));
         itemFeature.setDay1_cnt_features(user1dayActionFeature);
 
         // 3day features
         AdActionFeature user3dayActionFeature = new AdActionFeature();
-        user3dayActionFeature.setOriginAdView(record.getString("ad_view_3day"));
-        user3dayActionFeature.setOriginAdClick(record.getString("ad_click_3day"));
-        user3dayActionFeature.setOriginAdConversion(record.getString("ad_conversion_3day"));
-        user3dayActionFeature.setOriginCtr(record.getString("ad_ctr_3day"));
-        user3dayActionFeature.setOriginCvr(record.getString("ad_cvr_3day"));
+        user3dayActionFeature.setAdView(record.getString("ad_view_3day"));
+        user3dayActionFeature.setAdClick(record.getString("ad_click_3day"));
+        user3dayActionFeature.setAdConversion(record.getString("ad_conversion_3day"));
+        user3dayActionFeature.setCtr(record.getString("ad_ctr_3day"));
+        user3dayActionFeature.setCvr(record.getString("ad_cvr_3day"));
         itemFeature.setDay3_cnt_features(user3dayActionFeature);
 
 
         // 7day features
         AdActionFeature user7dayActionFeature = new AdActionFeature();
-        user7dayActionFeature.setOriginAdView(record.getString("ad_view_7day"));
-        user7dayActionFeature.setOriginAdClick(record.getString("ad_click_7day"));
-        user7dayActionFeature.setOriginAdConversion(record.getString("ad_conversion_7day"));
-        user7dayActionFeature.setOriginCtr(record.getString("ad_ctr_7day"));
-        user7dayActionFeature.setOriginCvr(record.getString("ad_cvr_7day"));
+        user7dayActionFeature.setAdView(record.getString("ad_view_7day"));
+        user7dayActionFeature.setAdClick(record.getString("ad_click_7day"));
+        user7dayActionFeature.setAdConversion(record.getString("ad_conversion_7day"));
+        user7dayActionFeature.setCtr(record.getString("ad_ctr_7day"));
+        user7dayActionFeature.setCvr(record.getString("ad_cvr_7day"));
         itemFeature.setDay7_cnt_features(user7dayActionFeature);
 
         // 3month features
         AdActionFeature user3MonthActionFeature = new AdActionFeature();
-        user3MonthActionFeature.setOriginAdView(record.getString("ad_view_3month"));
-        user3MonthActionFeature.setOriginAdClick(record.getString("ad_click_3month"));
-        user3MonthActionFeature.setOriginAdConversion(record.getString("ad_conversion_3month"));
-        user3MonthActionFeature.setOriginCtr(record.getString("ad_ctr_3month"));
-        user3MonthActionFeature.setOriginCvr(record.getString("ad_cvr_3month"));
+        user3MonthActionFeature.setAdView(record.getString("ad_view_3month"));
+        user3MonthActionFeature.setAdClick(record.getString("ad_click_3month"));
+        user3MonthActionFeature.setAdConversion(record.getString("ad_conversion_3month"));
+        user3MonthActionFeature.setCtr(record.getString("ad_ctr_3month"));
+        user3MonthActionFeature.setCvr(record.getString("ad_cvr_3month"));
         itemFeature.setMonth3_cnt_features(user3MonthActionFeature);
 
 
         //TODO  CREATIVE 维度  需要在样本中补齐
         AdActionFeature creative1dayFeature = new AdActionFeature();
-        creative1dayFeature.setOriginAdView(record.getString("view_creative_1day"));
-        creative1dayFeature.setOriginAdClick(record.getString("click_creative_1day"));
-        creative1dayFeature.setOriginAdConversion(record.getString("conversion_creative_1day"));
-        creative1dayFeature.setOriginCtr(record.getString("ctr_creative_1day"));
-        creative1dayFeature.setOriginCvr(record.getString("cvr_creative_1day"));
+        creative1dayFeature.setAdView(record.getString("view_creative_1day"));
+        creative1dayFeature.setAdClick(record.getString("click_creative_1day"));
+        creative1dayFeature.setAdConversion(record.getString("conversion_creative_1day"));
+        creative1dayFeature.setCtr(record.getString("ctr_creative_1day"));
+        creative1dayFeature.setCvr(record.getString("cvr_creative_1day"));
         itemFeature.setCreative_1day_cnt_features(creative1dayFeature);
 
         // 3day features
         AdActionFeature creative3dayFeature = new AdActionFeature();
-        creative3dayFeature.setOriginAdView(record.getString("view_creative_3day"));
-        creative3dayFeature.setOriginAdClick(record.getString("click_creative_3day"));
-        creative3dayFeature.setOriginAdConversion(record.getString("conversion_creative_3day"));
-        creative3dayFeature.setOriginCtr(record.getString("ctr_creative_3day"));
-        creative3dayFeature.setOriginCvr(record.getString("cvr_creative_3day"));
+        creative3dayFeature.setAdView(record.getString("view_creative_3day"));
+        creative3dayFeature.setAdClick(record.getString("click_creative_3day"));
+        creative3dayFeature.setAdConversion(record.getString("conversion_creative_3day"));
+        creative3dayFeature.setCtr(record.getString("ctr_creative_3day"));
+        creative3dayFeature.setCvr(record.getString("cvr_creative_3day"));
         itemFeature.setCreative_3day_cnt_features(creative3dayFeature);
 
 
         // 7day features
         AdActionFeature creative7dayFeature = new AdActionFeature();
-        creative7dayFeature.setOriginAdView(record.getString("view_creative_7day"));
-        creative7dayFeature.setOriginAdClick(record.getString("click_creative_7day"));
-        creative7dayFeature.setOriginAdConversion(record.getString("conversion_creative_7day"));
-        creative7dayFeature.setOriginCtr(record.getString("ctr_creative_7day"));
-        creative7dayFeature.setOriginCvr(record.getString("cvr_creative_7day"));
+        creative7dayFeature.setAdView(record.getString("view_creative_7day"));
+        creative7dayFeature.setAdClick(record.getString("click_creative_7day"));
+        creative7dayFeature.setAdConversion(record.getString("conversion_creative_7day"));
+        creative7dayFeature.setCtr(record.getString("ctr_creative_7day"));
+        creative7dayFeature.setCvr(record.getString("cvr_creative_7day"));
         itemFeature.setCreative_7day_cnt_features(creative7dayFeature);
 
         // 3month features
         AdActionFeature creative3MonthFeature = new AdActionFeature();
-        creative3MonthFeature.setOriginAdView(record.getString("view_creative_3month"));
-        creative3MonthFeature.setOriginAdClick(record.getString("click_creative_3month"));
-        creative3MonthFeature.setOriginAdConversion(record.getString("conversion_creative_3month"));
-        creative3MonthFeature.setOriginCtr(record.getString("ctr_creative_3month"));
-        creative3MonthFeature.setOriginCvr(record.getString("cvr_creative_3month"));
+        creative3MonthFeature.setAdView(record.getString("view_creative_3month"));
+        creative3MonthFeature.setAdClick(record.getString("click_creative_3month"));
+        creative3MonthFeature.setAdConversion(record.getString("conversion_creative_3month"));
+        creative3MonthFeature.setCtr(record.getString("ctr_creative_3month"));
+        creative3MonthFeature.setCvr(record.getString("cvr_creative_3month"));
         itemFeature.setCreative_3month_cnt_features(creative3MonthFeature);
 
 
         // advertiser id
         // 1day features
         AdActionFeature advertiser1dayFeature = new AdActionFeature();
-        advertiser1dayFeature.setOriginAdView(record.getString("advertiser_view_1day"));
-        advertiser1dayFeature.setOriginAdClick(record.getString("advertiser_click_1day"));
-        advertiser1dayFeature.setOriginAdConversion(record.getString("advertiser_conversion_1day"));
-        advertiser1dayFeature.setOriginCtr(record.getString("advertiser_ctr_1day"));
-        advertiser1dayFeature.setOriginCvr(record.getString("advertiser_cvr_1day"));
+        advertiser1dayFeature.setAdView(record.getString("advertiser_view_1day"));
+        advertiser1dayFeature.setAdClick(record.getString("advertiser_click_1day"));
+        advertiser1dayFeature.setAdConversion(record.getString("advertiser_conversion_1day"));
+        advertiser1dayFeature.setCtr(record.getString("advertiser_ctr_1day"));
+        advertiser1dayFeature.setCvr(record.getString("advertiser_cvr_1day"));
         itemFeature.setAdvertiser_1day_cnt_features(advertiser1dayFeature);
 
         // 3day features
         AdActionFeature advertiser3dayFeature = new AdActionFeature();
-        advertiser3dayFeature.setOriginAdView(record.getString("advertiser_view_3day"));
-        advertiser3dayFeature.setOriginAdClick(record.getString("advertiser_click_3day"));
-        advertiser3dayFeature.setOriginAdConversion(record.getString("advertiser_conversion_3day"));
-        advertiser3dayFeature.setOriginCtr(record.getString("advertiser_ctr_3day"));
-        advertiser3dayFeature.setOriginCvr(record.getString("advertiser_cvr_3day"));
+        advertiser3dayFeature.setAdView(record.getString("advertiser_view_3day"));
+        advertiser3dayFeature.setAdClick(record.getString("advertiser_click_3day"));
+        advertiser3dayFeature.setAdConversion(record.getString("advertiser_conversion_3day"));
+        advertiser3dayFeature.setCtr(record.getString("advertiser_ctr_3day"));
+        advertiser3dayFeature.setCvr(record.getString("advertiser_cvr_3day"));
         itemFeature.setAdvertiser_3day_cnt_features(advertiser3dayFeature);
 
 
         // 7day features
         AdActionFeature advertiser7dayFeature = new AdActionFeature();
-        advertiser7dayFeature.setOriginAdView(record.getString("advertiser_view_7day"));
-        advertiser7dayFeature.setOriginAdClick(record.getString("advertiser_click_7day"));
-        advertiser7dayFeature.setOriginAdConversion(record.getString("advertiser_conversion_7day"));
-        advertiser7dayFeature.setOriginCtr(record.getString("advertiser_ctr_7day"));
-        advertiser7dayFeature.setOriginCvr(record.getString("advertiser_cvr_7day"));
+        advertiser7dayFeature.setAdView(record.getString("advertiser_view_7day"));
+        advertiser7dayFeature.setAdClick(record.getString("advertiser_click_7day"));
+        advertiser7dayFeature.setAdConversion(record.getString("advertiser_conversion_7day"));
+        advertiser7dayFeature.setCtr(record.getString("advertiser_ctr_7day"));
+        advertiser7dayFeature.setCvr(record.getString("advertiser_cvr_7day"));
         itemFeature.setAdvertiser_7day_cnt_features(advertiser7dayFeature);
 
         // 3month features
         AdActionFeature advertiser3monthFeature = new AdActionFeature();
-        advertiser3monthFeature.setOriginAdView(record.getString("advertiser_view_3month"));
-        advertiser3monthFeature.setOriginAdClick(record.getString("advertiser_view_3month"));
-        advertiser3monthFeature.setOriginAdConversion(record.getString("advertiser_conversion_3month"));
-        advertiser3monthFeature.setOriginCtr(record.getString("advertiser_ctr_3month"));
-        advertiser3monthFeature.setOriginCvr(record.getString("advertiser_cvr_3month"));
+        advertiser3monthFeature.setAdView(record.getString("advertiser_view_3month"));
+        advertiser3monthFeature.setAdClick(record.getString("advertiser_view_3month"));
+        advertiser3monthFeature.setAdConversion(record.getString("advertiser_conversion_3month"));
+        advertiser3monthFeature.setCtr(record.getString("advertiser_ctr_3month"));
+        advertiser3monthFeature.setCvr(record.getString("advertiser_cvr_3month"));
         itemFeature.setAdvertiser_3month_cnt_features(advertiser3monthFeature);
 
 

src/main/java/examples/sparksql/SparkAdCTRSampleTester.java

@@ -2,12 +2,6 @@ package examples.sparksql;
 
 import com.aliyun.odps.TableSchema;
 import com.aliyun.odps.data.Record;
-import com.tzld.piaoquan.recommend.feature.domain.ad.base.*;
-import com.tzld.piaoquan.recommend.feature.domain.ad.feature.VlogAdCtrLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
-import examples.dataloader.AdSampleConstructor;
 import org.apache.spark.SparkConf;
 import org.apache.spark.aliyun.odps.OdpsOps;
 import org.apache.spark.api.java.JavaRDD;
@@ -38,7 +32,6 @@ public class SparkAdCTRSampleTester {
         JavaRDD<Record> readData = odpsOps.readTableWithJava(project, table, partition, new RecordsToSamples(), Integer.valueOf(30));
         readData.filter(row -> row.get("type") != null)
                 .filter(row -> row.get("lrsample") != null)
-                .filter(row -> row.get("type").equals("VlogAdCtrLRScorer") )
                 .map(line -> singleParse2(line))
                 .saveAsTextFile(hdfsPath);
     }
@@ -52,57 +45,13 @@ public class SparkAdCTRSampleTester {
     }
 
 
-    // 单条日志处理逻辑
-    public static String singleParse(Record record) {
-        // 数据解析
-        String label = record.getString("adclick_ornot");
-
-        // 从sql的 record中 初始化对象内容
-        AdRequestContext requestContext = AdSampleConstructor.constructRequestContext(record);
-        UserAdFeature userFeature = AdSampleConstructor.constructUserFeature(record);
-        AdItemFeature itemFeature = AdSampleConstructor.constructItemFeature(record);
-
-        // 转化成bytes
-        AdRequestContextBytesFeature adRequestContextBytesFeature = new AdRequestContextBytesFeature(requestContext);
-        UserAdBytesFeature userBytesFeature = new UserAdBytesFeature(userFeature);
-        AdItemBytesFeature adItemBytesFeature = new AdItemBytesFeature(itemFeature);
-
-        // 特征抽取
-        VlogAdCtrLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogAdCtrLRFeatureExtractor();
-
-        LRSamples lrSamples = bytesFeatureExtractor.single(userBytesFeature, adItemBytesFeature, adRequestContextBytesFeature);
-
-        return parseSamplesToString2(label, lrSamples);
-    }
-
     // 单条日志处理逻辑
     public static String singleParse2(Record record) {
         // 数据解析
         String label = record.getString("adclick_ornot");
         String samples = record.getString("lrsample");
-
         return label + "\t" +  samples;
     }
 
 
-
-    // 构建样本的字符串
-    public static String parseSamplesToString2(String label, LRSamples lrSamples) {
-        ArrayList<String> featureList = new ArrayList<String>();
-        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
-            GroupedFeature groupedFeature = lrSamples.getFeatures(i);
-            if (groupedFeature != null && groupedFeature.getFeaturesCount() != 0) {
-                for (int j = 0; j < groupedFeature.getFeaturesCount(); j++) {
-                    BaseFeature baseFeature = groupedFeature.getFeatures(j);
-                    if (baseFeature != null) {
-                        featureList.add(String.valueOf(baseFeature.getIdentifier()) + ":1");
-                    }
-                }
-            }
-        }
-        return label + "\t" + String.join("\t", featureList);
-    }
-
-
 }

src/main/java/examples/sparksql/SparkAdCVRSampleTester.java

@@ -2,12 +2,6 @@ package examples.sparksql;
 
 import com.aliyun.odps.TableSchema;
 import com.aliyun.odps.data.Record;
-import com.tzld.piaoquan.recommend.feature.domain.ad.base.*;
-import com.tzld.piaoquan.recommend.feature.domain.ad.feature.VlogAdCtrLRFeatureExtractor;
-import com.tzld.piaoquan.recommend.feature.model.sample.BaseFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.GroupedFeature;
-import com.tzld.piaoquan.recommend.feature.model.sample.LRSamples;
-import examples.dataloader.AdSampleConstructor;
 import org.apache.spark.SparkConf;
 import org.apache.spark.aliyun.odps.OdpsOps;
 import org.apache.spark.api.java.JavaRDD;
@@ -50,48 +44,10 @@ public class SparkAdCVRSampleTester {
         }
     }
 
-
-    // 单条日志处理逻辑
     public static String singleParse(Record record) {
         // 数据解析
-        String label = record.getString("pcvr");
-
-        // 从sql的 record中 初始化对象内容
-        AdRequestContext requestContext = AdSampleConstructor.constructRequestContext(record);
-        UserAdFeature userFeature = AdSampleConstructor.constructUserFeature(record);
-        AdItemFeature itemFeature = AdSampleConstructor.constructItemFeature(record);
-
-        // 转化成bytes
-        AdRequestContextBytesFeature adRequestContextBytesFeature = new AdRequestContextBytesFeature(requestContext);
-        UserAdBytesFeature userBytesFeature = new UserAdBytesFeature(userFeature);
-        AdItemBytesFeature adItemBytesFeature = new AdItemBytesFeature(itemFeature);
-
-        // 特征抽取
-        VlogAdCtrLRFeatureExtractor bytesFeatureExtractor;
-        bytesFeatureExtractor = new VlogAdCtrLRFeatureExtractor();
-
-        LRSamples lrSamples = bytesFeatureExtractor.single(userBytesFeature, adItemBytesFeature, adRequestContextBytesFeature);
-
-        return parseSamplesToString2(label, lrSamples);
+        String label = record.getString("adinvert_ornot");
+        String samples = record.getString("lrsample");
+        return label + "\t" + samples;
     }
-
-
-    // 构建样本的字符串
-    public static String parseSamplesToString2(String label, LRSamples lrSamples) {
-        ArrayList<String> featureList = new ArrayList<String>();
-        for (int i = 0; i < lrSamples.getFeaturesCount(); i++) {
-            GroupedFeature groupedFeature = lrSamples.getFeatures(i);
-            if (groupedFeature != null && groupedFeature.getFeaturesCount() != 0) {
-                for (int j = 0; j < groupedFeature.getFeaturesCount(); j++) {
-                    BaseFeature baseFeature = groupedFeature.getFeatures(j);
-                    if (baseFeature != null) {
-                        featureList.add(String.valueOf(baseFeature.getIdentifier()) + ":1");
-                    }
-                }
-            }
-        }
-        return label + "\t" + String.join("\t", featureList);
-    }
-
-
 }

src/main/java/examples/sparksql/SparkAdFeaToRedisLoader.java

@@ -74,7 +74,6 @@ public class SparkAdFeaToRedisLoader {
             UserAdFeature userFeature = AdRedisFeatureConstructor.constructUserFeature(record);
             List<String> kv = new ArrayList<String>();
             String key = String.format(userKeyFormat, userFeature.getKey());
-
             String value = userFeature.getValue();
             kv.add(key);
             kv.add(value);