|
@@ -32,6 +32,37 @@ public class FeatureDiff {
|
|
|
abCodes.add("ab3");
|
|
|
tableToCol = new HashMap<>();
|
|
|
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void addUserTable() {
|
|
|
tableToCol.put("alg_mid_feature_play", "c1_feature");
|
|
|
tableToCol.put("alg_mid_feature_share_and_return", "c2_feature");
|
|
|
tableToCol.put("alg_mid_feature_play_tags", "c3_feature");
|
|
@@ -41,6 +72,9 @@ public class FeatureDiff {
|
|
|
tableToCol.put("alg_mid_feature_feed_exp_return_tags", "c7_feature");
|
|
|
tableToCol.put("alg_mid_feature_sharecf", "c8_feature");
|
|
|
tableToCol.put("alg_mid_feature_returncf", "c9_feature");
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void addVideoTable() {
|
|
|
tableToCol.put("alg_vid_feature_all_exp", "b1_feature");
|
|
|
tableToCol.put("alg_vid_feature_all_share", "b2_feature");
|
|
|
tableToCol.put("alg_vid_feature_all_return", "b3_feature");
|
|
@@ -64,6 +98,18 @@ public class FeatureDiff {
|
|
|
|
|
|
public static void main(String[] args) {
|
|
|
|
|
|
+ log.info("args {}", JSONUtils.toJson(args));
|
|
|
+ if (args[3].equals("0")) {
|
|
|
+ addVideoTable();
|
|
|
+ }
|
|
|
+ if (args[3].equals("1")) {
|
|
|
+ addUserTable();
|
|
|
+ }
|
|
|
+ if (args[3].equals("2")) {
|
|
|
+ addVideoTable();
|
|
|
+ addUserTable();
|
|
|
+ }
|
|
|
+ log.info("tableToCol {}", JSONUtils.toJson(tableToCol));
|
|
|
|
|
|
SparkConf sparkConf = new SparkConf()
|
|
|
|
|
@@ -83,6 +129,9 @@ public class FeatureDiff {
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
+ long totalCount = fieldValues.count();
|
|
|
+ log.info("total count {}", totalCount);
|
|
|
+
|
|
|
long diffCount =
|
|
|
fieldValues.repartition(partitionNum).filter(new Function<Map<String, String>,
|
|
|
Boolean>() {
|