|
@@ -49,6 +49,10 @@ object diff_data_20240718 {
|
|
|
|
|
|
var result: List[String] = List.empty
|
|
|
|
|
|
+ result = result :+ "odpsData size =" + odpsData.count();
|
|
|
+
|
|
|
+ result = result :+ "hdfsData size =" + hdfsData.count();
|
|
|
+
|
|
|
// 以 logkey 为键进行转换
|
|
|
val rdd1Pairs: RDD[(String, Map[String, String])] = hdfsData.map(map => (map("logkey"), map))
|
|
|
val rdd2Pairs: RDD[(String, Map[String, String])] = odpsData.map(map => (map("logkey"), map))
|