Forráskód Böngészése

Update makedata_ad_33_bucketDataFromOriginToHive_20250228: coalesce

StrayWarrior 1 hónapja
szülő
commit
662c0b211b

+ 3 - 3
src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20250228.scala

@@ -71,7 +71,7 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250228 {
     val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
     for (dt <- dateRange) {
       val timeRange = MyDateUtils.getDateHourRange(dt + "08", dt + "23")
-      val list = timeRange.map { dt_hh =>
+      val recordRdd = timeRange.map { dt_hh =>
           val dt = dt_hh.substring(0, 8)
           val hh = dt_hh.substring(8, 10)
           val partition = s"dt=$dt,hh=$hh"
@@ -525,10 +525,10 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250228 {
             resultMap += ("logkey" -> logKey)
             (label.toInt, resultMap, Random.nextDouble())
         }.filter(r => r._3 < negSampleRate || r._1 > 0)
-        .map(r => r._2)
+        .map(r => r._2).coalesce(128)
 
       val partition = s"dt=$dt"
-      odpsOps.saveToTable(project, outputTable, partition, list, write, defaultCreate = true, overwrite = true)
+      odpsOps.saveToTable(project, outputTable, partition, recordRdd, write, defaultCreate = true, overwrite = true)
     }
   }