广告分桶

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/makedata_32_bucket_20240622.scala

@@ -49,7 +49,7 @@ object makedata_32_bucket_20240622 {
     val data = sc.textFile(readPath)
     val data1 = data.map(r => {
       val rList = r.split("\t")
-      val doubles = JSON.parseObject(rList(2)).mapValues(_.toString.toDouble)
+      val doubles = JSON.parseObject(rList(2)).mapValues(_.toString.toDouble).toMap
       doubles
     }).sample(false, sampleRate ).repartition(20)
 

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本-广告

@@ -15,7 +15,7 @@ table:alg_recsys_ad_sample_all \
 
 
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.examples.makedata.makedata_32_bucket_20240622 \
+--class com.aliyun.odps.spark.examples.makedata_ad.makedata_32_bucket_20240622 \
 --master yarn --driver-memory 16G --executor-memory 1G --executor-cores 1 --num-executors 16 \
 --conf spark.driver.maxResultSize=16G \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
@@ -23,7 +23,7 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 
 
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.examples.makedata.makedata_16_bucketData_20240609 \
+--class com.aliyun.odps.spark.examples.makedata_ad.makedata_16_bucketData_20240609 \
 --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 beginStr:20240615 endStr:20240615 repartition:1000 \