广告通过print数据验证一致性

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/makedata_ad_33_bucketDataPrint_20240628.scala

@@ -37,7 +37,7 @@ object makedata_ad_33_bucketDataPrint_20240628 {
 
 
     val loader = getClass.getClassLoader
-    val resourceUrl = loader.getResource("20240608_feature_name.txt")
+    val resourceUrl = loader.getResource("20240622_ad_feature_name.txt")
     val content =
       if (resourceUrl != null) {
         val content = Source.fromURL(resourceUrl).getLines().mkString("\n")
@@ -52,7 +52,7 @@ object makedata_ad_33_bucketDataPrint_20240628 {
       .filter(r => r.nonEmpty).toList
     val contentList_br = sc.broadcast(contentList)
 
-    val resourceUrlBucket = loader.getResource("20240609_bucket_274.txt")
+    val resourceUrlBucket = loader.getResource("20240622_ad_bucket_249.txt")
     val buckets =
       if (resourceUrlBucket != null) {
         val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
@@ -331,7 +331,7 @@ object makedata_ad_33_bucketDataPrint_20240628 {
           }
 
 
-          val flag = record.isNull("allfeature")
+          val flag = record.isNull("metafeature")
 
           val allfeaturemap = if (record.isNull("allfeature")) new JSONObject() else
             JSON.parseObject(record.getString("allfeature"))
@@ -485,7 +485,7 @@ object makedata_ad_33_bucketDataPrint_20240628 {
       println("路径不合法，无法写入:" + saveV1)
     }
 
-    val saveV2 = "/dw/recommend/model/33_for_check_v1/" + readDate
+    val saveV2 = "/dw/recommend/model/33_for_check_v2/" + readDate
     if (saveV2.nonEmpty && saveV2.startsWith("/dw/recommend/model/")) {
       println("删除路径并开始数据写入:" + saveV2)
       MyHdfsUtils.delete_hdfs_path(saveV2)
@@ -494,7 +494,7 @@ object makedata_ad_33_bucketDataPrint_20240628 {
       println("路径不合法，无法写入:" + saveV2)
     }
 
-    val saveV3 = "/dw/recommend/model/33_for_check_v1/" + readDate
+    val saveV3 = "/dw/recommend/model/33_for_check_v3/" + readDate
     if (saveV3.nonEmpty && saveV3.startsWith("/dw/recommend/model/")) {
       println("删除路径并开始数据写入:" + saveV3)
       MyHdfsUtils.delete_hdfs_path(saveV3)
@@ -503,7 +503,7 @@ object makedata_ad_33_bucketDataPrint_20240628 {
       println("路径不合法，无法写入:" + saveV3)
     }
 
-    val saveV4 = "/dw/recommend/model/33_for_check_v1/" + readDate
+    val saveV4 = "/dw/recommend/model/33_for_check_v4/" + readDate
     if (saveV4.nonEmpty && saveV4.startsWith("/dw/recommend/model/")) {
       println("删除路径并开始数据写入:" + saveV4)
       MyHdfsUtils.delete_hdfs_path(saveV4)
@@ -512,7 +512,7 @@ object makedata_ad_33_bucketDataPrint_20240628 {
       println("路径不合法，无法写入:" + saveV4)
     }
 
-    val saveV5 = "/dw/recommend/model/33_for_check_v1/" + readDate
+    val saveV5 = "/dw/recommend/model/33_for_check_v5/" + readDate
     if (saveV5.nonEmpty && saveV5.startsWith("/dw/recommend/model/")) {
       println("删除路径并开始数据写入:" + saveV5)
       MyHdfsUtils.delete_hdfs_path(saveV5)
@@ -521,7 +521,7 @@ object makedata_ad_33_bucketDataPrint_20240628 {
       println("路径不合法，无法写入:" + saveV5)
     }
 
-    val saveV6 = "/dw/recommend/model/33_for_check_v1/" + readDate
+    val saveV6 = "/dw/recommend/model/33_for_check_v6/" + readDate
     if (saveV6.nonEmpty && saveV6.startsWith("/dw/recommend/model/")) {
       println("删除路径并开始数据写入:" + saveV6)
       MyHdfsUtils.delete_hdfs_path(saveV6)

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本-广告

@@ -20,6 +20,7 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --conf spark.driver.maxResultSize=16G \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 fileName:20240620_100_fix \
+savePath:/dw/recommend/model/32_bucket_data/ \
 > p32_data.log 2>&1 &
 
 
@@ -52,7 +53,9 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 beginStr:2024062717 endStr:2024062723 \
 readDate:20240627 \
 table:alg_recsys_ad_sample_all_new \
-> p17_data_check.log 2>&1 &
+savePath:/dw/recommend/model/33_for_check/ \
+
+> p33_data_check.log 2>&1 &
 
 
 /dw/recommend/model/33_for_check_v1/