xueyiming пре 5 дана
родитељ
комит
b5d0523f88

+ 15 - 5
src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20260703.scala

@@ -90,6 +90,15 @@ object makedata_ad_33_bucketDataFromOriginToHive_20260703 {
       "is_weekday", "day_of_the_week", "user_conver_ad_class", "category_name",
       "material_md5", "user_layer", "user_class", "user_click_ad_class", "user_view_ad_class",
       "customer", "landing", "flag")
+    val labelFields = Set(
+      "has_click",
+      "has_conversion",
+      "has_scan",
+      "has_addwechat",
+      "has_scan_addwechat",
+      "is_landing3",
+      "is_landing3_with_has_scan"
+    )
 
 
     // 2 读取odps+表信息
@@ -99,15 +108,16 @@ object makedata_ad_33_bucketDataFromOriginToHive_20260703 {
 
     // 检查所有字段,收集非法字段
     val invalidFields = tableSchema.flatMap { case (fieldName, _) =>
-      // 跳过 has_click 和 has_conversion 列
-      if (fieldName != "has_click" && fieldName != "has_conversion") {
+      // 如果是标签字段,直接跳过不校验
+      if (labelFields.contains(fieldName)) {
+        None
+      } else {
+        // 否则,校验是否在特征列表里
         if (!lowerCaseDenseFeatureNames.contains(fieldName) && !sparseFeatureNames.contains(fieldName)) {
-          Some(fieldName) // 收集缺少字段
+          Some(fieldName) // 收集未知字段
         } else {
           None
         }
-      } else {
-        None
       }
     }.toList