|
@@ -90,6 +90,15 @@ object makedata_ad_33_bucketDataFromOriginToHive_20260703 {
|
|
|
"is_weekday", "day_of_the_week", "user_conver_ad_class", "category_name",
|
|
"is_weekday", "day_of_the_week", "user_conver_ad_class", "category_name",
|
|
|
"material_md5", "user_layer", "user_class", "user_click_ad_class", "user_view_ad_class",
|
|
"material_md5", "user_layer", "user_class", "user_click_ad_class", "user_view_ad_class",
|
|
|
"customer", "landing", "flag")
|
|
"customer", "landing", "flag")
|
|
|
|
|
+ val labelFields = Set(
|
|
|
|
|
+ "has_click",
|
|
|
|
|
+ "has_conversion",
|
|
|
|
|
+ "has_scan",
|
|
|
|
|
+ "has_addwechat",
|
|
|
|
|
+ "has_scan_addwechat",
|
|
|
|
|
+ "is_landing3",
|
|
|
|
|
+ "is_landing3_with_has_scan"
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
|
|
|
|
|
// 2 读取odps+表信息
|
|
// 2 读取odps+表信息
|
|
@@ -99,15 +108,16 @@ object makedata_ad_33_bucketDataFromOriginToHive_20260703 {
|
|
|
|
|
|
|
|
// 检查所有字段,收集非法字段
|
|
// 检查所有字段,收集非法字段
|
|
|
val invalidFields = tableSchema.flatMap { case (fieldName, _) =>
|
|
val invalidFields = tableSchema.flatMap { case (fieldName, _) =>
|
|
|
- // 跳过 has_click 和 has_conversion 列
|
|
|
|
|
- if (fieldName != "has_click" && fieldName != "has_conversion") {
|
|
|
|
|
|
|
+ // 如果是标签字段,直接跳过不校验
|
|
|
|
|
+ if (labelFields.contains(fieldName)) {
|
|
|
|
|
+ None
|
|
|
|
|
+ } else {
|
|
|
|
|
+ // 否则,校验是否在特征列表里
|
|
|
if (!lowerCaseDenseFeatureNames.contains(fieldName) && !sparseFeatureNames.contains(fieldName)) {
|
|
if (!lowerCaseDenseFeatureNames.contains(fieldName) && !sparseFeatureNames.contains(fieldName)) {
|
|
|
- Some(fieldName) // 收集缺少字段
|
|
|
|
|
|
|
+ Some(fieldName) // 收集未知字段
|
|
|
} else {
|
|
} else {
|
|
|
None
|
|
None
|
|
|
}
|
|
}
|
|
|
- } else {
|
|
|
|
|
- None
|
|
|
|
|
}
|
|
}
|
|
|
}.toList
|
|
}.toList
|
|
|
|
|
|