|
|
@@ -65,16 +65,27 @@ object test {
|
|
|
"is_weekday", "day_of_the_week")
|
|
|
|
|
|
|
|
|
- // 2 读取odps+表信息
|
|
|
+ // 2 读取odps表信息
|
|
|
val odpsOps = env.getODPS(sc)
|
|
|
val tableSchema = odpsOps.getTableSchema(project, outputTable, isPartition = false)
|
|
|
- for (t <- tableSchema) {
|
|
|
+
|
|
|
+ // 检查所有字段，收集非法字段
|
|
|
+ val invalidFields = tableSchema.flatMap { case (fieldName, _) =>
|
|
|
// 跳过 has_click 和 has_conversion 列
|
|
|
- if (t._1 != "has_click" && t._1 != "has_conversion") {
|
|
|
- if (!lowerCaseDenseFeatureNames.contains(t._1) && !sparseFeatureNames.contains(t._1)) {
|
|
|
- println(t._1)
|
|
|
+ if (fieldName != "has_click" && fieldName != "has_conversion") {
|
|
|
+ if (!lowerCaseDenseFeatureNames.contains(fieldName) && !sparseFeatureNames.contains(fieldName)) {
|
|
|
+ Some(fieldName) // 收集非法字段
|
|
|
+ } else {
|
|
|
+ None
|
|
|
}
|
|
|
+ } else {
|
|
|
+ None
|
|
|
}
|
|
|
+ }.toList
|
|
|
+
|
|
|
+ // 如果存在非法字段，抛出标准异常
|
|
|
+ if (invalidFields.nonEmpty) {
|
|
|
+ throw new IllegalArgumentException(s"发现未知字段: ${invalidFields.mkString(", ")}")
|
|
|
}
|
|
|
}
|
|
|
}
|
xueyiming