|
@@ -45,6 +45,7 @@ object test {
|
|
|
}).toMap
|
|
|
val bucketsMap_br = sc.broadcast(bucketsMap)
|
|
|
val denseFeatureNames = bucketsMap.keySet
|
|
|
+ val lowerCaseDenseFeatureNames = bucketsMap.keySet.map(_.toLowerCase)
|
|
|
val sparseFeatureNames = Set(
|
|
|
"cid", "adid", "adverid", "targeting_conversion",
|
|
|
"region", "city", "brand",
|
|
@@ -59,14 +60,16 @@ object test {
|
|
|
"user_adverid_click_3d", "user_adverid_click_7d", "user_adverid_click_30d",
|
|
|
"user_adverid_conver_3d", "user_adverid_conver_7d", "user_adverid_conver_30d",
|
|
|
"user_skuid_view_3d", "user_skuid_view_7d", "user_skuid_view_30d",
|
|
|
- "user_skuid_click_3d", "user_skuid_click_7d")
|
|
|
+ "user_skuid_click_3d", "user_skuid_click_7d", "user_skuid_click_30d",
|
|
|
+ "user_skuid_conver_3d", "user_skuid_conver_7d", "user_skuid_conver_30d",
|
|
|
+ "is_weekday", "day_of_the_week")
|
|
|
|
|
|
|
|
|
// 2 读取odps+表信息
|
|
|
val odpsOps = env.getODPS(sc)
|
|
|
val tableSchema = odpsOps.getTableSchema(project, outputTable, isPartition = false)
|
|
|
for (t <- tableSchema) {
|
|
|
- if (!denseFeatureNames.contains(t._1) && !sparseFeatureNames.contains(t._1)) {
|
|
|
+ if (!lowerCaseDenseFeatureNames.contains(t._1) && !sparseFeatureNames.contains(t._1)) {
|
|
|
println(t._1)
|
|
|
}
|
|
|
}
|