Explorar o código

增加新的特征表制作

xueyiming hai 1 día
pai
achega
c7bbd9a59f

+ 5 - 1
src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20260808.scala

@@ -806,8 +806,12 @@ object makedata_ad_33_bucketDataFromOriginToHive_20260808 {
             val labelObject = JSON.parseObject(labelKey)
             val label = labelObject.getOrDefault(whatLabel, "0").toString
             val bucketsMap = bucketsMap_br.value
+            // k1~k4 新特征不受 filterNames 影响(避免 _4h_ 等把 k*_4h_* 误杀);原有 b* 等仍按 filterNames 过滤
+            def keepDense(name: String): Boolean =
+              name.startsWith("k1_") || name.startsWith("k2_") || name.startsWith("k3_") || name.startsWith("k4_") ||
+                !filterNames.exists(name.contains)
             var resultMap = denseFeatures.collect {
-              case (name, score) if !filterNames.exists(name.contains) && score > 1E-8 =>
+              case (name, score) if keepDense(name) && score > 1E-8 =>
                 val value = if (bucketsMap.contains(name)) {
                   val (bucketsNum, buckets) = bucketsMap(name)
                   1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)