Bläddra i källkod

增加过滤,只保留请求了adengine的数据

xueyiming 4 dagar sedan
förälder
incheckning
b48439a0d8

+ 8 - 1
src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20250522.scala

@@ -12,7 +12,7 @@ import java.time.{Instant, ZoneId, ZonedDateTime}
 import scala.collection.JavaConversions._
 import scala.collection.mutable.ArrayBuffer
 import scala.io.Source
-import scala.language.postfixOps
+import scala.language.{existentials, postfixOps}
 import scala.util.Random
 
 object makedata_ad_33_bucketDataFromOriginToHive_20250522 {
@@ -111,6 +111,13 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250522 {
             .filter(record => {
               AdUtil.isApi(record)
             })
+            .filter(record => {
+              val extendAlg = Option(record.getString("extend_alg"))
+                .filter(_.nonEmpty)
+                .map(JSON.parseObject)
+                .getOrElse(new JSONObject())
+              Option(extendAlg.getString("extractstrategy")).contains("engine")
+            })
             .filter(record => {
               val appType = record.getString("apptype")
               !Set("12", "13").contains(appType)