去掉重复数据

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20250522.scala

@@ -693,7 +693,14 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250522 {
             resultMap += ("has_conversion" -> labelObject.getString("ad_is_conversion"))
             resultMap += ("logkey" -> logKey)
             resultMap
-        }.coalesce(128)
+        }
+        .map(featMap=>{
+          val logkey = featMap.getOrElse("logkey", "")
+          (logkey, featMap)
+        })
+        .reduceByKey((a, b) => a)
+        .map(_._2)
+        .coalesce(128)
 
       val partition = s"dt=$dt"
       if (outputTable2.isEmpty) {