回滚(去掉重复数据)

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20250522.scala

@@ -694,15 +694,15 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250522 {
             resultMap += ("logkey" -> logKey)
             resultMap
         }
-        .map(featMap => {
-          val apptype = featMap.getOrElse("apptype", "")
-          val mid = featMap.getOrElse("mid", "")
-          val pqtid = featMap.getOrElse("pqtid", "")
-          val uniqKey = (apptype, mid, pqtid).productIterator.mkString(",")
-          (uniqKey, featMap)
-        })
-        .reduceByKey((a, b) => a)
-        .map(_._2)
+//        .map(featMap => {
+//          val apptype = featMap.getOrElse("apptype", "")
+//          val mid = featMap.getOrElse("mid", "")
+//          val pqtid = featMap.getOrElse("pqtid", "")
+//          val uniqKey = (apptype, mid, pqtid).productIterator.mkString(",")
+//          (uniqKey, featMap)
+//        })
+//        .reduceByKey((a, b) => a)
+//        .map(_._2)
         .coalesce(128)
 
       val partition = s"dt=$dt"