package com.aliyun.odps.spark.examples.makedata_ad.v20240718

import com.alibaba.fastjson.JSON
import com.aliyun.odps.spark.examples.myUtils.{MyHdfsUtils, ParamUtils}
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.Partitioner
import org.apache.spark.sql.SparkSession

import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import scala.io.Source

/**
 * 20260807 分桶边界生成：
 * 1) 基于 origin 样本计算 k1~k4 事件特征分桶
 * 2) 与已有分桶文件（默认 20260410_ad_bucket_920.txt）合并后写出
 */
object makedata_ad_32_bucket_20260807 {
  class FeaturePartitioner(featureNames: Map[String, Int]) extends Partitioner {
    override def numPartitions: Int = featureNames.size

    override def getPartition(key: Any): Int = {
      val featureName = key.asInstanceOf[String]
      featureNames.getOrElse(featureName, 0)
    }
  }

  def main(args: Array[String]): Unit = {

    val spark = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .getOrCreate()
    val sc = spark.sparkContext

    // 1 读取参数
    val param = ParamUtils.parseArgs(args)
    val readPath = param.getOrElse("readPath", "/dw/recommend/model/31_ad_sample_data/20240620*")
    val savePath = param.getOrElse("savePath", "/dw/recommend/model/32_bucket_data/")
    val fileName = param.getOrElse("fileName", "20260807_ad_bucket_1112")
    val sampleRate = param.getOrElse("sampleRate", "1.0").toDouble
    val bucketNum = param.getOrElse("bucketNum", "100").toInt
    // 仅计算这些特征的分桶；默认自动生成全部 k 特征名。也可传 featureNameFile 覆盖
    val featureNameFile = param.getOrElse("featureNameFile", "20260807_ad_feature_name.txt")
    // 已有分桶文件（resources），与新计算的 k 分桶合并
    val existingBucketFile = param.getOrElse("existingBucketFile", "20260410_ad_bucket_920.txt")

    val loader = getClass.getClassLoader
    val featureNames =
      if (featureNameFile.nonEmpty) {
        loadResourceLines(loader, featureNameFile)
          .map(_.replace(" ", ""))
          .filter(_.nonEmpty)
          .distinct
          .toList
      } else {
        buildKFeatureNames()
      }
    println(s"featureNameFile=${if (featureNameFile.isEmpty) "<auto:k1~k4>" else featureNameFile}")
    println(s"featureNames.size=${featureNames.size}")
    val featureNamesSet = featureNames.toSet

    val existingBucketLines = loadResourceLines(loader, existingBucketFile)
      .filter(_.nonEmpty)
      .filter(line => !featureNamesSet.contains(line.split("\t")(0)))
    println(s"existingBucketFile=$existingBucketFile keepLines=${existingBucketLines.size}")

    val data = sc.textFile(readPath)

    // 2 读取特征数据、打平后分区
    val flattenData = data
      .sample(false, sampleRate)
      .flatMap(r => {
        val rList = r.split("\t")
        val jsons = JSON.parseObject(rList(2))
        jsons.map(r => (r._1, jsons.getDoubleValue(r._1)))
      }).filter(r => r._2 > 1E-8)
      .filter(r => featureNamesSet.contains(r._1))
      .partitionBy(new FeaturePartitioner(featureNames.zipWithIndex.toMap))

    // 3 计算分桶值
    val newBucketRdd = flattenData.mapPartitions(iter => {
      if (iter.isEmpty) {
        Array[String]().iterator
      } else {
        val headValue = iter.next()
        val key = headValue._1
        val leftValues = iter.map(_._2).toArray
        val sortedValues = (Array(headValue._2) ++ leftValues).sorted
        val len = sortedValues.length
        val oneBucketNum = (len - 1) / (bucketNum - 1) + 1 // 确保每个桶至少有一个元素
        val buffers = new ArrayBuffer[Double]()

        var lastBucketValue = sortedValues(0) // 记录上一个桶的切分点
        for (j <- 0 until len by oneBucketNum) {
          val d = sortedValues(j)
          if (j > 0 && d != lastBucketValue) {
            // 如果当前切分点不同于上一个切分点，则保存当前切分点
            buffers += d
          }
          lastBucketValue = d // 更新上一个桶的切分点
        }

        // 最后一个桶的结束点应该是数组的最后一个元素
        if (!buffers.contains(sortedValues.last)) {
          buffers += sortedValues.last
        }
        Array(key + "\t" + bucketNum.toString + "\t" + buffers.mkString(",")).iterator
      }
    })

    val existingBucketRdd = sc.parallelize(existingBucketLines, 1)
    val resultRdd = existingBucketRdd.union(newBucketRdd)

    // 4 保存数据到hdfs
    val hdfsPath = savePath + "/" + fileName
    if (hdfsPath.nonEmpty && fileName.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
      println("删除路径并开始数据写入:" + hdfsPath)
      MyHdfsUtils.delete_hdfs_path(hdfsPath)
      resultRdd.repartition(1).saveAsTextFile(hdfsPath, classOf[GzipCodec])
    } else {
      println("路径不合法，无法写入:" + hdfsPath)
    }
  }

  def buildKFeatureNames(): List[String] = {
    val kPrefixes = List("k1", "k2", "k3", "k4")
    val kPeriods = List("2h", "4h", "6h", "12h", "1d", "3d", "today", "1w")
    val kMetrics = List("view", "click", "conver", "ctr", "cvr", "ctvr")
    for {
      kPrefix <- kPrefixes
      period <- kPeriods
      metric <- kMetrics
    } yield s"${kPrefix}_${period}_${metric}"
  }

  def loadResourceLines(loader: ClassLoader, fileName: String): Array[String] = {
    if (fileName == null || fileName.isEmpty) {
      return Array.empty[String]
    }
    val resourceUrl = loader.getResource(fileName)
    if (resourceUrl == null) {
      println(s"existing bucket file not found in resources: $fileName")
      return Array.empty[String]
    }
    val source = Source.fromURL(resourceUrl)
    try {
      source.getLines().map(_.replace(" ", "")).filter(_.nonEmpty).toArray
    } finally {
      source.close()
    }
  }
}
