package com.aliyun.odps.spark.examples.makedata_recsys_r_rate

import com.alibaba.fastjson.JSON
import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils}
import examples.extractor.ExtractorUtils
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.sql.SparkSession

import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import scala.io.Source
import scala.util.Random


object makedata_recsys_71_rov_sample_20250109 {
  def main(args: Array[String]): Unit = {

    // 1 读取参数
    val param = ParamUtils.parseArgs(args)
    val readPath = param.getOrElse("readPath", "/dw/recommend/model/71_origin_data/")
    val beginStr = param.getOrElse("beginStr", "20250103")
    val endStr = param.getOrElse("endStr", "20250103")
    val whatApps = param.getOrElse("whatApps", "0,3,4,21,17").split(",").toSet
    val whatLabel = param.getOrElse("whatLabel", "is_return_1")
    val fuSampleRate = param.getOrElse("fuSampleRate", "1.0").toDouble
    val notUseBucket = param.getOrElse("notUseBucket", "0").toInt
    val featureNameFile = param.getOrElse("featureName", "20241209_recsys_rov_name.txt")
    val featureBucketFile = param.getOrElse("featureBucket", "20241209_recsys_rov_bucket.txt")
    val repartition = param.getOrElse("repartition", "100").toInt
    val savePath = param.getOrElse("savePath", "/dw/recommend/model/71_recsys_rov_train_data/")

    val spark = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .getOrCreate()
    val sc = spark.sparkContext

    val loader = getClass.getClassLoader
    val featureNameSet = loadUseFeatureNames(loader, featureNameFile)
    val featureBucketMap = loadUseFeatureBuckets(loader, notUseBucket, featureBucketFile)
    val bucketsMap_br = sc.broadcast(featureBucketMap)

    val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
    for (date <- dateRange) {
      println("开始执行:" + date)
      val data = sc.textFile(readPath + "/" + date + "*").map(r => {
          // logKey + "\t" + labelKey + "\t" + scoresMap + "\t" + featureKey
          val rList = r.split("\t")
          val logKey = rList(0)
          val labelKey = rList(1)
          val scoresMap = rList(2)
          val featData = rList(3)
          (logKey, labelKey, scoresMap, featData)
        })
        .filter {
          case (logKey, labelKey, scoresMap, featData) =>
            validData(logKey, whatApps)
        }.filter {
          case (logKey, labelKey, scoresMap, featData) =>
            val label = parseLabel(labelKey, whatLabel)
            "1".equals(label) || new Random().nextDouble() <= fuSampleRate
        }
        .map {
          case (logKey, labelKey, scoresMap, featData) =>
            val label = parseLabel(labelKey, whatLabel)
            val features = parseFeature(featData)
            (logKey, label, scoresMap, features)
        }
        .mapPartitions(row => {
          val result = new ArrayBuffer[String]()
          val bucketsMap = bucketsMap_br.value
          row.foreach {
            case (logKey, label, scoresMap, features) =>
              val featuresBucket = bucketFeature(featureNameSet, bucketsMap, features)
              result.add(logKey + "\t" + label + "\t" + scoresMap + "\t" + featuresBucket.mkString("\t"))
          }
          result.iterator
        })

      // 4 保存数据到hdfs
      val hdfsPath = savePath + "/" + date
      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
        println("删除路径并开始数据写入:" + hdfsPath)
        MyHdfsUtils.delete_hdfs_path(hdfsPath)
        data.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
      } else {
        println("路径不合法，无法写入:" + hdfsPath)
      }
    }
  }

  private def loadFileData(loader: ClassLoader, nameFile: String): String = {
    val resourceUrlBucket = loader.getResource(nameFile)
    val data =
      if (resourceUrlBucket != null) {
        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
        Source.fromURL(resourceUrlBucket).close()
        buckets
      } else {
        ""
      }
    data
  }

  private def loadUseFeatureNames(loader: ClassLoader, nameFile: String): Set[String] = {
    val names = loadFileData(loader, nameFile)
    println(names)
    names.split("\n")
      .map(r => r.replace(" ", "").replaceAll("\n", ""))
      .filter(r => r.nonEmpty)
      .toSet
  }

  private def loadUseFeatureBuckets(loader: ClassLoader, notUseBucket: Int, nameFile: String): Map[String, (Double, Array[Double])] = {
    if (notUseBucket > 0) {
      return Map[String, (Double, Array[Double])]()
    }
    val buckets = loadFileData(loader, nameFile)
    println(buckets)
    buckets.split("\n")
      .map(r => r.replace(" ", "").replaceAll("\n", ""))
      .filter(r => r.nonEmpty)
      .map(r => {
        val rList = r.split("\t")
        (rList(0), (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
      }).toMap
  }

  private def recommendFlow(flowPool: String): Boolean = {
    if (flowPool.isEmpty || flowPool.endsWith("#1")) {
      return true
    }
    false
  }

  private def validData(logKey: String, whatApps: Set[String]): Boolean = {
    // apptype, page, pagesource, recommendpagetype, flowpool, abcode, mid, vid, level, ts
    val cells = logKey.split(",")
    val apptype = cells(0)
    val page = cells(1)
    //val pagesource = cells(2)
    val recommendpagetype = cells(3)
    val flowpool = cells(4)
    if (whatApps.contains(apptype)) {
      if (recommendFlow(flowpool)) {
        if (page.equals("详情后沉浸页")) {
          return true
        } else if (page.equals("回流后沉浸页&内页feed")) {
          //if (recommendpagetype.endsWith("pages/user-videos-share-recommend-detail")) {
          return true
          //}
        } else if (page.equals("首页feed")) {
          return true
        }
      }
    }
    false
  }

  private def parseLabel(data: String, key: String, default: String = "0"): String = {
    JSON.parseObject(data).getOrDefault(key, default).toString
  }

  private def parseFeature(data: String): scala.collection.mutable.Map[String, Double] = {
    val features = scala.collection.mutable.Map[String, Double]()
    if (data.nonEmpty) {
      val obj = JSON.parseObject(data)
      obj.foreach(r => {
        features.put(r._1, obj.getDoubleValue(r._1))
      })
    }
    features
  }

  private def bucketFeature(nameSet: Set[String], bucketMap: Map[String, (Double, Array[Double])], features: scala.collection.mutable.Map[String, Double]): Iterable[String] = {
    features.map {
      case (name, score) =>
        if (!nameSet.contains(name)) {
          ""
        } else {
          if (score > 1E-8) {
            if (bucketMap.contains(name)) {
              val (bucketsNum, buckets) = bucketMap(name)
              val scoreNew = 1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
              name + ":" + scoreNew.toString
            } else {
              name + ":" + score.toString
            }
          } else {
            ""
          }
        }
    }.filter(_.nonEmpty)
  }
}
