package com.aliyun.odps.spark.examples.myUtils

import com.alibaba.fastjson.{JSON, JSONObject}
import com.aliyun.odps.TableSchema
import com.aliyun.odps.data.Record
import examples.extractor.ExtractorUtils
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD

import scala.collection.JavaConversions.mapAsScalaMap
import scala.io.Source
import scala.util.Random

object DataUtils {
  def getODPSData(sc: SparkContext, project: String, table: String, partition: String, numPartition: Int): RDD[Record] = {
    val odpsOps = env.getODPS(sc)
    odpsOps.readTable(project = project,
      table = table,
      partition = partition,
      transfer = func,
      numPartition = numPartition)
  }

  def resample(name: String, negRate: Double, rdd: RDD[Record]): RDD[Record] = {
    rdd.filter(record => {
      val label = getStringValue(record, name, "0").toDouble
      label > 0 || new Random().nextDouble() <= negRate
    })
  }

  def resampleWithoutInterception(name: String, negRate: Double, rdd: RDD[Record]): RDD[Record] = {
    rdd.filter(record => {
      val label = getStringValue(record, name, "0").toDouble
      label > 0 || (!interception(record, "extend", "animationSceneType", "interception") && new Random().nextDouble() <= negRate)
    })
  }

  def getLogKey(record: java.util.Map[String, String]): String = {
    val apptype = record.getOrElse("apptype", "")
    val page = record.getOrElse("page", "")
    val pagesource = record.getOrElse("pagesource", "")
    val recommendpagetype = record.getOrElse("recommendpagetype", "")
    val flowpool = record.getOrElse("flowpool", "")
    val abcode = record.getOrElse("abcode", "")
    val mid = record.getOrElse("mid", "")
    val vid = record.getOrElse("vid", "")
    val level = record.getOrElse("level", "0")
    val ts = record.getOrElse("ts", "0")
    (apptype, page, pagesource, recommendpagetype, flowpool, abcode, mid, vid, level, ts).productIterator.mkString(",")
  }

  def getLabels(names: List[String], record: java.util.Map[String, String]): JSONObject = {
    val labels = new JSONObject
    for (name <- names) {
      if (record.containsKey(name)) {
        labels.put(name, record.get(name))
      }
    }
    labels
  }

  def saveData(rdd: RDD[String], hdfsPath: String, repartition: Int = 64): Unit = {
    if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
      println("删除路径并开始数据写入:" + hdfsPath)
      MyHdfsUtils.delete_hdfs_path(hdfsPath)
      rdd.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
    } else {
      println("路径不合法，无法写入:" + hdfsPath)
    }
  }

  def func(record: Record, schema: TableSchema): Record = {
    record
  }

  def getStringValue(record: Record, key: String, default: String = ""): String = {
    if (!record.isNull(key)) {
      val value = record.getString(key)
      return value.trim
    }
    default
  }

  def getSubJson(record: java.util.Map[String, String], key1: String, key2: String): JSONObject = {
    if (record.containsKey(key1)) {
      val obj = JSON.parseObject(record.get(key1))
      if (obj.nonEmpty && obj.containsKey(key2)) {
        val data = obj.getString(key2)
        return JSON.parseObject(data.replace("\\", ""))
      }
    }
    new JSONObject()
  }

  def interception(record: Record, key1: String, key2: String, value2: String): Boolean = {
    if (!record.isNull(key1)) {
      val obj = JSON.parseObject(record.getString(key1))
      if (obj.nonEmpty && obj.containsKey(key2)) {
        val data = obj.getString(key2)
        return data.equals(value2)
      }
    }
    false
  }

  def loadFileData(loader: ClassLoader, nameFile: String): String = {
    val resourceUrlBucket = loader.getResource(nameFile)
    val data =
      if (resourceUrlBucket != null) {
        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
        Source.fromURL(resourceUrlBucket).close()
        buckets
      } else {
        ""
      }
    data
  }

  def loadUseFeatureNames(loader: ClassLoader, nameFile: String): Set[String] = {
    val names = loadFileData(loader, nameFile)
    println(names)
    names.split("\n")
      .map(r => r.replace(" ", "").replaceAll("\n", ""))
      .filter(r => r.nonEmpty)
      .toSet
  }

  def loadUseFeatureBuckets(loader: ClassLoader, notUseBucket: Int, nameFile: String): Map[String, (Double, Array[Double])] = {
    if (notUseBucket > 0) {
      return Map[String, (Double, Array[Double])]()
    }
    val buckets = loadFileData(loader, nameFile)
    println(buckets)
    buckets.split("\n")
      .map(r => r.replace(" ", "").replaceAll("\n", ""))
      .filter(r => r.nonEmpty)
      .map(r => {
        val rList = r.split("\t")
        (rList(0), (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
      }).toMap
  }

  def parseLabel(data: String, key: String, default: String = "0"): String = {
    JSON.parseObject(data).getOrDefault(key, default).toString
  }

  def parseFeature(data: String): scala.collection.mutable.Map[String, Double] = {
    val features = scala.collection.mutable.Map[String, Double]()
    if (data.nonEmpty) {
      val obj = JSON.parseObject(data)
      obj.foreach(r => {
        features.put(r._1, obj.getDoubleValue(r._1))
      })
    }
    features
  }

  def bucketFeature(nameSet: Set[String], bucketMap: Map[String, (Double, Array[Double])], features: scala.collection.mutable.Map[String, Double]): Iterable[String] = {
    features.map {
      case (name, score) =>
        if (!nameSet.contains(name)) {
          ""
        } else {
          if (score > 1E-8) {
            if (bucketMap.contains(name)) {
              val (bucketsNum, buckets) = bucketMap(name)
              val scoreNew = 1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
              name + ":" + scoreNew.toString
            } else {
              name + ":" + score.toString
            }
          } else {
            ""
          }
        }
    }.filter(_.nonEmpty)
  }
}
