package com.aliyun.odps.spark.examples.makedata_recsys_r_rate

import com.aliyun.odps.data.Record
import com.aliyun.odps.spark.examples.myUtils._
import examples.utils.SimilarityUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession

import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer

/*
   20250221 提取特征
 */

object makedata_recsys_82_originData_20250221 {
  private val labelNames = List(
    "is_share", "share_cnt",
    "is_return_1", "return_1_uv",
    "is_return_n", "return_n_uv",
    "is_return_noself", "return_1_uv_noself",
    "is_return_n_noself", "return_n_uv_noself"
  )

  private def parseVideoRdd(videoRdd: RDD[Record]): RDD[(String, java.util.Map[String, String])] = {
    videoRdd
      .map(record => {
        val vid = record.getString("vid")
        val feature = ConvertUtils.getRecordCol(record, "feature")
        feature.put("vid", vid)
        (vid, feature)
      })
      .reduceByKey((a, b) => if (a.size() > b.size()) a else b)
  }

  private def getVidMidRdd(logRdd: RDD[java.util.Map[String, String]]): RDD[(String, String)] = {
    logRdd
      .map(raw => {
        val mid = raw.getOrElse("mid", "")
        val c9 = raw.getOrElse("c9_feature", "")
        (mid, c9)
      })
      .filter(_._1.nonEmpty)
      .reduceByKey((a, b) => a)
      .flatMap(raw => {
        val result = new ArrayBuffer[(String, String)]
        for (hVid <- ConvertUtils.getVidList(raw._2)) {
          result += ((hVid, raw._1)) // (vid, mid)
        }
        result
      })
  }

  private def getMidSeqRdd(vidMidRdd: RDD[(String, String)], videoRdd: RDD[(String, java.util.Map[String, String])]): RDD[(String, List[java.util.Map[String, String]])] = {
    vidMidRdd
      .join(videoRdd)
      .map(raw => {
        (raw._2._1, raw._2._2)
      })
      .groupByKey()
      .map(raw => {
        (raw._1, raw._2.toList)
      })
  }

  private def joinMidSeq(logRdd: RDD[java.util.Map[String, String]], midSeqRdd: RDD[(String, List[java.util.Map[String, String]])]): RDD[(java.util.Map[String, String], List[java.util.Map[String, String]])] = {
    logRdd
      .map(raw => {
        val mid = raw.getOrElse("mid", "")
        (mid, raw)
      })
      .leftOuterJoin(midSeqRdd)
      .map(raw => {
        if (raw._2._2.isDefined) {
          (raw._2._1, raw._2._2.get)
        } else {
          (raw._2._1, List())
        }
      })
  }

  private def getFeature(rdd: RDD[(java.util.Map[String, String], List[java.util.Map[String, String]])]): RDD[String] = {
    rdd.mapPartitions(partition => {
      SimilarityUtils.init()
      partition.map(raw => {
        val record = raw._1
        val videoSeq = raw._2
        val logKey = DataUtils.getLogKey(record)
        val labels = DataUtils.getLabels(labelNames, record).toString
        val features = ConvertUtils.getFeature(record, videoSeq, 6).toString
        val scoresMap = DataUtils.getSubJson(record, "extend_alg", "scoresMap").toString
        logKey + "\t" + labels + "\t" + scoresMap + "\t" + features
      })
    })
  }

  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .getOrCreate()
    val sc = spark.sparkContext

    // 1. 解析参数
    val param = ParamUtils.parseArgs(args)
    val project = param.getOrElse("project", "loghubods")
    val table = param.getOrElse("table", "dwd_recsys_alg_sample_all_20250212")
    val tablePart = param.getOrElse("tablePart", "64").toInt
    val beginStr = param.getOrElse("beginStr", "2025022317")
    val endStr = param.getOrElse("endStr", "2025022317")
    val whatLabel = param.getOrElse("whatLabel", "is_return_n_noself")
    val fuSampleRate = param.getOrElse("fuSampleRate", "0.1").toDouble
    val savePath = param.getOrElse("savePath", "/dw/recommend/model/82_origin_data/")
    val repartition = param.getOrElse("repartition", "32").toInt

    // 2. 数据处理
    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
    for (dt_hh <- timeRange) {
      // a. 分区
      val dt = dt_hh.substring(0, 8)
      val hh = dt_hh.substring(8, 10)
      val partition = "dt=%s,hh=%s".format(dt, hh)
      println("开始执行partition:" + partition)

      // b. 加载视频信息
      val originVideo = DataUtils.getODPSData(sc, project, "alg_recsys_feature_user_behavior_video", partition, tablePart)
      val uniqVideo = parseVideoRdd(originVideo)

      // c. 加载样本数据
      val odpsData = DataUtils.getODPSData(sc, project, table, partition, tablePart)

      // d. 样本重采样
      val resampleData = DataUtils.resample(whatLabel, fuSampleRate, odpsData)
        .map(record => {
          ConvertUtils.record2Map(record)
        })

      // e. get vid mid rdd
      val vidMidRdd = getVidMidRdd(resampleData)

      // f. get mid seq rdd
      val midSeqRdd = getMidSeqRdd(vidMidRdd, uniqVideo)

      // g. 历史行为关联video
      val seqSampleData = joinMidSeq(resampleData, midSeqRdd)

      // h. 特征转换
      val featureData = getFeature(seqSampleData)

      // i. 保存数据
      val hdfsPath = "%s/%s%s".format(savePath, dt, hh)
      DataUtils.saveData(featureData, hdfsPath, repartition)
    }
  }
}
