package com.aliyun.odps.spark.examples.ana


import com.alibaba.fastjson.JSONObject
import com.alibaba.fastjson.JSON
import com.aliyun.odps.TableSchema
import com.aliyun.odps.data.Record
import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils, env}
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.sql.SparkSession

import java.util
import java.util.{HashMap, Map}
import scala.collection.JavaConversions._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
/*
   所有获取不到的特征，给默认值0.
 */

object ana_01_cidvidpk {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .getOrCreate()
    val sc = spark.sparkContext

    // 1 读取参数
    val param = ParamUtils.parseArgs(args)
    val tablePart = param.getOrElse("tablePart", "64").toInt
    val project = param.getOrElse("project", "loghubods")
    val table = param.getOrElse("table", "ad_engine_statistics_log_per5min")
    val beginStr = param.getOrElse("beginStr", "2024060208")
    val endStr = param.getOrElse("endStr", "2024060223")
    val vidSelect = param.getOrElse("vidSelect", "")
    val cidsSelect = param.getOrElse("cidsSelect", "").split(",").toSet
    val apptypeSelect = param.getOrElse("apptype", "")
    val partitionPrefix = param.getOrElse("partitionPrefix", "dt=")

    // 2 读取odps+表信息
    val odpsOps = env.getODPS(sc)
    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
    val partitions = new ArrayBuffer[String]()
    for (dt_hh <- timeRange) {
      for (mi <- List (
        "0000", "0500", "1000", "1500", "2000", "2500",
        "3000", "3500", "4000", "4500", "5000", "5500"
      )){
        val partition = dt_hh + mi
        println("partition:" + partition)
        partitions.add(partition)
      }
    }
    val rdds = partitions.map(p => {
      odpsOps.readTable(project = project,
        table = table,
        partition = partitionPrefix + p,
        transfer = func,
        numPartition = tablePart)
    }).reduce((r1, r2) => r1.union(r2))

    val data = rdds.map(record=>{
      val vid = if (record.isNull("videoid")) "" else record.getString("videoid")
      val recalls = if (record.isNull("creativelist")) "" else record.getString("creativelist")
      val ranks = if (record.isNull("scoreresult")) "" else record.getString("scoreresult")
      val apptype = if (record.isNull("apptype")) "" else record.getString("apptype")
      val abcode = if (record.isNull("adabgroup")) "" else record.getString("adabgroup")
      (apptype, abcode, vid, recalls, ranks)
    }).filter(r => r._1.equals(apptypeSelect) && !r._3.equals("") && !r._4.equals("") && !r._5.equals(""))
      .filter(r=> r._3.equals(vidSelect)) // 过滤的vid
      .map{
        case (apptype, abcode, vid, recalls, ranks) =>
          val recalls_json = JSON.parseArray(recalls).map(r=>{
            val j = JSON.parseObject(r.toString)
            j.getOrElse("creativeId", 0).toString
          }).filter(!_.equals("0")).toSet
          val ranks_json = JSON.parseArray(ranks).map(r => {
            val j = JSON.parseObject(r.toString)
            val adId = j.getOrElse("adId", 0).toString
            val score = j.getOrElse("score", 0.0)
            (adId, score.toString.toDouble)
          })
          var rankId = ranks_json.get(0)._1
          var score = ranks_json.get(0)._2
//          for (i <- 1 until ranks_json.size){
//            val item = ranks_json.get(i)
//            if (item._2 > score){
//              rankId = item._1
//              score = item._2
//            }
//          }
          (apptype, abcode, vid, recalls_json, rankId)
      }.flatMap({
        case (apptype, abcode, vid, recalls_json, rankId) =>
          recalls_json.map(recallId=> {
            (apptype, abcode, vid, recallId, rankId, recalls_json)
          })
      }).filter(r=> cidsSelect.contains(r._4)) // 过滤的cid
      .map({
        case (apptype, abcode, vid, recallId, rankId, recalls_json) =>
          val x1 = 1
          val x2 = if (recallId.equals(rankId)) 1 else 0
          val x3 = if (cidsSelect.subsetOf(recalls_json)) 1 else 0
          val x4 = if (cidsSelect.subsetOf(recalls_json) && cidsSelect.contains(rankId)) 1 else 0
          val x5 = if (cidsSelect.subsetOf(recalls_json) && recallId.equals(rankId)) 1 else 0
          ((apptype, abcode, vid, recallId), (x1, x2, x3, x4, x5))
      }).aggregateByKey(
        (0, 0, 0, 0, 0)
      )(
        seqOp = (runningSum, x) => (runningSum._1 + x._1, runningSum._2 + x._2, runningSum._3 + x._3, runningSum._4 + x._4, runningSum._5 + x._5),
        combOp = (sum1, sum2) => (sum1._1 + sum2._1, sum1._2 + sum2._2, sum1._3 + sum2._3, sum1._4 + sum2._4, sum1._5 + sum2._5)
      )

    data.collect().foreach(r => println("结果\t" + r._1.productIterator.mkString("\t") + "\t" + r._2.productIterator.mkString("\t")))

  }



  def func(record: Record, schema: TableSchema): Record = {
    record
  }

}
