package com.aliyun.odps.spark.examples.makedata

import com.aliyun.odps.TableSchema
import com.aliyun.odps.data.Record
import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils, env}
import com.google.gson.GsonBuilder
import examples.dataloader.RequestContextOffline
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.sql.SparkSession
import com.aliyun.odps.spark.examples.makedata.makedata_06_originData.getFeatureFromSet
import com.alibaba.fastjson.JSONObject

import java.util
import java.util.concurrent.TimeUnit
import scala.collection.JavaConversions._


object makedata_09_user2redis {
  def main(args: Array[String]) {
    val spark = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .getOrCreate()
    val sc = spark.sparkContext

    // 1 读取参数
    val param = ParamUtils.parseArgs(args)
    val partitionPrefix = param.getOrElse("partitionPrefix", "dt=")
    val tablePart = param.getOrElse("tablePart", "64").toInt
    val ifUser = param.getOrDefault("ifUser", "False").toBoolean
    val ifVideo = param.getOrDefault("ifVideo", "False").toBoolean
    val date = param.getOrDefault("date", "20231220")
    val expireDay = param.getOrDefault("expireDay", "2").toInt
    val ifDebug = param.getOrDefault("ifDebug", "False").toBoolean
    val ifDeleteRedisUser = param.getOrDefault("ifDeleteRedisUser", "False").toBoolean
    val ifWriteRedisUser = param.getOrDefault("ifWriteRedisUser", "False").toBoolean
    val ifWriteRedis = param.getOrDefault("ifWriteRedis", "True").toBoolean
    val partition = partitionPrefix + date
    val savePathUser = param.getOrDefault("savePathUser", "")
    val savePathVideo = param.getOrDefault("savePathVideo", "")
    val userSampleIDs = param.getOrDefault("userSampleIDs", "")
    val sampleRate = param.getOrDefault("sampleRate", "1.0").toDouble
    val midDays = param.getOrDefault("midDays", "3").toInt


    // 2 读取数据库odps
    val odpsOps = env.getODPS(sc)
    val project = "loghubods"
    val tableUser = "alg_recsys_user_info"
    val userRedisKeyPrefix = "user_info_4video_"



    // 3-1 用户测特征处理
    if (ifUser){
      println("user特征处理")


      var userData = odpsOps.readTable(project = project, table = tableUser, partition = partition,
        transfer = func, numPartition = tablePart)
        .map(record =>{
          val userKey = "mids"
          val mid = record.getString(userKey)
          val originFeatureName = Set(
            "gender", "machineinfo_brand", "machineinfo_model", "machineinfo_platform", "machineinfo_sdkversion",
            "machineinfo_system", "machineinfo_wechatversion",
//            "gmt_create_user",
            "u_1day_exp_cnt", "u_1day_click_cnt", "u_1day_share_cnt", "u_1day_return_cnt",
            "u_3day_exp_cnt", "u_3day_click_cnt", "u_3day_share_cnt", "u_3day_return_cnt",
            "u_7day_exp_cnt", "u_7day_click_cnt", "u_7day_share_cnt", "u_7day_return_cnt",
            "u_3month_exp_cnt", "u_3month_click_cnt", "u_3month_share_cnt", "u_3month_return_cnt"
          )
          val originFeatureMap = getFeatureFromSet(originFeatureName, record)
          val resultNew = new JSONObject
          originFeatureName.foreach(r => {
            if (originFeatureMap.containsKey(r)) {
              val v = originFeatureMap.get(r).get
              resultNew.put(r, v)
            }
          })
          (mid, resultNew.toString())
        })
//      userData = userData.join(midRdd.map(r=> (r, 1))).map(r=> (r._1, r._2._1))

      if (userSampleIDs.nonEmpty){
        val IDs = userSampleIDs.split(",").filter(_.nonEmpty).map(_.toInt).toList
        userData = userData.filter(r => IDs.contains(r._1.hashCode % 10))
      }
      if (ifDebug){
        println("user特征处理-debug开启-只保留5条数据-特征数量大于1")
        val userDataTake = userData.take(5)
        userDataTake.foreach(r=> println(r._1 + "\t" + r._2))
        userData = sc.parallelize(userDataTake)
      }
      if (savePathUser.nonEmpty && savePathUser.startsWith("/dw/recommend/model/")) {
        var savePathPart = savePathUser + "/" + partition
        if (userSampleIDs.nonEmpty) {
          savePathPart = savePathPart + "_" + userSampleIDs
        }
        MyHdfsUtils.delete_hdfs_path(savePathPart)
        userData.map(r => r._1 + "\t" + r._2).saveAsTextFile(savePathPart, classOf[GzipCodec])
      }
      println("user.action.count=" + userData.count())
    } else {
      println("不处理user")
    }

    if (ifDeleteRedisUser){
      println("user redis 删除")
      var savePathPart = savePathUser + "/" + partition
      if (userSampleIDs.nonEmpty) {
        savePathPart = savePathPart + "_" + userSampleIDs
      }
      println("读取数据路径:" + savePathPart)
      val userDataRead = sc.textFile(savePathPart)
      val userDataRead2 = userDataRead.filter(_.split("\t").length >= 2).map(r => {
        val rList = r.split("\t")
        (rList(0), rList(1))
      })
      println("预计删除数据量:" + userDataRead2.count())
      val userDataTakeRddRun = userDataRead2.mapPartitions(row => {
        val redisFormat = new util.HashMap[String, String]
        val redisTemplate = env.getRedisTemplate()
        var i = 1
        row.foreach {
          case (key, value) =>
            if (key.nonEmpty) {
              redisFormat.put(userRedisKeyPrefix + key, value)
            }
            if (i % 1000 == 0) {
              redisTemplate.delete(redisFormat.map(_._1))
              redisFormat.clear()
            }
            i = i + 1
        }
        redisTemplate.delete(redisFormat.map(_._1))
        redisFormat.clear()
        redisFormat.iterator
      })
      println("delete redis.count=" + userDataTakeRddRun.count())
    } else {
      println("不处理user的redis删除")
    }

    if (ifWriteRedisUser){

      // 3-2 读取最近2个月有播放行为的mid集合
      var midRdd = sc.emptyRDD[String]
      MyDateUtils.getDateRange(MyDateUtils.getNumDaysBefore(date, midDays), date).foreach(d => {
        println("-----------读取播放信息:" + d)
        val partitionMid = "dt=" + d
        val data = odpsOps.readTable(project = "loghubods", table = "play_action_log",
            partition = partitionMid, transfer = func, numPartition = tablePart)
          .map(r => {
            if (r.isNull("machinecode")) "" else r.getString("machinecode")
          }).filter(_.nonEmpty)
        midRdd = midRdd.union(data).distinct()
      })
      println("------------mid处理完毕：" + midRdd.count() + "------------------")


      println("user redis 写入")
      var savePathPart = savePathUser + "/" + partition
      if (userSampleIDs.nonEmpty) {
        savePathPart = savePathPart + "_" + userSampleIDs
      }
      val userDataRead = sc.textFile(savePathPart).filter(_.split("\t").length >= 2)
        .sample(false, sampleRate)
        .map(r => {
        val rList = r.split("\t")
        (rList(0), rList(1))
      })
        .join(midRdd.map(r=> (r, 1))).map(r=> (r._1, r._2._1))

      val userDataTakeRddRun = userDataRead.mapPartitions(row => {
        val redisFormat = new util.HashMap[String, String]
        val redisTemplate = env.getRedisTemplate()
        var i = 1
        row.foreach {
          case (key, value) =>
            if (key.nonEmpty) {
              redisFormat.put(userRedisKeyPrefix + key, value)
            }
            if (i % 1000 == 0) {
              redisTemplate.opsForValue.multiSet(redisFormat)
              redisFormat.keySet.foreach(r => redisTemplate.expire(r, 24 * expireDay, TimeUnit.HOURS))
              redisFormat.clear()
            }
            i = i + 1
        }
        redisTemplate.opsForValue.multiSet(redisFormat)
        redisFormat.keySet.foreach(r => redisTemplate.expire(r, 24 * expireDay, TimeUnit.HOURS))
        redisFormat.clear()
        redisFormat.iterator
      })
      println("user写入成功：put in redis.count=" + userDataTakeRddRun.count())
    } else {
      println("不处理user的redis写入")
    }



  }

  def handleUser(record: Record, schema: TableSchema): Tuple3[String, String, Int] = {
    val userKey = "mids"
    val mid = record.getString(userKey)
    val reqContext: RequestContextOffline = new RequestContextOffline()
    reqContext.putUserFeature(record)
    // reqContext.featureMap.put("mid", mid)
    val gson = (new GsonBuilder).serializeSpecialFloatingPointValues.create
    val value = gson.toJson(reqContext.featureMap)
    (mid, value, reqContext.featureMap.size())
  }

  def func(record: Record, schema: TableSchema): Record = {
    record
  }

}