package com.aliyun.odps.spark.examples.makedata_dssm

import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.aliyun.odps.TableSchema
import com.aliyun.odps.data.Record
import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils, env}
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.sql.SparkSession
import examples.extractor.ExtractorUtils
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import scala.io.Source

object makedata_i2i_05_trainData_20241129 {
  def func(record: Record, schema: TableSchema): Record = {
    record
  }
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .getOrCreate()
    val sc = spark.sparkContext

    // 1 读取参数
    val param = ParamUtils.parseArgs(args)
    val beginStr = param.getOrElse("beginStr", "2024062008")
    val endStr = param.getOrElse("endStr", "2024062023")
    val readPath = param.getOrElse("readPath", "/dw/recommend/model/52_dssm_i2i_joinfeature/")
    val savePath = param.getOrElse("savePath", "/dw/recommend/model/55_dssm_i2i_traindata/")
    val onehotPath = param.getOrElse("onehotPath", "/dw/recommend/model/53_dssm_i2i_onehot/20241128")
    val bucketFile = param.getOrElse("bucketFile", "20241128_recsys_i2i_bucket_47_v2.txt")
    val repartition = param.getOrElse("repartition", "100").toInt
    val filterHours = param.getOrElse("filterHours", "25").split(",").toSet
    val ifDebug = param.getOrElse("ifDebug", "false").toBoolean

    // 2 读取onehot文件
    val onehotMap_br = sc.broadcast(
      sc.textFile(onehotPath).map(r => {
        val rList = r.split("\t")
        (rList(0), rList(1))
      }).collectAsMap()
    )

    // 3 读取dense分桶文件
    val resourceUrlBucket = this.getClass.getClassLoader.getResource(bucketFile)
    val buckets =
      if (resourceUrlBucket != null) {
        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
        Source.fromURL(resourceUrlBucket).close()
        buckets
      } else {
        ""
      }
    println(buckets)
    val bucketsMap_br = sc.broadcast(
      buckets.split("\n")
        .map(r => r.replace(" ", "").replaceAll("\n", ""))
        .filter(r => r.nonEmpty)
        .map(r => {
          val rList = r.split("\t")
          (rList(0), (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
        }).toMap
    )

    // 4 循环执行数据生产
    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
    for (dt_hh <- timeRange) {
      val dt = dt_hh.substring(0, 8)
      val hh = dt_hh.substring(8, 10)
      val data = sc.textFile(readPath + "/" + dt_hh).map(r=>{
        val rList = r.split("\t")
        val logKey = rList(0)
        val label = rList(1)
        val vid_left = rList(2)
        val vid_right = rList(3)
        val feature_left = rList(4)
        val feature_right = rList(5)
        val feature_left_action = rList(6)
        val feature_right_action = rList(7)
        val feature_left_cate1 = rList(8)
        val feature_right_cate1 = rList(9)
        val feature_left_cate2 = rList(10)
        val feature_right_cate2 = rList(11)
        (logKey, label, vid_left, vid_right, feature_left, feature_right, feature_left_action, feature_right_action,
          feature_left_cate1, feature_right_cate1, feature_left_cate2, feature_right_cate2)
      }).mapPartitions(row =>{
        val result = new ArrayBuffer[String]()
        val onehotMap = onehotMap_br.value
        val bucketsMap = bucketsMap_br.value
        row.foreach{
          case (logKey, label, vid_left, vid_right, feature_left, feature_right, feature_left_action, feature_right_action,
          feature_left_cate1, feature_right_cate1, feature_left_cate2, feature_right_cate2) =>
            val left = new ArrayBuffer[String]()
            val right = new ArrayBuffer[String]()
            val left_dense1 = new ArrayBuffer[String]()
            val right_dense1 = new ArrayBuffer[String]()
            val left_dense2 = new ArrayBuffer[String]()
            val right_dense2 = new ArrayBuffer[String]()
            // 1 sparse 特征 16个
            // vid cate1 cate2 video_style valid_time captions_color audience_age_group
            // audience_value_type font_size cover_persons_num audience_gender sentiment_tendency
            // video_type background_music_type captions has_end_credit_guide
            left += onehotMap.getOrElse("vid:" + vid_left, "0")
            right += onehotMap.getOrElse("vid:" + vid_right, "0")
            var jsonLeft = JSON.parseObject(feature_left)
            left += getOnehotValue(jsonLeft, onehotMap, "category1", "cate1:")
            left += getOnehotValue(jsonLeft, onehotMap, "category2_1", "cate2:")
            left += getOnehotValue(jsonLeft, onehotMap, "video_style", "video_style:")
            left += getOnehotValue(jsonLeft, onehotMap, "valid_time", "valid_time:")
            left += getOnehotValue(jsonLeft, onehotMap, "captions_color", "captions_color:")
            left += getOnehotValue(jsonLeft, onehotMap, "audience_age_group", "audience_age_group:")
            left += getOnehotValue(jsonLeft, onehotMap, "audience_value_type", "audience_value_type:")
            left += getOnehotValue(jsonLeft, onehotMap, "font_size", "font_size:")
            left += getOnehotValue(jsonLeft, onehotMap, "cover_persons_num", "cover_persons_num:")
            left += getOnehotValue(jsonLeft, onehotMap, "audience_gender", "audience_gender:")
            left += getOnehotValue(jsonLeft, onehotMap, "sentiment_tendency", "sentiment_tendency:")
            left += getOnehotValue(jsonLeft, onehotMap, "video_type", "video_type:")
            left += getOnehotValue(jsonLeft, onehotMap, "background_music_type", "background_music_type:")
            left += getOnehotValue(jsonLeft, onehotMap, "captions", "captions:")
            left += getOnehotValue(jsonLeft, onehotMap, "has_end_credit_guide", "has_end_credit_guide:")
            var jsonRight = JSON.parseObject(feature_right)
            right += getOnehotValue(jsonRight, onehotMap, "category1", "cate1:")
            right += getOnehotValue(jsonRight, onehotMap, "category2_1", "cate2:")
            right += getOnehotValue(jsonRight, onehotMap, "video_style", "video_style:")
            right += getOnehotValue(jsonRight, onehotMap, "valid_time", "valid_time:")
            right += getOnehotValue(jsonRight, onehotMap, "captions_color", "captions_color:")
            right += getOnehotValue(jsonRight, onehotMap, "audience_age_group", "audience_age_group:")
            right += getOnehotValue(jsonRight, onehotMap, "audience_value_type", "audience_value_type:")
            right += getOnehotValue(jsonRight, onehotMap, "font_size", "font_size:")
            right += getOnehotValue(jsonRight, onehotMap, "cover_persons_num", "cover_persons_num:")
            right += getOnehotValue(jsonRight, onehotMap, "audience_gender", "audience_gender:")
            right += getOnehotValue(jsonRight, onehotMap, "sentiment_tendency", "sentiment_tendency:")
            right += getOnehotValue(jsonRight, onehotMap, "video_type", "video_type:")
            right += getOnehotValue(jsonRight, onehotMap, "background_music_type", "background_music_type:")
            right += getOnehotValue(jsonRight, onehotMap, "captions", "captions:")
            right += getOnehotValue(jsonRight, onehotMap, "has_end_credit_guide", "has_end_credit_guide:")
            // 2 dense通过分桶转换成sparse特征 47个 * 3 * 2
            jsonLeft = JSON.parseObject(feature_left_action)
            var res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day1", "action:str_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day1", "action:rov_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day1", "action:ros_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day7", "action:str_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day7", "action:rov_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day7", "action:ros_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day21", "action:str_day21")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day21", "action:rov_day21")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day21", "action:ros_day21")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day336", "action:str_day336")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day336", "action:rov_day336")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day336", "action:ros_day336")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day7", "action:vovd1_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day21", "action:vovd1_day21")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day336", "action:vovd1_day336")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString

            jsonRight = JSON.parseObject(feature_right_action)
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day1", "action:str_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day1", "action:rov_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day1", "action:ros_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day7", "action:str_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day7", "action:rov_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day7", "action:ros_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day21", "action:str_day21")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day21", "action:rov_day21")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day21", "action:ros_day21")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day336", "action:str_day336")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day336", "action:rov_day336")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day336", "action:ros_day336")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day7", "action:vovd1_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day21", "action:vovd1_day21")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day336", "action:vovd1_day336")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString

            //----------------------cate1-----------------------------cate1---------------------------cate1----------------------
            jsonLeft = JSON.parseObject(feature_left_cate1)
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day1", "cate1:str_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day1", "cate1:rov_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day1", "cate1:ros_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day3", "cate1:str_day3")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day3", "cate1:rov_day3")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day3", "cate1:ros_day3")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day7", "cate1:str_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day7", "cate1:rov_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day7", "cate1:ros_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day30", "cate1:str_day30")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day30", "cate1:rov_day30")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day30", "cate1:ros_day30")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day1", "cate1:vovd1_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day3", "cate1:vovd1_day3")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day7", "cate1:vovd1_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day30", "cate1:vovd1_day30")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString

            jsonRight = JSON.parseObject(feature_right_cate1)
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day1", "cate1:str_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day1", "cate1:rov_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day1", "cate1:ros_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day3", "cate1:str_day3")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day3", "cate1:rov_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day3", "cate1:ros_day3")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day7", "cate1:str_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day7", "cate1:rov_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day7", "cate1:ros_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day30", "cate1:str_day30")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day30", "cate1:rov_day30")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day30", "cate1:ros_day30")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day1", "cate1:vovd1_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day3", "cate1:vovd1_day3")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day7", "cate1:vovd1_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day30", "cate1:vovd1_day30")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString

          //----------------------cate2-----------------------------cate2---------------------------cate2----------------------
            jsonLeft = JSON.parseObject(feature_left_cate2)
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day1", "cate2:str_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day1", "cate2:rov_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day1", "cate2:ros_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day3", "cate2:str_day3")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day3", "cate2:rov_day3")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day3", "cate2:ros_day3")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day7", "cate2:str_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day7", "cate2:rov_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day7", "cate2:ros_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "str_day30", "cate2:str_day30")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "rov_day30", "cate2:rov_day30")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "ros_day30", "cate2:ros_day30")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day1", "cate2:vovd1_day1")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day3", "cate2:vovd1_day3")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day7", "cate2:vovd1_day7")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString
            res = getDenseBucketValue(jsonLeft, bucketsMap, "vovd1_day30", "cate2:vovd1_day30")
            left += res._1.toString
            left_dense1 += res._2.toString
            left_dense2 += res._3.toString

            jsonRight = JSON.parseObject(feature_right_cate2)
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day1", "cate2:str_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day1", "cate2:rov_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day1", "cate2:ros_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day3", "cate2:str_day3")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day3", "cate2:rov_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day3", "cate2:ros_day3")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day7", "cate2:str_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day7", "cate2:rov_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day7", "cate2:ros_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "str_day30", "cate2:str_day30")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "rov_day30", "cate2:rov_day30")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "ros_day30", "cate2:ros_day30")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day1", "cate2:vovd1_day1")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day3", "cate2:vovd1_day3")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day7", "cate2:vovd1_day7")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString
            res = getDenseBucketValue(jsonRight, bucketsMap, "vovd1_day30", "cate2:vovd1_day30")
            right += res._1.toString
            right_dense1 += res._2.toString
            right_dense2 += res._3.toString

          // 3 left 和 right 分别 16+47*3=16+141 = 157
            left ++= left_dense1
            left ++= left_dense2
            right ++= right_dense1
            right ++= right_dense2

            result.add(
              (logKey, label, vid_left, vid_right, left.mkString(","), right.mkString(",")).productIterator.mkString("\t")
            )
        }
        result.iterator
      })

      // 4 保存数据到hdfs
      val hdfsPath = savePath + "/" + dt_hh
      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
        println("删除路径并开始数据写入:" + hdfsPath)
        MyHdfsUtils.delete_hdfs_path(hdfsPath)
        data.coalesce(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
      } else {
        println("路径不合法，无法写入:" + hdfsPath)
      }
    }
  }

  def getOnehotValue(obj: JSONObject, m: scala.collection.Map[String, String], key1: String, key2: String): String = {
    if (obj.containsKey(key1)) {
      val value1 = obj.get(key1)
      val value2 = if (value1 == null) "无" else value1.toString
      m.getOrElse(key2 + value2, "0")
    } else {
      "0"
    }
  }
  def getDenseBucketValue(obj: JSONObject, bucketsMap: Map[String, (Double, Array[Double])], key1: String, name: String): (Int, Double, Double) = {
    if (obj.containsKey(key1)) {
      val value1 = obj.get(key1)
      val score = try {
        value1.toString.toDouble
      } catch {
        case _: Exception => 0D
      }
      if (score > 1E-8) {
        if (bucketsMap.contains(name)) {
          val (bucketsNum, buckets) = bucketsMap(name)
          val index = ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0
          val scoreNew = 1.0 / bucketsNum * index
          (index.toInt, scoreNew, score)
        } else {
          (0, 0D, score)
        }
      } else {
        (0, 0D, 0D)
      }
    } else {
      (0, 0D, 0D)
    }
  }
}
