package com.aliyun.odps.spark.examples.makedata_ad.v20240718

import com.alibaba.fastjson.{JSON, JSONObject}
import com.aliyun.odps.TableSchema
import com.aliyun.odps.data.Record
import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, ParamUtils, env}
import examples.extractor.{ExtractorUtils, RankExtractorFeature_20240530}
import examples.utils.{AdUtil, DateTimeUtil, SimilarityUtils}
import org.apache.spark.sql.SparkSession

import java.time.{Instant, ZoneId, ZonedDateTime}
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import scala.io.Source
import scala.language.postfixOps
import scala.util.Random

object makedata_ad_33_bucketDataFromOriginToHive_20260703 {
  val CTR_SMOOTH_BETA_FACTOR = 25
  val CVR_SMOOTH_BETA_FACTOR = 10
  val CTCVR_SMOOTH_BETA_FACTOR = 100

  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .getOrCreate()
    val sc = spark.sparkContext


    // 1 读取参数
    val param = ParamUtils.parseArgs(args)
    val tablePart = param.getOrElse("tablePart", "64").toInt
    val beginStr = param.getOrElse("beginStr", "20250216")
    val endStr = param.getOrElse("endStr", "20250216")
    val project = param.getOrElse("project", "loghubods")
    val inputTable = param.getOrElse("inputTable", "alg_recsys_ad_sample_all")
    val outputTable = param.getOrElse("outputTable", "ad_easyrec_train_data_v1_sampled")
    val outputTable2 = param.getOrElse("outputTable2", "")
    val filterHours = param.getOrElse("filterHours", "00,01,02,03,04,05,06,07").split(",").toSet
    val idDefaultValue = param.getOrElse("idDefaultValue", "1.0").toDouble
    val filterNames = param.getOrElse("filterNames", "").split(",").filter(_.nonEmpty).toSet
    val filterAdverIds = param.getOrElse("filterAdverIds", "").split(",").filter(_.nonEmpty).toSet
//    val whatLabel = param.getOrElse("whatLabel", "ad_is_conversion")
    val negSampleRate = param.getOrElse("negSampleRate", "1").toDouble
    // 分割样本集的比例，splitRate部分输出至outputTable，补集输出至outputTable2（如果outputTable2不为空）
    val splitRate = param.getOrElse("splitRate", "0.9").toDouble
    val maskFeatureRate = param.getOrElse("maskFeatureRate", "0.0").toDouble
    val bucketFile = param.getOrElse("bucketFile", "20250217_ad_bucket_688.txt")
    val flag = param.getOrElse("flag", "0")

    val loader = getClass.getClassLoader
    val resourceUrlBucket = loader.getResource(bucketFile)
    val buckets =
      if (resourceUrlBucket != null) {
        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
        Source.fromURL(resourceUrlBucket).close()
        buckets
      } else {
        ""
      }
    val bucketsMap = buckets.split("\n")
      .map(r => r.replace(" ", "").replaceAll("\n", ""))
      .filter(r => r.nonEmpty)
      .map(r => {
        val rList = r.split("\t")
        val featureName = rList(0).replace("*", "_x_").replace("(view)", "_view")
        (featureName, (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
      }).toMap
    println(bucketsMap.keySet)
    val bucketsMap_br = sc.broadcast(bucketsMap)
    val denseFeatureNames = bucketsMap.keySet
    val lowerCaseDenseFeatureNames = bucketsMap.keySet.map(_.toLowerCase)
    val sparseFeatureNames = Set(
      "cid", "adid", "adverid", "targeting_conversion",
      "region", "city", "brand",
      "vid", "cate1", "cate2",
      "user_cid_click_list", "user_cid_conver_list",
      "user_vid_return_tags_2h", "user_vid_return_tags_1d", "user_vid_return_tags_3d", "user_vid_return_tags_7d",
      "user_vid_return_tags_14d", "apptype", "ts", "mid", "pqtid", "hour", "hour_quarter", "root_source_scene",
      "root_source_channel", "is_first_layer", "title_split", "profession", "user_vid_share_tags_1d", "user_vid_share_tags_14d",
      "user_vid_return_cate1_14d", "user_vid_return_cate2_14d", "user_vid_share_cate1_14d", "user_vid_share_cate2_14d",
      "creative_type", "creative_hook_embedding", "creative_why_embedding", "creative_action_embedding", "user_has_conver_1y",
      "user_adverid_view_3d", "user_adverid_view_7d", "user_adverid_view_30d",
      "user_adverid_click_3d", "user_adverid_click_7d", "user_adverid_click_30d",
      "user_adverid_conver_3d", "user_adverid_conver_7d", "user_adverid_conver_30d",
      "user_skuid_view_3d", "user_skuid_view_7d", "user_skuid_view_30d",
      "user_skuid_click_3d", "user_skuid_click_7d", "user_skuid_click_30d",
      "user_skuid_conver_3d", "user_skuid_conver_7d", "user_skuid_conver_30d",
      "is_weekday", "day_of_the_week", "user_conver_ad_class", "category_name",
      "material_md5", "user_layer", "user_class", "user_click_ad_class", "user_view_ad_class",
      "customer", "landing", "flag")
    val labelFields = Set(
      "has_click",
      "has_conversion",
      "has_scan",
      "has_addwechat",
      "has_scan_addwechat",
      "is_landing3",
      "is_landing3_with_has_scan"
    )


    // 2 读取odps+表信息
    val odpsOps = env.getODPS(sc)

    val tableSchema = odpsOps.getTableSchema(project, outputTable, isPartition = false)

    // 检查所有字段，收集非法字段
    val invalidFields = tableSchema.flatMap { case (fieldName, _) =>
      // 如果是标签字段，直接跳过不校验
      if (labelFields.contains(fieldName)) {
        None
      } else {
        // 否则，校验是否在特征列表里
        if (!lowerCaseDenseFeatureNames.contains(fieldName) && !sparseFeatureNames.contains(fieldName)) {
          Some(fieldName) // 收集未知字段
        } else {
          None
        }
      }
    }.toList

    // 如果存在非法字段，抛出标准异常
    if (invalidFields.nonEmpty) {
      throw new IllegalArgumentException(s"缺少字段: ${invalidFields.mkString(", ")}")
    }

    // 3 循环执行数据生产
    val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
    for (dt <- dateRange) {
      val timeRange = MyDateUtils.getDateHourRange(dt + "06", dt + "23")
      val recordRdd = timeRange.map { dt_hh =>
          val dt = dt_hh.substring(0, 8)
          val hh = dt_hh.substring(8, 10)
          val partition = s"dt=$dt,hh=$hh"
          if (filterHours.nonEmpty && filterHours.contains(hh)) {
            None
          } else {
            Some(partition)
          }
        }.collect {
          case Some(partition) => partition
        }.map(partition => {
          val odpsData = odpsOps.readTable(project = project,
              table = inputTable,
              partition = partition,
              transfer = func,
              numPartition = tablePart)
            .filter(record => {
              AdUtil.isApi(record)
            })
            .filter(record => {
              val extendAlg = Option(record.getString("extend_alg"))
                .filter(_.nonEmpty)
                .map(JSON.parseObject)
                .getOrElse(new JSONObject())
              Option(extendAlg.getString("extractstrategy")).contains("engine")
            })
            .filter(record => {
              val appType = record.getString("apptype")
              !Set("12", "13").contains(appType)
            })
            .filter(record => {
              val adverId = record.getString("adverid")
              !filterAdverIds.contains(adverId)
            })
            .filter(record => {
              val labelJson = getJsonObject(record, "label_json")
              val extendAlg = getJsonObject(record, "extend_alg")
              val hasScanLabel = labelJson.getString("ad_is_scan").toInt
              val hasConversionLabel = labelJson.getString("ad_is_conversion").toInt
              val isLanding3 = extendAlg.getString("landing_page_type") == "3"
              val randVal = Random.nextDouble()
              if (isLanding3) {
                (hasScanLabel > 0) || (randVal < negSampleRate)
              } else {
                hasConversionLabel > 0 || (randVal < negSampleRate)
              }
            })
            .map(record => {
              val featureMap = new JSONObject()
              val ts = record.getString("ts").toInt
              val instant = Instant.ofEpochSecond(ts)
              // 设置时区为中国时区
              val chinaZone = ZoneId.of("Asia/Shanghai")
              // 将 Instant 对象转换为中国时区的 ZonedDateTime 对象
              val zonedDateTime = ZonedDateTime.ofInstant(instant, chinaZone)
              // 获取星期几（1=周一，7=周日）
              val dayOfTheWeek = zonedDateTime.getDayOfWeek.getValue()
              val isWeekday = if (dayOfTheWeek <= 5) 1 else 2
              val cid = record.getString("cid")
              val mid = record.getString("mid")
              val pqtid = record.getString("pqtid")
              val apptype = record.getString("apptype")
              val targetingConversion = record.getString("targeting_conversion")
              featureMap.put("apptype", apptype)
              featureMap.put("ts", ts)
              featureMap.put("mid", mid)
              featureMap.put("pqtid", pqtid)
              featureMap.put("targeting_conversion", targetingConversion)
              val extend: JSONObject = if (record.isNull("extend")) new JSONObject() else
                JSON.parseObject(record.getString("extend"))
              val extendAlg: JSONObject = getJsonObject(record, "extend_alg")
              val mateFeature: JSONObject = if (record.isNull("metafeaturemap")) new JSONObject() else
                JSON.parseObject(record.getString("metafeaturemap"))
              val reqFeature: JSONObject = if (!mateFeature.containsKey("reqFeature")) new JSONObject() else
                mateFeature.getJSONObject("reqFeature")
              val sceneFeature: JSONObject = if (!mateFeature.containsKey("sceneFeature")) new JSONObject() else
                mateFeature.getJSONObject("sceneFeature")
              val b1: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_basic_info")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_basic_info")
              val b2: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_adver_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_adver_action")
              val b3: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_cid_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_cid_action")
              val b4: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_region_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_region_action")
              val b5: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_app_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_app_action")
              val b6: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_week_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_week_action")
              val b7: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_hour_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_hour_action")
              val b8: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_brand_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_brand_action")
              val b9: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_weChatVersion_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_weChatVersion_action")
              val j1: JSONObject = getJsonObject(record, "j1_feature") // user_layer
              val j2: JSONObject = getJsonObject(record, "j2_feature") // user_layer x advertiser
              val j3: JSONObject = getJsonObject(record, "j3_feature") // user_layer x customer
              val j4: JSONObject = getJsonObject(record, "j4_feature") // user_layer x profession
              val j5: JSONObject = getJsonObject(record, "j5_feature") // user_layer x category
              val j6: JSONObject = getJsonObject(record, "j6_feature") // user_layer x cid
              val j7: JSONObject = getJsonObject(record, "j7_feature") // landingpage
              val j8: JSONObject = getJsonObject(record, "j8_feature") // landingpage x advertiser
              val j9: JSONObject = getJsonObject(record, "j9_feature") // landingpage x customer
              val j10: JSONObject = getJsonObject(record, "j10_feature") // landingpage x profession
              val j11: JSONObject = getJsonObject(record, "j11_feature") // landingpage x category

              featureMap.put("cid_" + cid, idDefaultValue)
              if (b1.containsKey("adid") && b1.getString("adid").nonEmpty) {
                featureMap.put("adid_" + b1.getString("adid"), idDefaultValue)
              }
              if (b1.containsKey("adverid") && b1.getString("adverid").nonEmpty) {
                featureMap.put("adverid_" + b1.getString("adverid"), idDefaultValue)
              }
              if (b1.containsKey("targeting_conversion") && b1.getString("targeting_conversion").nonEmpty) {
                featureMap.put("targeting_conversion_" + b1.getString("targeting_conversion"), idDefaultValue)
              }
              if (b1.containsKey("creative_type") && b1.getString("creative_type").nonEmpty) {
                featureMap.put("creative_type", b1.getString("creative_type"))
              }
              if (b1.containsKey("creative_hook_embedding") && b1.getString("creative_hook_embedding").nonEmpty) {
                featureMap.put("creative_hook_embedding", b1.getString("creative_hook_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
              }
              if (b1.containsKey("creative_why_embedding") && b1.getString("creative_why_embedding").nonEmpty) {
                featureMap.put("creative_why_embedding", b1.getString("creative_why_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
              }
              if (b1.containsKey("creative_action_embedding") && b1.getString("creative_action_embedding").nonEmpty) {
                featureMap.put("creative_action_embedding", b1.getString("creative_action_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
              }
              if (extendAlg.containsKey("customer_id")) {
                featureMap.put("customer", extendAlg.getString("customer_id"))
              }
              if (sceneFeature.containsKey("hour") && sceneFeature.getString("hour").nonEmpty) {
                featureMap.put("hour", sceneFeature.getString("hour"))
              }
              if (sceneFeature.containsKey("hour_quarter") && sceneFeature.getString("hour_quarter").nonEmpty) {
                featureMap.put("hour_quarter", sceneFeature.getString("hour_quarter"))
              }
              featureMap.put("is_weekday", isWeekday)
              featureMap.put("day_of_the_week", dayOfTheWeek)

              val hour = DateTimeUtil.getHourByTimestamp(ts)
              featureMap.put("hour_" + hour, idDefaultValue)

              val dayOfWeek = DateTimeUtil.getDayOrWeekByTimestamp(ts)
              featureMap.put("dayofweek_" + dayOfWeek, idDefaultValue);

              featureMap.put("apptype_" + apptype, idDefaultValue);

              if (extend.containsKey("abcode") && extend.getString("abcode").nonEmpty) {
                featureMap.put("abcode_" + extend.getString("abcode"), idDefaultValue)
              }

              // 定义需要处理的键名列表
              val reqFeatureKeys = List(
                "cid", "adid", "adverid", "profession", "region",
                "city", "is_first_layer", "root_source_scene",
                "root_source_channel", "brand", "vid", "category_name", "material_md5"
              )

              // 使用函数式方式处理所有键
              reqFeatureKeys.foreach { key =>
                val value = reqFeature.getString(key)

                // 检查值是否非空
                if (value != null && value.nonEmpty) {
                  featureMap.put(key, value)
                }
              }
              if (extendAlg.containsKey("landing_page_type")) {
                featureMap.put("landing", extendAlg.getString("landing_page_type"))
              }
              if (reqFeature.containsKey("layer_l4")) {
                featureMap.put("user_layer", reqFeature.getString("layer_l4"))
              }
              if (reqFeature.containsKey("clazz_l4")) {
                featureMap.put("user_class", reqFeature.getString("clazz_l4"))
              }
              if (b1.containsKey("cpa")) {
                featureMap.put("cpa", b1.getString("cpa").toDouble)
              }
              if (b1.containsKey("weight") && b1.getString("weight").nonEmpty) {
                featureMap.put("weight", b1.getString("weight").toDouble)
              }

              for ((bn, prefix1) <- List(
                (b2, "b2"), (b3, "b3"), (b4, "b4"), (b5, "b5"), (b8, "b8"), (b9, "b9")
              )) {
                for (prefix2 <- List(
                  "3h", "6h", "12h", "1d", "3d", "7d", "today", "yesterday"
                )) {
                  val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
                  val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
                  val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
                  val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
                  // NOTE(zhoutian):
                  // 这里cpc只是为了计算cpm的平滑的工具量，没有实际业务意义，因为cpm并非比率，本身不适合直接计算Wilson平滑
                  // 不使用cpa的原因是未来可能出现广告采用cpc计费的情况或者无法获取转化量的情况，用点击更为稳定
                  // 其它几组特征亦采用相同逻辑
                  // 2025-02-17改为增加固定分母平滑，income实际已经可以直接参与cpm平滑计算
                  val cpc = if (click == 0) 0D else income / click
                  val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
                  val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
                  val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
                  val f4 = conver
                  val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)

                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
                }
              }

              for ((bn, prefix1) <- List(
                (b6, "b6"), (b7, "b7")
              )) {
                for (prefix2 <- List(
                  "7d", "14d"
                )) {
                  val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
                  val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
                  val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
                  val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
                  val cpc = if (click == 0) 0D else income / click
                  val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
                  val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
                  val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
                  val f4 = conver
                  val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)

                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
                }
              }


              val c1: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_ad_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_mid_feature_ad_action")

              val midActionList = if (c1.containsKey("action") && c1.getString("action").nonEmpty) {
                c1.getString("action").split(",").map(r => {
                  val rList = r.split(":")
                  (rList(0), (rList(1).toInt, rList(2).toInt, rList(3).toInt, rList(4).toInt, rList(5)))
                }).sortBy(-_._2._1).toList
              } else {
                new ArrayBuffer[(String, (Int, Int, Int, Int, String))]().toList
              }
              // u特征
              val viewAll = midActionList.size.toDouble
              val clickAll = midActionList.map(_._2._2).sum.toDouble
              val converAll = midActionList.map(_._2._3).sum.toDouble
              val incomeAll = midActionList.map(_._2._4).sum.toDouble
              featureMap.put("viewAll", viewAll)
              featureMap.put("clickAll", clickAll)
              featureMap.put("converAll", converAll)
              featureMap.put("incomeAll", incomeAll)
              featureMap.put("ctr_all", RankExtractorFeature_20240530.calDiv(clickAll, viewAll))
              featureMap.put("ctcvr_all", RankExtractorFeature_20240530.calDiv(converAll, viewAll))
              featureMap.put("cvr_all", RankExtractorFeature_20240530.calDiv(clickAll, converAll))
              featureMap.put("ecpm_all", RankExtractorFeature_20240530.calDiv(incomeAll * 1000, viewAll))

              if (c1.containsKey("user_has_conver_1y") && c1.getInteger("user_has_conver_1y") != null) {
                featureMap.put("user_has_conver_1y", c1.getInteger("user_has_conver_1y"))
              }
              if (c1.containsKey("user_conver_ad_class") && c1.getString("user_conver_ad_class") != null) {
                featureMap.put("user_conver_ad_class", c1.getString("user_conver_ad_class"))
              }
              if (c1.containsKey("user_click_ad_class") && c1.getString("user_click_ad_class") != null) {
                featureMap.put("user_click_ad_class", c1.getString("user_click_ad_class"))
              }
              if (c1.containsKey("user_view_ad_class") && c1.getString("user_view_ad_class") != null) {
                featureMap.put("user_view_ad_class", c1.getString("user_view_ad_class"))
              }

              // ui特征
              val midTimeDiff = scala.collection.mutable.Map[String, Double]()
              midActionList.foreach {
                case (cid, (ts_history, click, conver, income, title)) =>
                  if (!midTimeDiff.contains("timediff_view_" + cid)) {
                    midTimeDiff.put("timediff_view_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
                  }
                  if (!midTimeDiff.contains("timediff_click_" + cid) && click > 0) {
                    midTimeDiff.put("timediff_click_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
                  }
                  if (!midTimeDiff.contains("timediff_conver_" + cid) && conver > 0) {
                    midTimeDiff.put("timediff_conver_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
                  }
              }

              val midActionStatic = scala.collection.mutable.Map[String, Double]()
              midActionList.foreach {
                case (cid, (ts_history, click, conver, income, title)) =>
                  midActionStatic.put("actionstatic_view_" + cid, 1.0 + midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0))
                  midActionStatic.put("actionstatic_click_" + cid, click + midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0))
                  midActionStatic.put("actionstatic_conver_" + cid, conver + midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0))
                  midActionStatic.put("actionstatic_income_" + cid, income + midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0))
              }

              val clickCidList = collection.mutable.ListBuffer[String]()
              val converCidList = collection.mutable.ListBuffer[String]()
              midActionList.foreach {
                case (cid, (ts_history, click, conver, income, title)) =>
                  if (click == 1) clickCidList += cid
                  if (conver == 1) converCidList += cid
              }
              if (clickCidList.nonEmpty) {
                featureMap.put("user_cid_click_list", clickCidList.takeRight(50).mkString(","))
              } else {
                featureMap.put("user_cid_click_list", "")
              }
              if (converCidList.nonEmpty) {
                featureMap.put("user_cid_conver_list", converCidList.takeRight(50).mkString(","))
              } else {
                featureMap.put("user_cid_conver_list", "")
              }
              if (midTimeDiff.contains("timediff_view_" + cid)) {
                featureMap.put("timediff_view", midTimeDiff.getOrDefault("timediff_view_" + cid, 0.0))
              }
              if (midTimeDiff.contains("timediff_click_" + cid)) {
                featureMap.put("timediff_click", midTimeDiff.getOrDefault("timediff_click_" + cid, 0.0))
              }
              if (midTimeDiff.contains("timediff_conver_" + cid)) {
                featureMap.put("timediff_conver", midTimeDiff.getOrDefault("timediff_conver_" + cid, 0.0))
              }
              if (midActionStatic.contains("actionstatic_view_" + cid)) {
                featureMap.put("actionstatic_view", midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0))
              }
              if (midActionStatic.contains("actionstatic_click_" + cid)) {
                featureMap.put("actionstatic_click", midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0))
              }
              if (midActionStatic.contains("actionstatic_conver_" + cid)) {
                featureMap.put("actionstatic_conver", midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0))
              }
              if (midActionStatic.contains("actionstatic_income_" + cid)) {
                featureMap.put("actionstatic_income", midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0))
              }
              if (midActionStatic.contains("actionstatic_view_" + cid) && midActionStatic.contains("actionstatic_click_" + cid)) {
                featureMap.put("actionstatic_ctr", RankExtractorFeature_20240530.calDiv(
                  midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
                  midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
                ))
              }
              if (midActionStatic.contains("actionstatic_view_" + cid) && midActionStatic.contains("actionstatic_conver_" + cid)) {
                featureMap.put("actionstatic_ctcvr", RankExtractorFeature_20240530.calDiv(
                  midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
                  midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
                ))
              }
              if (midActionStatic.contains("actionstatic_conver_" + cid) && midActionStatic.contains("actionstatic_click_" + cid)) {
                featureMap.put("actionstatic_cvr", RankExtractorFeature_20240530.calDiv(
                  midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
                  midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)
                ))
              }

              val e1: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_return_tags")) new JSONObject() else
                mateFeature.getJSONObject("alg_mid_feature_return_tags")
              val e2: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_share_tags")) new JSONObject() else
                mateFeature.getJSONObject("alg_mid_feature_share_tags")
              val title = b1.getOrDefault("cidtitle", "").toString
              if (title.nonEmpty) {
                for ((en, prefix1) <- List((e1, "e1"), (e2, "e2"))) {
                  for (prefix2 <- List("tags_3d", "tags_7d", "tags_14d")) {
                    if (en.nonEmpty && en.containsKey(prefix2) && en.getString(prefix2).nonEmpty) {
                      val (f1, f2, f3, f4) = funcC34567ForTagsNew(en.getString(prefix2), title)
                      featureMap.put(prefix1 + "_" + prefix2 + "_matchnum", f1)
                      featureMap.put(prefix1 + "_" + prefix2 + "_maxscore", f3)
                      featureMap.put(prefix1 + "_" + prefix2 + "_avgscore", f4)

                    }
                  }
                }
              }

              if (e1.containsKey("tags_2h") && e1.getString("tags_2h").nonEmpty) {
                featureMap.put("user_vid_return_tags_2h", e1.getString("tags_2h"))
              }
              if (e1.containsKey("tags_1d") && e1.getString("tags_1d").nonEmpty) {
                featureMap.put("user_vid_return_tags_1d", e1.getString("tags_1d"))
              }
              if (e1.containsKey("tags_3d") && e1.getString("tags_3d").nonEmpty) {
                featureMap.put("user_vid_return_tags_3d", e1.getString("tags_3d"))
              }
              if (e1.containsKey("tags_7d") && e1.getString("tags_7d").nonEmpty) {
                featureMap.put("user_vid_return_tags_7d", e1.getString("tags_7d"))
              }
              if (e1.containsKey("tags_14d") && e1.getString("tags_14d").nonEmpty) {
                featureMap.put("user_vid_return_tags_14d", e1.getString("tags_14d"))
              }

              if (e2.containsKey("tags_1d") && e2.getString("tags_1d").nonEmpty) {
                featureMap.put("user_vid_share_tags_1d", e2.getString("tags_1d"))
              }
              if (e2.containsKey("tags_14d") && e2.getString("tags_14d").nonEmpty) {
                featureMap.put("user_vid_share_tags_14d", e2.getString("tags_14d"))
              }

              val g1: JSONObject = if (!mateFeature.containsKey("mid_return_video_cate")) new JSONObject() else
                mateFeature.getJSONObject("mid_return_video_cate")
              val g2: JSONObject = if (!mateFeature.containsKey("mid_share_video_cate")) new JSONObject() else
                mateFeature.getJSONObject("mid_share_video_cate")
              if (g1.containsKey("cate1_14d") && g1.getString("cate1_14d").nonEmpty) {
                featureMap.put("user_vid_return_cate1_14d", g1.getString("cate1_14d"))
              }
              if (g1.containsKey("cate2_14d") && g1.getString("cate2_14d").nonEmpty) {
                featureMap.put("user_vid_return_cate2_14d", g1.getString("cate2_14d"))
              }
              if (g2.containsKey("cate1_14d") && g2.getString("cate1_14d").nonEmpty) {
                featureMap.put("user_vid_share_cate1_14d", g2.getString("cate1_14d"))
              }
              if (g2.containsKey("cate2_14d") && g2.getString("cate2_14d").nonEmpty) {
                featureMap.put("user_vid_share_cate2_14d", g2.getString("cate2_14d"))
              }

              val h1: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_adver_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_mid_feature_adver_action")
              val h2: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_sku_action")) new JSONObject() else
                mateFeature.getJSONObject("alg_mid_feature_sku_action")

              // 定义时间维度和对应的前缀
              val timeDimensions = Seq("3d", "7d", "30d")
              for (dimension <- timeDimensions) {
                if (h1.containsKey(dimension) && h1.getString(dimension).nonEmpty) {
                  val action = h1.getString(dimension).split(",")
                  if (action.length >= 3) {
                    featureMap.put(s"user_adverid_view_${dimension}", action(0))
                    featureMap.put(s"user_adverid_click_${dimension}", action(1))
                    featureMap.put(s"user_adverid_conver_${dimension}", action(2))
                  }
                }
                if (h2.containsKey(dimension) && h2.getString(dimension).nonEmpty) {
                  val action = h2.getString(dimension).split(",")
                  if (action.length >= 3) {
                    featureMap.put(s"user_skuid_view_${dimension}", action(0))
                    featureMap.put(s"user_skuid_click_${dimension}", action(1))
                    featureMap.put(s"user_skuid_conver_${dimension}", action(2))
                  }
                }
              }


              val d1: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_vid_cf")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_vid_cf")
              val d2: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_vid_cf_rank")) new JSONObject() else
                mateFeature.getJSONObject("alg_cid_feature_vid_cf_rank")
              val d3: JSONObject = if (!mateFeature.containsKey("alg_vid_feature_basic_info")) new JSONObject() else
                mateFeature.getJSONObject("alg_vid_feature_basic_info")

              if (d1.nonEmpty) {
                for (prefix <- List("3h", "6h", "12h", "1d", "3d", "7d")) {
                  val view = if (!d1.containsKey("ad_view_" + prefix)) 0D else d1.getIntValue("ad_view_" + prefix).toDouble
                  val click = if (!d1.containsKey("ad_click_" + prefix)) 0D else d1.getIntValue("ad_click_" + prefix).toDouble
                  val conver = if (!d1.containsKey("ad_conversion_" + prefix)) 0D else d1.getIntValue("ad_conversion_" + prefix).toDouble
                  val income = if (!d1.containsKey("ad_income_" + prefix)) 0D else d1.getIntValue("ad_income_" + prefix).toDouble
                  val cpc = if (click == 0) 0D else income / click
                  val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
                  val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
                  val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
                  val f4 = conver
                  val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
                  featureMap.put("d1_feature" + "_" + prefix + "_" + "ctr", f1)
                  featureMap.put("d1_feature" + "_" + prefix + "_" + "ctcvr", f2)
                  featureMap.put("d1_feature" + "_" + prefix + "_" + "cvr", f3)
                  featureMap.put("d1_feature" + "_" + prefix + "_" + "conver", f4)
                  featureMap.put("d1_feature" + "_" + prefix + "_" + "ecpm", f5)
                }
              }

              val vidRankMaps = scala.collection.mutable.Map[String, scala.collection.immutable.Map[String, Double]]()
              if (d2.nonEmpty) {
                d2.foreach(r => {
                  val key = r._1
                  val value = d2.getString(key).split(",").map(r => {
                    val rList = r.split(":")
                    (rList(0), rList(2).toDouble)
                  }).toMap
                  vidRankMaps.put(key, value)
                })
              }
              for (prefix1 <- List("ctr", "ctcvr", "ecpm")) {
                for (prefix2 <- List("1d", "3d", "7d", "14d")) {
                  if (vidRankMaps.contains(prefix1 + "_" + prefix2)) {
                    val rank = vidRankMaps(prefix1 + "_" + prefix2).getOrDefault(cid, 0.0)
                    if (rank >= 1.0) {
                      featureMap.put("vid_rank_" + prefix1 + "_" + prefix2, 1.0 / rank)
                    }
                  }
                }
              }

              if (d3.nonEmpty) {
                val vTitle = d3.getString("title")
                featureMap.put("cate1", d3.getOrDefault("merge_first_level_cate", ""))
                featureMap.put("cate2", d3.getOrDefault("merge_second_level_cate", ""))
                featureMap.put("title_split", d3.getOrDefault("title_split", ""))
              }

              // 随机mask部分特征供模型训练
              if (Random.nextDouble() < maskFeatureRate) {
                featureMap.put("cid", "")
                featureMap.put("adid", "")
                featureMap.put("adverid", "")
                featureMap.put("customer", "")
              }
              featureMap.put("flag", flag)

              val jList: List[(String, JSONObject, List[String])] = List(
                ("j1", j1, List("3h", "3d")),
                ("j2", j2, List("3h", "3d")),
                ("j3", j3, List("3h", "3d")),
                ("j4", j4, List("3h", "3d")),
                ("j5", j5, List("3h", "3d")),
                ("j6", j6, List("1h", "2h", "3h", "6h", "12h", "1d", "3d", "today", "yesterday")),
                ("j7", j7, List("3h", "3d")),
                ("j8", j8, List("3h", "3d")),
                ("j9", j9, List("3h", "3d")),
                ("j10", j10, List("3h", "3d")),
                ("j11", j11, List("3h", "3d"))
              )
              for ((prefix1, bn, periods) <- jList) {
                for (prefix2 <- periods) {
                  val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
                  val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
                  val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
                  val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
                  val cpc = if (click == 0) 0D else income / click
                  val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
                  val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
                  val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
                  val f4 = conver
                  val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)

                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
                }
              }

              /*
            广告
              sparse：cid adid adverid targeting_conversion

              cpa --> 1个
              adverid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr conver ecpm  --> 30个
              cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
              地理//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
              app//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
              手机品牌//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
              系统 无数据
              week//cid下的 7d 14d、 ctr ctcvr cvr ecpm conver --> 10个
              hour//cid下的 7d 14d、 ctr ctcvr cvr ecpm conver --> 10个

            用户
              用户历史 点击/转化 的title tag；3d 7d 14d； cid的title； 数量/最高分/平均分 --> 18个
              用户历史 14d 看过/点过/转化次数/income； ctr cvr ctcvr ecpm；  --> 8个

              用户到cid的ui特征 --> 10个
                1/用户最近看过这个cid的时间间隔
                1/用户最近点过这个cid的时间间隔
                1/用户最近转过这个cid的时间间隔
                用户看过这个cid多少次
                用户点过这个cid多少次
                用户转过这个cid多少次
                用户对这个cid花了多少钱
                用户对这个cid的ctr ctcvr cvr

            视频
              vid//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
              vid//cid下的 1d 3d 7d 14d、 ctr ctcvr ecpm 的rank值 倒数 --> 12个

             */


              //4 处理label信息。
              val labels = new JSONObject
              val landing = featureMap.getString("landing")
              val labelJson: JSONObject = getJsonObject(record, "label_json") // label_json
              if (!labelJson.isEmpty) {
                labels.put("has_scan", labelJson.getInteger("ad_is_scan"))
                labels.put("has_addwechat", labelJson.getInteger("ad_is_addwechat"))
                labels.put("has_scan_addwechat", labelJson.getInteger("ad_is_addwechat"))
                if (landing == "3") {
                  labels.put("is_landing3", 1)
                  if (labelJson.getIntValue("ad_is_scan") == 1 && targetingConversion == "10004") {
                    labels.put("is_landing3_with_has_scan", 1)
                  } else {
                    labels.put("is_landing3_with_has_scan", 0)
                  }
                } else {
                  labels.put("is_landing3", 0)
                  labels.put("is_landing3_with_has_scan", 0)
                }
                labels.put("has_click", labelJson.getIntValue("ad_is_click"))
                labels.put("has_conversion", labelJson.getIntValue("ad_is_conversion"))
              }
              //5 处理log key表头。
              val headvideoid = record.getString("headvideoid")
              val logKey = (apptype, mid, cid, ts, headvideoid).productIterator.mkString(",")
              val labelKey = labels.toString()
              (logKey, labelKey, featureMap)
            })
          odpsData
        }).reduce(_ union _)
        .map { case (logKey, labelKey, jsons) =>
          val denseFeatures = scala.collection.mutable.Map[String, Double]()
          val sparseFeatures = scala.collection.mutable.Map[String, String]()
          denseFeatureNames.foreach(r => {
            if (jsons.containsKey(r)) {
              denseFeatures.put(r, jsons.getDoubleValue(r))
            }
          })
          sparseFeatureNames.foreach(r => {
            if (jsons.get(r) != null) {
              sparseFeatures.put(r, jsons.get(r).toString)
            }
          })
          (logKey, labelKey, denseFeatures, sparseFeatures)
        }
        .map {
          case (logKey, labelKey, denseFeatures, sparseFeatures) =>
            val labelObject = JSON.parseObject(labelKey)
//            val label = labelObject.getOrDefault(whatLabel, "0").toString
            val bucketsMap = bucketsMap_br.value
            var resultMap = denseFeatures.collect {
              case (name, score) if !filterNames.exists(name.contains) && score > 1E-8 =>
                val value = if (bucketsMap.contains(name)) {
                  val (bucketsNum, buckets) = bucketsMap(name)
                  1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
                } else {
                  score
                }
                name -> value.toString
            }.toMap
            sparseFeatures.foreach(kv => {
              resultMap += (kv._1 -> kv._2)
            })
            resultMap += ("has_click" -> labelObject.getString("has_click"))
            resultMap += ("has_conversion" -> labelObject.getString("has_conversion"))
            resultMap += ("has_scan" -> labelObject.getString("has_scan"))
            resultMap += ("has_addwechat" -> labelObject.getString("has_addwechat"))
            resultMap += ("has_scan_addwechat" -> labelObject.getString("has_scan_addwechat"))
            resultMap += ("is_landing3" -> labelObject.getString("is_landing3"))
            resultMap += ("is_landing3_with_has_scan" -> labelObject.getString("is_landing3_with_has_scan"))
            resultMap += ("logkey" -> logKey)
            resultMap
        }.coalesce(128)

      val partition = s"dt=$dt"
      if (outputTable2.isEmpty) {
        odpsOps.saveToTable(project, outputTable, partition, recordRdd, write, defaultCreate = true, overwrite = true)
      } else {
        // 固定seed以保证可重入
        val splitRdds = recordRdd.randomSplit(Array(splitRate, 1 - splitRate), seed = dt.toLong)
        odpsOps.saveToTable(project, outputTable, partition, splitRdds(0), write, defaultCreate = true, overwrite = true)
        odpsOps.saveToTable(project, outputTable2, partition, splitRdds(1), write, defaultCreate = true, overwrite = true)
      }
    }
  }

  def write(map: Map[String, String], record: Record, schema: TableSchema): Unit = {
    for ((columnName, value) <- map) {
      try {
        // 查找列名在表结构中的索引
        val columnIndex = schema.getColumnIndex(columnName.toLowerCase)
        // 获取列的类型
        val columnType = schema.getColumn(columnIndex).getTypeInfo
        try {
          columnType.getTypeName match {
            case "STRING" =>
              record.setString(columnIndex, value)
            case "BIGINT" =>
              record.setBigint(columnIndex, value.toLong)
            case "DOUBLE" =>
              record.setDouble(columnIndex, value.toDouble)
            case "BOOLEAN" =>
              record.setBoolean(columnIndex, value.toBoolean)
            case other =>
              throw new IllegalArgumentException(s"Unsupported column type: $other")
          }
        } catch {
          case e: NumberFormatException =>
            println(s"Error converting value $value to type ${columnType.getTypeName} for column $columnName: ${e.getMessage}")
          case e: Exception =>
            println(s"Unexpected error writing value $value to column $columnName: ${e.getMessage}")
        }
      } catch {
        case e: IllegalArgumentException => {
          println(e.getMessage)
        }
      }
    }
  }


  def func(record: Record, schema: TableSchema): Record = {
    record
  }

  def funcC34567ForTagsNew(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
    // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
    val tagsList = tags.split(",")
    var d1 = 0.0
    val d2 = new ArrayBuffer[String]()
    var d3 = 0.0
    var d4 = 0.0
    for (tag <- tagsList) {
      if (title.contains(tag)) {
        d1 = d1 + 1.0
        d2.add(tag)
      }
      val score = SimilarityUtils.word2VecSimilarity(tag, title)
      d3 = if (score > d3) score else d3
      d4 = d4 + score
    }
    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
    (d1, d2.mkString(","), d3, d4)
  }

  def getJsonObject(record: Record, name: String): JSONObject = {
    if (record.isNull(name)) {
      new JSONObject()
    } else {
      JSON.parseObject(record.getString(name))
    }
  }
}
