去掉无用信息

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/make_rov_online_feature.scala

@@ -1,113 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.{JSON, JSONObject}
-import com.aliyun.odps.TableSchema
-import com.aliyun.odps.data.Record
-import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils, env}
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-
-/*
-   20250109 提取特征
- */
-
-object make_rov_online_feature {
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-
-    val beginStr = param.getOrElse("beginStr", "2025011015")
-    val endStr = param.getOrElse("endStr", "2025011015")
-    val project = param.getOrElse("project", "loghubods")
-    val table = param.getOrElse("table", "statistics_log_hour")
-    val tablePart = param.getOrElse("tablePart", "64").toInt
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/61_recsys_rov_train_data/")
-    val repartition = param.getOrElse("repartition", "100").toInt
-
-    // 2 odps
-    val odpsOps = env.getODPS(sc)
-
-    // 3 循环执行数据生产
-    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
-    for (dt_hh <- timeRange) {
-      val dt = dt_hh.substring(0, 8)
-      val hh = dt_hh.substring(8, 10)
-      val partition = s"dt=$dt$hh"
-      println("开始执行partiton:" + partition)
-      val odpsData = odpsOps.readTable(project = project,
-          table = table,
-          partition = partition,
-          transfer = func,
-          numPartition = tablePart)
-        .filter(record => {
-          val apptype = record.getString("apptype")
-          val scoresMap = JSON.parseObject(record.getString("scoresmap"))
-          "31".equals(apptype) && scoresMap.containsKey("RovXGBScore")
-        })
-        .mapPartitions(p => {
-          p.map(record => {
-            // a 特征
-            val jsons = getJsonObject(record, "allfeaturemap")
-            val features = scala.collection.mutable.Map[String, Double]()
-            jsons.foreach(r => {
-              features.put(r._1, jsons.getDoubleValue(r._1))
-            })
-
-            val featuresBucket = features.map {
-              case (name, score) =>
-                name + ":" + score.toString
-            }.filter(_.nonEmpty)
-
-            //5 处理log key表头。
-            val apptype = record.getString("apptype")
-            val pagesource = record.getString("pagesource")
-            val mid = record.getString("mid")
-            val vid = if (record.isNull("videoid")) "" else record.getString("videoid")
-
-            val recommendtraceid = record.getString("recommendtraceid")
-            val traceid = record.getString("traceid")
-            val level = if (record.isNull("level")) "0" else record.getString("level")
-            val logKey = (apptype, pagesource, mid, vid, recommendtraceid, traceid, level).productIterator.mkString(",")
-
-            val scoresMap = JSON.parseObject(record.getString("scoresmap")).toString()
-            //6 拼接数据，保存。
-            logKey + "\t" + "1" + "\t" + scoresMap + "\t" + featuresBucket.mkString("\t")
-          })
-        })
-
-      // 4 保存数据到hdfs
-      val savePartition = dt
-      val hdfsPath = savePath + "/" + savePartition
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        odpsData.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  def func(record: Record, schema: TableSchema): Record = {
-    record
-  }
-
-  def getJsonObject(record: Record, key: String): JSONObject = {
-    val data = if (record.isNull(key)) new JSONObject() else JSON.parseObject(record.getString(key))
-    val data2 = new JSONObject()
-    data.foreach(r => {
-      if (r._2 != null) {
-        data2.put(r._1, r._2)
-      }
-    })
-    data2
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_61_bucket_20241209.scala

@@ -1,102 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys
-
-import com.alibaba.fastjson.JSON
-import com.aliyun.odps.spark.examples.myUtils.{MyHdfsUtils, ParamUtils}
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.io.Source
-
-/*
-
- */
-
-object makedata_recsys_61_bucket_20241209 {
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    val loader = getClass.getClassLoader
-    val resourceUrl = loader.getResource("20241209_recsys_feature_name_756.txt")
-    val content =
-      if (resourceUrl != null) {
-        val content = Source.fromURL(resourceUrl).getLines().mkString("\n")
-        Source.fromURL(resourceUrl).close()
-        content
-      } else {
-        ""
-      }
-    println(content)
-    val contentList = content.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty).toList
-
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val readPath = param.getOrElse("readPath", "/dw/recommend/model/61_origin_data/20241210*")
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/61_recsys_bucket/")
-    val fileName = param.getOrElse("fileName", "20241209_756_200")
-    val sampleRate = param.getOrElse("sampleRate", "1.0").toDouble
-    val bucketNum = param.getOrElse("bucketNum", "200").toInt
-
-    val data = sc.textFile(readPath)
-    println("问题数据数量：" + data.filter(r => r.split("\t").length < 3).count())
-    val data1 = data.map(r => {
-      val rList = r.split("\t")
-      val jsons = JSON.parseObject(rList(rList.length - 1))
-      val doubles = scala.collection.mutable.Map[String, Double]()
-      jsons.foreach(r => {
-        doubles.put(r._1, jsons.getDoubleValue(r._1))
-      })
-      doubles
-    }).sample(false, sampleRate).repartition(32).persist()
-
-    val result = new ArrayBuffer[String]()
-
-    for (i <- contentList.indices) {
-      println("特征:" + contentList(i))
-      val data2 = data1.map(r => r.getOrDefault(contentList(i), 0D)).filter(_ > 1E-8).collect().sorted
-      val len = data2.length
-      if (len == 0) {
-        result.add(contentList(i) + "\t" + bucketNum.toString + "\t" + "0")
-      } else {
-        val oneBucketNum = (len - 1) / (bucketNum - 1) + 1 // 确保每个桶至少有一个元素
-        val buffers = new ArrayBuffer[Double]()
-
-        var lastBucketValue = data2(0) // 记录上一个桶的切分点
-        for (j <- 0 until len by oneBucketNum) {
-          val d = data2(j)
-          if (j > 0 && d != lastBucketValue) {
-            // 如果当前切分点不同于上一个切分点，则保存当前切分点
-            buffers += d
-          }
-          lastBucketValue = d // 更新上一个桶的切分点
-        }
-
-        // 最后一个桶的结束点应该是数组的最后一个元素
-        if (!buffers.contains(data2.last)) {
-          buffers += data2.last
-        }
-        result.add(contentList(i) + "\t" + bucketNum.toString + "\t" + buffers.mkString(","))
-      }
-    }
-    val data3 = sc.parallelize(result)
-
-
-    // 4 保存数据到hdfs
-    val hdfsPath = savePath + "/" + fileName
-    if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-      println("删除路径并开始数据写入:" + hdfsPath)
-      MyHdfsUtils.delete_hdfs_path(hdfsPath)
-      data3.repartition(1).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-    } else {
-      println("路径不合法，无法写入:" + hdfsPath)
-    }
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_61_nor_sample_20241209.scala

@@ -1,149 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.JSON
-import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils}
-import examples.extractor.ExtractorUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.io.Source
-import scala.util.Random
-
-object makedata_recsys_61_nor_sample_20241209 {
-  def main(args: Array[String]): Unit = {
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val readPath = param.getOrElse("readPath", "/dw/recommend/model/61_origin_data/")
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/61_recsys_nor_train_data/")
-    val beginStr = param.getOrElse("beginStr", "20241210")
-    val endStr = param.getOrElse("endStr", "20241210")
-    val repartition = param.getOrElse("repartition", "100").toInt
-    val whatLabel = param.getOrElse("whatLabel", "total_return_uv_new")
-    val whatApps = param.getOrElse("whatApps", "0,4,5,21,3,6").split(",").toSet
-    val fuSampleRate = param.getOrElse("fuSampleRate", "-1.0").toDouble
-    val notUseBucket = param.getOrElse("notUseBucket", "0").toInt
-    val featureNameFile = param.getOrElse("featureName", "20241209_recsys_nor_name.txt")
-    val featureBucketFile = param.getOrElse("featureBucket", "20241209_recsys_nor_bucket.txt")
-
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    val loader = getClass.getClassLoader
-    val featureNameSet = loadUseFeatureNames(loader, featureNameFile)
-    val featureBucketMap = loadUseFeatureBuckets(loader, notUseBucket, featureBucketFile)
-    val bucketsMap_br = sc.broadcast(featureBucketMap)
-
-    val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
-    for (date <- dateRange) {
-      println("开始执行:" + date)
-      val data = sc.textFile(readPath + "/" + date + "*").map(r => {
-          val rList = r.split("\t")
-          val logKey = rList(0)
-          val labelKey = rList(1)
-          val scoresMap = rList(2)
-          val jsons = JSON.parseObject(rList(3))
-          val features = scala.collection.mutable.Map[String, Double]()
-          jsons.foreach(r => {
-            features.put(r._1, jsons.getDoubleValue(r._1))
-          })
-          (logKey, labelKey, scoresMap, features)
-        })
-        .filter {
-          case (logKey, labelKey, scoresMap, features) =>
-            val logKeyList = logKey.split(",")
-            val apptype = logKeyList(0)
-            val pagesource = logKeyList(1)
-            whatApps.contains(apptype) && pagesource.endsWith("recommend")
-        }.filter {
-          case (logKey, labelKey, scoresMap, features) =>
-            val label = JSON.parseObject(labelKey).getOrDefault(whatLabel, "0").toString.toDouble
-            label > 0 || new Random().nextDouble() <= fuSampleRate
-        }
-        .map {
-          case (logKey, labelKey, scoresMap, features) =>
-            val label = JSON.parseObject(labelKey).getOrDefault(whatLabel, "0").toString.toDouble
-            (logKey, label, scoresMap, features)
-        }
-        .mapPartitions(row => {
-          val result = new ArrayBuffer[String]()
-          val bucketsMap = bucketsMap_br.value
-          row.foreach {
-            case (logKey, label, scoresMap, features) =>
-              val featuresBucket = features.map {
-                case (name, score) =>
-                  if (!featureNameSet.contains(name)) {
-                    ""
-                  } else {
-                    if (score > 1E-8) {
-                      if (bucketsMap.contains(name)) {
-                        val (bucketsNum, buckets) = bucketsMap(name)
-                        val scoreNew = 1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
-                        name + ":" + scoreNew.toString
-                      } else {
-                        name + ":" + score.toString
-                      }
-                    } else {
-                      ""
-                    }
-                  }
-              }.filter(_.nonEmpty)
-              result.add(logKey + "\t" + label + "\t" + scoresMap + "\t" + featuresBucket.mkString("\t"))
-          }
-          result.iterator
-        })
-
-      // 4 保存数据到hdfs
-      val hdfsPath = savePath + "/" + date
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        data.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  private def loadFileData(loader: ClassLoader, nameFile: String): String = {
-    val resourceUrlBucket = loader.getResource(nameFile)
-    val data =
-      if (resourceUrlBucket != null) {
-        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
-        Source.fromURL(resourceUrlBucket).close()
-        buckets
-      } else {
-        ""
-      }
-    data
-  }
-
-  private def loadUseFeatureNames(loader: ClassLoader, nameFile: String): Set[String] = {
-    val names = loadFileData(loader, nameFile)
-    println(names)
-    names.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .toSet
-  }
-
-  private def loadUseFeatureBuckets(loader: ClassLoader, notUseBucket: Int, nameFile: String): Map[String, (Double, Array[Double])] = {
-    if (notUseBucket > 0) {
-      return Map[String, (Double, Array[Double])]()
-    }
-    val buckets = loadFileData(loader, nameFile)
-    println(buckets)
-    buckets.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .map(r => {
-        val rList = r.split("\t")
-        (rList(0), (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
-      }).toMap
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_61_originData_20241209.scala

@@ -1,473 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.{JSON, JSONObject}
-import com.aliyun.odps.TableSchema
-import com.aliyun.odps.data.Record
-import com.aliyun.odps.spark.examples.myUtils._
-import examples.extractor.RankExtractorFeature_20240530
-import examples.utils.SimilarityUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import java.util.Calendar
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.util.Random
-
-/*
-   20241211 提取特征
- */
-
-object makedata_recsys_61_originData_20241209 {
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-
-    val beginStr = param.getOrElse("beginStr", "2024120912")
-    val endStr = param.getOrElse("endStr", "2024120912")
-    val project = param.getOrElse("project", "loghubods")
-    val table = param.getOrElse("table", "alg_recsys_sample_all_v2")
-    val tablePart = param.getOrElse("tablePart", "64").toInt
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/61_origin_data/")
-    val repartition = param.getOrElse("repartition", "32").toInt
-    val whatLabel = param.getOrElse("whatLabel", "is_share")
-    val fuSampleRate = param.getOrElse("fuSampleRate", "0.1").toDouble
-    val smoothPlus = param.getOrElse("smoothPlus", "5").toDouble
-
-    // 2 odps
-    val odpsOps = env.getODPS(sc)
-
-    // 3 循环执行数据生产
-    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
-    for (dt_hh <- timeRange) {
-      val dt = dt_hh.substring(0, 8)
-      val hh = dt_hh.substring(8, 10)
-      val partition = s"dt=$dt,hh=$hh"
-      println("开始执行partiton:" + partition)
-      val odpsData = odpsOps.readTable(project = project,
-          table = table,
-          partition = partition,
-          transfer = func,
-          numPartition = tablePart)
-        .filter(record => {
-          val label = if (record.isNull(whatLabel)) "0" else record.getString(whatLabel)
-          "1".equals(label) || new Random().nextDouble() <= fuSampleRate
-        })
-        .mapPartitions(p => {
-          SimilarityUtils.init()
-          p.map(record => {
-            val featureMap = new JSONObject()
-
-            // a 视频特征
-            val b1: JSONObject = getJsonObject(record, "b1_feature")
-            val b2: JSONObject = getJsonObject(record, "b2_feature")
-            val b3: JSONObject = getJsonObject(record, "b3_feature")
-            val b6: JSONObject = getJsonObject(record, "b6_feature")
-            val b7: JSONObject = getJsonObject(record, "b7_feature")
-
-            val b8: JSONObject = getJsonObject(record, "b8_feature")
-            val b9: JSONObject = getJsonObject(record, "b9_feature")
-            val b10: JSONObject = getJsonObject(record, "b10_feature")
-            val b11: JSONObject = getJsonObject(record, "b11_feature")
-            val b12: JSONObject = getJsonObject(record, "b12_feature")
-            val b13: JSONObject = getJsonObject(record, "b13_feature")
-            val b17: JSONObject = getJsonObject(record, "b17_feature")
-            val b18: JSONObject = getJsonObject(record, "b18_feature")
-            val b19: JSONObject = getJsonObject(record, "b19_feature")
-
-            val origin_data = List(
-              (b1, b2, b3, "b123"), (b1, b6, b7, "b167"),
-              (b8, b9, b10, "b8910"), (b11, b12, b13, "b111213"),
-              (b17, b18, b19, "b171819")
-            )
-            for ((b_1, b_2, b_3, prefix1) <- origin_data) {
-              for (prefix2 <- List(
-                "1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d"
-              )) {
-                val exp = if (b_1.isEmpty) 0D else b_1.getIntValue("exp_pv_" + prefix2).toDouble
-                val share = if (b_2.isEmpty) 0D else b_2.getIntValue("share_pv_" + prefix2).toDouble
-                val returns = if (b_3.isEmpty) 0D else b_3.getIntValue("return_uv_" + prefix2).toDouble
-                val f0 = RankExtractorFeature_20240530.calLog(exp)
-                val f1 = RankExtractorFeature_20240530.calDiv(share, exp)
-                val f2 = RankExtractorFeature_20240530.calLog(share)
-                val f3 = RankExtractorFeature_20240530.calDiv(returns, exp)
-                val f4 = RankExtractorFeature_20240530.calLog(returns)
-                val f5 = f3 * f4
-                val f6 = RankExtractorFeature_20240530.calDiv(returns, share)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "EXP", f0)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "STR", f1)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "log(share)", f2)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROV", f3)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "log(return)", f4)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROV*log(return)", f5)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROS", f6)
-              }
-            }
-
-            val video_info: JSONObject = getJsonObject(record, "t_v_info_feature")
-            featureMap.put("total_time", if (video_info.containsKey("total_time")) video_info.getIntValue("total_time").toDouble else 0D)
-            featureMap.put("bit_rate", if (video_info.containsKey("bit_rate")) video_info.getIntValue("bit_rate").toDouble else 0D)
-
-            val c1: JSONObject = getJsonObject(record, "c1_feature")
-            if (c1.nonEmpty) {
-              featureMap.put("playcnt_6h", if (c1.containsKey("playcnt_6h")) c1.getIntValue("playcnt_6h").toDouble else 0D)
-              featureMap.put("playcnt_1d", if (c1.containsKey("playcnt_1d")) c1.getIntValue("playcnt_1d").toDouble else 0D)
-              featureMap.put("playcnt_3d", if (c1.containsKey("playcnt_3d")) c1.getIntValue("playcnt_3d").toDouble else 0D)
-              featureMap.put("playcnt_7d", if (c1.containsKey("playcnt_7d")) c1.getIntValue("playcnt_7d").toDouble else 0D)
-            }
-            val c2: JSONObject = getJsonObject(record, "c2_feature")
-            if (c2.nonEmpty) {
-              featureMap.put("share_pv_12h", if (c2.containsKey("share_pv_12h")) c2.getIntValue("share_pv_12h").toDouble else 0D)
-              featureMap.put("share_pv_1d", if (c2.containsKey("share_pv_1d")) c2.getIntValue("share_pv_1d").toDouble else 0D)
-              featureMap.put("share_pv_3d", if (c2.containsKey("share_pv_3d")) c2.getIntValue("share_pv_3d").toDouble else 0D)
-              featureMap.put("share_pv_7d", if (c2.containsKey("share_pv_7d")) c2.getIntValue("share_pv_7d").toDouble else 0D)
-              featureMap.put("return_uv_12h", if (c2.containsKey("return_uv_12h")) c2.getIntValue("return_uv_12h").toDouble else 0D)
-              featureMap.put("return_uv_1d", if (c2.containsKey("return_uv_1d")) c2.getIntValue("return_uv_1d").toDouble else 0D)
-              featureMap.put("return_uv_3d", if (c2.containsKey("return_uv_3d")) c2.getIntValue("return_uv_3d").toDouble else 0D)
-              featureMap.put("return_uv_7d", if (c2.containsKey("return_uv_7d")) c2.getIntValue("return_uv_7d").toDouble else 0D)
-            }
-
-            val title = if (video_info.containsKey("title")) video_info.getString("title") else ""
-            if (!title.equals("")) {
-              for (key_feature <- List("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                val c34567: JSONObject = if (record.isNull(key_feature)) new JSONObject() else
-                  JSON.parseObject(record.getString(key_feature))
-                for (key_time <- List("tags_1d", "tags_3d", "tags_7d")) {
-                  val tags = if (c34567.containsKey(key_time)) c34567.getString(key_time) else ""
-                  if (!tags.equals("")) {
-                    val (f1, f2, f3, f4) = funcC34567ForTagsW2V(tags, title)
-                    featureMap.put(key_feature + "_" + key_time + "_matchnum", f1)
-                    featureMap.put(key_feature + "_" + key_time + "_maxscore", f3)
-                    featureMap.put(key_feature + "_" + key_time + "_avgscore", f4)
-                  }
-                }
-              }
-            }
-
-            val vid = if (record.isNull("vid")) "" else record.getString("vid")
-            if (!vid.equals("")) {
-              for (key_feature <- List("c8_feature", "c9_feature")) {
-                val c89: JSONObject = if (record.isNull(key_feature)) new JSONObject() else
-                  JSON.parseObject(record.getString(key_feature))
-                for (key_action <- List("share", "return")) {
-                  val cfListStr = if (c89.containsKey(key_action)) c89.getString(key_action) else ""
-                  if (!cfListStr.equals("")) {
-                    val cfMap = cfListStr.split(",").map(r => {
-                      val rList = r.split(":")
-                      (rList(0), (rList(1), rList(2), rList(3)))
-                    }).toMap
-                    if (cfMap.contains(vid)) {
-                      val (score, num, rank) = cfMap(vid)
-                      featureMap.put(key_feature + "_" + key_action + "_score", score.toDouble)
-                      featureMap.put(key_feature + "_" + key_action + "_num", num.toDouble)
-                      featureMap.put(key_feature + "_" + key_action + "_rank", 1.0 / rank.toDouble)
-                    }
-                  }
-                }
-              }
-            }
-
-            val d1: JSONObject = getJsonObject(record, "d1_feature")
-            if (d1.nonEmpty) {
-              featureMap.put("d1_exp", if (d1.containsKey("exp")) d1.getString("exp").toDouble else 0D)
-              featureMap.put("d1_return_n", if (d1.containsKey("return_n")) d1.getString("return_n").toDouble else 0D)
-              featureMap.put("d1_rovn", if (d1.containsKey("rovn")) d1.getString("rovn").toDouble else 0D)
-            }
-
-            // ************* new feature *************
-            val shortPeriod = List("1h", "2h", "4h", "6h", "12h", "24h", "7d")
-            val middlePeriod = List("14d", "30d")
-            val longPeriod = List("7d", "35d", "90d", "365d")
-            val vidStatFeat = List(
-              ("b20", shortPeriod, getJsonObject(record, "b20_feature")), // cate2_feature
-              ("b21", shortPeriod, getJsonObject(record, "b21_feature")), // cate1_feature
-              ("b22", shortPeriod, getJsonObject(record, "b22_feature")), // source_feature
-              ("b28", shortPeriod, getJsonObject(record, "b28_feature")), // sence_type_feature
-              ("b29", shortPeriod, getJsonObject(record, "b29_feature")), // alg_videoid_feature
-              ("b23", middlePeriod, getJsonObject(record, "b23_feature")), // cate2_feature_day
-              ("b24", middlePeriod, getJsonObject(record, "b24_feature")), // cate1_feature_day
-              ("b25", middlePeriod, getJsonObject(record, "b25_feature")), // source_feature_day
-              ("b26", longPeriod, getJsonObject(record, "b26_feature")), // unionid_feature_day
-              ("b27", longPeriod, getJsonObject(record, "b27_feature")) // vid_feature_day
-            )
-            for ((featType, featPeriod, featData) <- vidStatFeat) {
-              for (period <- featPeriod) {
-                val view = if (featData.isEmpty) 0D else featData.getDoubleValue("view_" + period)
-                val share = if (featData.isEmpty) 0D else featData.getDoubleValue("share_" + period)
-                val return_ = if (featData.isEmpty) 0D else featData.getDoubleValue("return_" + period)
-                val view_hasreturn = if (featData.isEmpty) 0D else featData.getDoubleValue("view_hasreturn_" + period)
-                val share_hasreturn = if (featData.isEmpty) 0D else featData.getDoubleValue("share_hasreturn_" + period)
-                var ros = if (featData.isEmpty) 0D else featData.getDoubleValue("ros_" + period)
-                var rov = if (featData.isEmpty) 0D else featData.getDoubleValue("rov_" + period)
-                var r_cnt = if (featData.isEmpty) 0D else featData.getDoubleValue("r_cnt_" + period)
-                var r_rate = if (featData.isEmpty) 0D else featData.getDoubleValue("r_rate_" + period)
-                var r_cnt4s = if (featData.isEmpty) 0D else featData.getDoubleValue("r_cnt4s_" + period)
-                var str = if (featData.isEmpty) 0D else featData.getDoubleValue("str_" + period)
-                var s2r_rate = 0D
-                if (smoothPlus > 0) {
-                  ros = RankExtractorFeature_20240530.smoothDiv(return_, share, smoothPlus)
-                  rov = RankExtractorFeature_20240530.smoothDiv(return_, view, smoothPlus)
-                  r_cnt = RankExtractorFeature_20240530.smoothDiv(return_, view_hasreturn, smoothPlus)
-                  r_rate = RankExtractorFeature_20240530.smoothDiv(view_hasreturn, view, smoothPlus)
-                  r_cnt4s = RankExtractorFeature_20240530.smoothDiv(return_, share_hasreturn, smoothPlus)
-                  str = RankExtractorFeature_20240530.smoothDiv(share, view, smoothPlus)
-                  s2r_rate = RankExtractorFeature_20240530.smoothDiv(share_hasreturn, share, smoothPlus)
-                }
-
-                // scale
-                val view_s = RankExtractorFeature_20240530.calLog(view)
-                val share_s = RankExtractorFeature_20240530.calLog(share)
-                val return_s = RankExtractorFeature_20240530.calLog(return_)
-                val view_hasreturn_s = RankExtractorFeature_20240530.calLog(view_hasreturn)
-                val share_hasreturn_s = RankExtractorFeature_20240530.calLog(share_hasreturn)
-
-                featureMap.put(featType + "_" + period + "_" + "view", view_s)
-                featureMap.put(featType + "_" + period + "_" + "share", share_s)
-                featureMap.put(featType + "_" + period + "_" + "return", return_s)
-                featureMap.put(featType + "_" + period + "_" + "view_hasreturn", view_hasreturn_s)
-                featureMap.put(featType + "_" + period + "_" + "share_hasreturn", share_hasreturn_s)
-                featureMap.put(featType + "_" + period + "_" + "ros", ros)
-                featureMap.put(featType + "_" + period + "_" + "rov", rov)
-                featureMap.put(featType + "_" + period + "_" + "r_cnt", r_cnt)
-                featureMap.put(featType + "_" + period + "_" + "r_rate", r_rate)
-                featureMap.put(featType + "_" + period + "_" + "r_cnt4s", r_cnt4s)
-                featureMap.put(featType + "_" + period + "_" + "str", str)
-                featureMap.put(featType + "_" + period + "_" + "s2r_rate", s2r_rate)
-              }
-            }
-
-            // new cf
-            val d2345Data = List(
-              ("d2", "rosn", getJsonObject(record, "d2_feature")),
-              ("d3", "rosn", getJsonObject(record, "d3_feature")),
-              ("d4", "rovn", getJsonObject(record, "d4_feature")),
-              ("d5", "rovn", getJsonObject(record, "d5_feature"))
-            )
-            for ((featType, valType, featData) <- d2345Data) {
-              if (featData.nonEmpty) {
-                val exp = if (featData.containsKey("exp")) featData.getString("exp").toDouble else 0D
-                val return_n = if (featData.containsKey("return_n")) featData.getString("return_n").toDouble else 0D
-                val value = if (featData.containsKey(valType)) featData.getString(valType).toDouble else 0D
-                // scale
-                val exp_s = RankExtractorFeature_20240530.calLog(exp)
-                val return_n_s = RankExtractorFeature_20240530.calLog(return_n)
-                featureMap.put(featType + "_exp", exp_s)
-                featureMap.put(featType + "_return_n", return_n_s)
-                featureMap.put(featType + "_" + valType, value)
-              }
-            }
-
-            if (!vid.equals("")) {
-              val idScoreObj = getJsonObject(getJsonObject(record, "d6_feature"), "vids", "scores")
-              if (idScoreObj.nonEmpty && idScoreObj.containsKey(vid)) {
-                val score = idScoreObj.getString(vid).toDouble
-                featureMap.put("d6", score)
-              }
-            }
-
-            // head video & rank video
-            val headVideo = getJsonObject(record, "v2_feature")
-            val rankVideo = getJsonObject(record, "v1_feature")
-            if (headVideo.nonEmpty && rankVideo.nonEmpty) {
-              val videoAttrs = List("title", "topic", "keywords", "cate1_list", "cate2", "cate2_list", "style", "theme", "user_value")
-              for (attr <- videoAttrs) {
-                val headAttr = if (headVideo.containsKey(attr)) headVideo.getString(attr) else ""
-                val rankAttr = if (rankVideo.containsKey(attr)) rankVideo.getString(attr) else ""
-                if (!headAttr.equals("") && !rankAttr.equals("")) {
-                  val simScore = SimilarityUtils.word2VecSimilarity(headAttr, rankAttr)
-                  featureMap.put("video_sim_" + attr, simScore)
-                }
-              }
-            }
-
-            // category
-            if (rankVideo.nonEmpty) {
-              val attrList = List(FeatureUtils.cate1Attr, FeatureUtils.cate2Attr, FeatureUtils.festive1Attr)
-              for (attr <- attrList) {
-                val attrVal = if (rankVideo.containsKey(attr)) rankVideo.getString(attr) else ""
-                val attrId = FeatureUtils.getAttrId(attr, attrVal)
-                if (attrId > 0) {
-                  val key = "%s@%s@%d".format("r", attr, attrId)
-                  featureMap.put(key, 1)
-                }
-              }
-            }
-
-            // time
-            val ts = record.getString("ts")
-            val calendar = tsToCalendar(ts)
-            if (null != calendar) {
-              val week = calendar.get(Calendar.DAY_OF_WEEK)
-              val hour = calendar.get(Calendar.HOUR_OF_DAY) + 1
-              featureMap.put("week", week)
-              featureMap.put("hour", hour)
-            }
-
-            /*
-            视频特征： 5*6*5 = 240个
-                      曝光使用pv 分享使用pv 回流使用uv --> 1h 2h 3h 4h 12h 1d 3d 7d
-                      STR log(share) ROV log(return) ROV*log(return) ROS
-                      整体、整体曝光对应、推荐非冷启root、推荐冷启root、分省份root
-            视频基础： 2个   视频时长、比特率
-            用户： 4+8 = 12个
-                      播放次数 --> 6h 1d 3d 7d --> 4个
-                      带回来的分享pv 回流uv --> 12h 1d 3d 7d --> 8个
-            人+vid-title:  5*3*3 = 45
-                      播放点/回流点/分享点/累积分享/累积回流 --> 1d 3d 7d --> 匹配数量 语义最高相似度分 语义平均相似度分 --> 45个
-            人+vid-cf： 2*3*3 = 12
-                      基于分享行为/基于回流行为 -->  “分享cf”+”回流点击cf“ 相似分 相似数量 相似rank的倒数 --> 12个
-            头部视频：  3
-                      曝光 回流 ROVn 3个特征
-            场景：     小时 星期 apptype city province pagesource 机器型号
-            总量：     240+2+12+45+12+3 = 314
-            ---------------------------------------------------------------
-            视频特征：(4*7+3*2+2*4)*10 = 420个
-            CF: 13个
-            视频相似特征: 9个
-
-
-             */
-
-
-            //4 处理label信息。
-            val labels = new JSONObject
-            for (labelKey <- List(
-              "is_play", "is_share", "is_return", "noself_is_return", "return_uv", "noself_return_uv", "total_return_uv",
-              "share_pv", "total_share_uv", "view_24h", "total_return_uv_new"
-            )) {
-              if (!record.isNull(labelKey)) {
-                labels.put(labelKey, record.getString(labelKey))
-              }
-            }
-            //5 处理log key表头。
-            val apptype = record.getString("apptype")
-            val pagesource = record.getString("pagesource")
-            val mid = record.getString("mid")
-            // vid 已经提取了
-            val abcode = record.getString("abcode")
-            val level = if (record.isNull("level")) "0" else record.getString("level")
-            val logKey = (apptype, pagesource, mid, vid, ts, abcode, level).productIterator.mkString(",")
-            val labelKey = labels.toString()
-            // val featureKey = featureMap.toString()
-            val featureKey = truncateDecimal(featureMap).toString()
-            val scoresMap = getSubJson(record, "extend_alg", "scoresMap").toString()
-            //6 拼接数据，保存。
-            logKey + "\t" + labelKey + "\t" + scoresMap + "\t" + featureKey
-
-          })
-        })
-
-      // 4 保存数据到hdfs
-      val savePartition = dt + hh
-      val hdfsPath = savePath + "/" + savePartition
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        odpsData.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  def func(record: Record, schema: TableSchema): Record = {
-    record
-  }
-
-  def getJsonObject(record: Record, key: String): JSONObject = {
-    val data = if (record.isNull(key)) new JSONObject() else JSON.parseObject(record.getString(key))
-    val data2 = new JSONObject()
-    data.foreach(r => {
-      if (r._2 != null) {
-        data2.put(r._1, r._2)
-      }
-    })
-    data2
-  }
-
-  def getJsonObject(obj: JSONObject, keyName: String, valueName: String): JSONObject = {
-    val map = new JSONObject()
-    if (obj.nonEmpty) {
-      val keys = if (obj.containsKey(keyName)) obj.getString(keyName) else ""
-      val values = if (obj.containsKey(valueName)) obj.getString(valueName) else ""
-      if (!keys.equals("") && !values.equals("")) {
-        val key_list = keys.split(",")
-        val value_list = values.split(",")
-        if (key_list.length == value_list.length) {
-          for (index <- 0 until key_list.length) {
-            map.put(key_list(index), value_list(index))
-          }
-        }
-      }
-    }
-    map
-  }
-
-  def truncateDecimal(obj: JSONObject, scale: Int = 6): JSONObject = {
-    val data = new JSONObject()
-    for (key <- obj.keySet()) {
-      try {
-        val value = obj.getDoubleValue(key)
-        if (value == value.floor) {
-          data.put(key, value)
-        } else {
-          val newValue = BigDecimal(value).setScale(scale, BigDecimal.RoundingMode.HALF_UP).toDouble
-          data.put(key, newValue)
-        }
-      } catch {
-        case e: Exception => System.err.println(e.getMessage)
-      }
-    }
-    data
-  }
-
-
-  def funcC34567ForTagsW2V(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
-    // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
-    val tagsList = tags.split(",")
-    var d1 = 0.0
-    val d2 = new ArrayBuffer[String]()
-    var d3 = 0.0
-    var d4 = 0.0
-    for (tag <- tagsList) {
-      if (title.contains(tag)) {
-        d1 = d1 + 1.0
-        d2.add(tag)
-      }
-      val score = SimilarityUtils.word2VecSimilarity(tag, title)
-      d3 = if (score > d3) score else d3
-      d4 = d4 + score
-    }
-    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
-    (d1, d2.mkString(","), d3, d4)
-  }
-
-  def getSubJson(record: Record, key1: String, key2: String): JSONObject = {
-    if (!record.isNull(key1)) {
-      val obj = JSON.parseObject(record.getString(key1))
-      if (obj.nonEmpty && obj.containsKey(key2)) {
-        val data = obj.getString(key2)
-        return JSON.parseObject(data.replace("\\", ""))
-      }
-    }
-    new JSONObject()
-  }
-
-  def tsToCalendar(ts: String): Calendar = {
-    if (null != ts && ts.nonEmpty) {
-      val ms = ts.toLong * 1000L
-      if (ms > 1107658247000L && ms < 2685495047000L) {
-        val calendar = Calendar.getInstance
-        calendar.setTimeInMillis(ms)
-        return calendar
-      }
-    }
-    null
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_61_rov_sample_20241209.scala

@@ -1,153 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.JSON
-import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils}
-import examples.extractor.ExtractorUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.io.Source
-import scala.util.Random
-
-/*
-
- */
-
-object makedata_recsys_61_rov_sample_20241209 {
-  def main(args: Array[String]): Unit = {
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val readPath = param.getOrElse("readPath", "/dw/recommend/model/61_origin_data/")
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/61_recsys_rov_train_data/")
-    val beginStr = param.getOrElse("beginStr", "20241210")
-    val endStr = param.getOrElse("endStr", "20241210")
-    val repartition = param.getOrElse("repartition", "100").toInt
-    val whatLabel = param.getOrElse("whatLabel", "is_return")
-    val whatApps = param.getOrElse("whatApps", "0,4,5,21,3,6").split(",").toSet
-    val fuSampleRate = param.getOrElse("fuSampleRate", "1.0").toDouble
-    val notUseBucket = param.getOrElse("notUseBucket", "0").toInt
-    val featureNameFile = param.getOrElse("featureName", "20241209_recsys_rov_name.txt")
-    val featureBucketFile = param.getOrElse("featureBucket", "20241209_recsys_rov_bucket.txt")
-
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    val loader = getClass.getClassLoader
-    val featureNameSet = loadUseFeatureNames(loader, featureNameFile)
-    val featureBucketMap = loadUseFeatureBuckets(loader, notUseBucket, featureBucketFile)
-    val bucketsMap_br = sc.broadcast(featureBucketMap)
-
-    val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
-    for (date <- dateRange) {
-      println("开始执行:" + date)
-      val data = sc.textFile(readPath + "/" + date + "*").map(r => {
-          val rList = r.split("\t")
-          val logKey = rList(0)
-          val labelKey = rList(1)
-          val scoresMap = rList(2)
-          val jsons = JSON.parseObject(rList(3))
-          val features = scala.collection.mutable.Map[String, Double]()
-          jsons.foreach(r => {
-            features.put(r._1, jsons.getDoubleValue(r._1))
-          })
-          (logKey, labelKey, scoresMap, features)
-        })
-        .filter {
-          case (logKey, labelKey, scoresMap, features) =>
-            val logKeyList = logKey.split(",")
-            val apptype = logKeyList(0)
-            val pagesource = logKeyList(1)
-            whatApps.contains(apptype) && pagesource.endsWith("recommend")
-        }.filter {
-          case (logKey, labelKey, scoresMap, features) =>
-            val label = JSON.parseObject(labelKey).getOrDefault(whatLabel, "0").toString
-            "1".equals(label) || new Random().nextDouble() <= fuSampleRate
-        }
-        .map {
-          case (logKey, labelKey, scoresMap, features) =>
-            val label = JSON.parseObject(labelKey).getOrDefault(whatLabel, "0").toString
-            (logKey, label, scoresMap, features)
-        }
-        .mapPartitions(row => {
-          val result = new ArrayBuffer[String]()
-          val bucketsMap = bucketsMap_br.value
-          row.foreach {
-            case (logKey, label, scoresMap, features) =>
-              val featuresBucket = features.map {
-                case (name, score) =>
-                  if (!featureNameSet.contains(name)) {
-                    ""
-                  } else {
-                    if (score > 1E-8) {
-                      if (bucketsMap.contains(name)) {
-                        val (bucketsNum, buckets) = bucketsMap(name)
-                        val scoreNew = 1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
-                        name + ":" + scoreNew.toString
-                      } else {
-                        name + ":" + score.toString
-                      }
-                    } else {
-                      ""
-                    }
-                  }
-              }.filter(_.nonEmpty)
-              result.add(logKey + "\t" + label + "\t" + scoresMap + "\t" + featuresBucket.mkString("\t"))
-          }
-          result.iterator
-        })
-
-      // 4 保存数据到hdfs
-      val hdfsPath = savePath + "/" + date
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        data.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  private def loadFileData(loader: ClassLoader, nameFile: String): String = {
-    val resourceUrlBucket = loader.getResource(nameFile)
-    val data =
-      if (resourceUrlBucket != null) {
-        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
-        Source.fromURL(resourceUrlBucket).close()
-        buckets
-      } else {
-        ""
-      }
-    data
-  }
-
-  private def loadUseFeatureNames(loader: ClassLoader, nameFile: String): Set[String] = {
-    val names = loadFileData(loader, nameFile)
-    println(names)
-    names.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .toSet
-  }
-
-  private def loadUseFeatureBuckets(loader: ClassLoader, notUseBucket: Int, nameFile: String): Map[String, (Double, Array[Double])] = {
-    if (notUseBucket > 0) {
-      return Map[String, (Double, Array[Double])]()
-    }
-    val buckets = loadFileData(loader, nameFile)
-    println(buckets)
-    buckets.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .map(r => {
-        val rList = r.split("\t")
-        (rList(0), (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
-      }).toMap
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_71_nor_sample_20250109.scala

@@ -1,194 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.JSON
-import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils}
-import examples.extractor.ExtractorUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.io.Source
-import scala.util.Random
-
-object makedata_recsys_71_nor_sample_20250109 {
-  def main(args: Array[String]): Unit = {
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val readPath = param.getOrElse("readPath", "/dw/recommend/model/71_origin_data/")
-    val beginStr = param.getOrElse("beginStr", "20250103")
-    val endStr = param.getOrElse("endStr", "20250103")
-    val whatApps = param.getOrElse("whatApps", "0,3,4,21,17").split(",").toSet
-    val whatLabel = param.getOrElse("whatLabel", "return_n_uv")
-    val fuSampleRate = param.getOrElse("fuSampleRate", "-1.0").toDouble
-    val notUseBucket = param.getOrElse("notUseBucket", "0").toInt
-    val featureNameFile = param.getOrElse("featureName", "20241209_recsys_nor_name.txt")
-    val featureBucketFile = param.getOrElse("featureBucket", "20241209_recsys_nor_bucket.txt")
-    val repartition = param.getOrElse("repartition", "100").toInt
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/71_recsys_nor_train_data/")
-
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    val loader = getClass.getClassLoader
-    val featureNameSet = loadUseFeatureNames(loader, featureNameFile)
-    val featureBucketMap = loadUseFeatureBuckets(loader, notUseBucket, featureBucketFile)
-    val bucketsMap_br = sc.broadcast(featureBucketMap)
-
-    val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
-    for (date <- dateRange) {
-      println("开始执行:" + date)
-      val data = sc.textFile(readPath + "/" + date + "*").map(r => {
-          // logKey + "\t" + labelKey + "\t" + scoresMap + "\t" + featureKey
-          val rList = r.split("\t")
-          val logKey = rList(0)
-          val labelKey = rList(1)
-          val scoresMap = rList(2)
-          val featData = rList(3)
-          (logKey, labelKey, scoresMap, featData)
-        })
-        .filter {
-          case (logKey, labelKey, scoresMap, featData) =>
-            validData(logKey, whatApps)
-        }.filter {
-          case (logKey, labelKey, scoresMap, featData) =>
-            val label = parseLabel(labelKey, whatLabel).toDouble
-            label > 0 || new Random().nextDouble() <= fuSampleRate
-        }
-        .map {
-          case (logKey, labelKey, scoresMap, featData) =>
-            val label = parseLabel(labelKey, whatLabel).toDouble
-            val features = parseFeature(featData)
-            (logKey, label, scoresMap, features)
-        }
-        .mapPartitions(row => {
-          val result = new ArrayBuffer[String]()
-          val bucketsMap = bucketsMap_br.value
-          row.foreach {
-            case (logKey, label, scoresMap, features) =>
-              val featuresBucket = bucketFeature(featureNameSet, bucketsMap, features)
-              result.add(logKey + "\t" + label + "\t" + scoresMap + "\t" + featuresBucket.mkString("\t"))
-          }
-          result.iterator
-        })
-
-      // 4 保存数据到hdfs
-      val hdfsPath = savePath + "/" + date
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        data.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  private def loadFileData(loader: ClassLoader, nameFile: String): String = {
-    val resourceUrlBucket = loader.getResource(nameFile)
-    val data =
-      if (resourceUrlBucket != null) {
-        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
-        Source.fromURL(resourceUrlBucket).close()
-        buckets
-      } else {
-        ""
-      }
-    data
-  }
-
-  private def loadUseFeatureNames(loader: ClassLoader, nameFile: String): Set[String] = {
-    val names = loadFileData(loader, nameFile)
-    println(names)
-    names.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .toSet
-  }
-
-  private def loadUseFeatureBuckets(loader: ClassLoader, notUseBucket: Int, nameFile: String): Map[String, (Double, Array[Double])] = {
-    if (notUseBucket > 0) {
-      return Map[String, (Double, Array[Double])]()
-    }
-    val buckets = loadFileData(loader, nameFile)
-    println(buckets)
-    buckets.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .map(r => {
-        val rList = r.split("\t")
-        (rList(0), (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
-      }).toMap
-  }
-
-  private def recommendFlow(flowPool: String): Boolean = {
-    if (flowPool.isEmpty || flowPool.endsWith("#1")) {
-      return true
-    }
-    false
-  }
-
-  private def validData(logKey: String, whatApps: Set[String]): Boolean = {
-    // apptype, page, pagesource, recommendpagetype, flowpool, abcode, mid, vid, level, ts
-    val cells = logKey.split(",")
-    val apptype = cells(0)
-    val page = cells(1)
-    //val pagesource = cells(2)
-    val recommendpagetype = cells(3)
-    val flowpool = cells(4)
-    if (whatApps.contains(apptype)) {
-      if (recommendFlow(flowpool)) {
-        if (page.equals("详情后沉浸页")) {
-          return true
-        } else if (page.equals("回流后沉浸页&内页feed")) {
-          //if (recommendpagetype.endsWith("pages/user-videos-share-recommend-detail")) {
-          return true
-          //}
-        } else if (page.equals("首页feed")) {
-          return true
-        }
-      }
-    }
-    false
-  }
-
-  private def parseLabel(data: String, key: String, default: String = "0"): String = {
-    JSON.parseObject(data).getOrDefault(key, default).toString
-  }
-
-  private def parseFeature(data: String): scala.collection.mutable.Map[String, Double] = {
-    val features = scala.collection.mutable.Map[String, Double]()
-    if (data.nonEmpty) {
-      val obj = JSON.parseObject(data)
-      obj.foreach(r => {
-        features.put(r._1, obj.getDoubleValue(r._1))
-      })
-    }
-    features
-  }
-
-  private def bucketFeature(nameSet: Set[String], bucketMap: Map[String, (Double, Array[Double])], features: scala.collection.mutable.Map[String, Double]): Iterable[String] = {
-    features.map {
-      case (name, score) =>
-        if (!nameSet.contains(name)) {
-          ""
-        } else {
-          if (score > 1E-8) {
-            if (bucketMap.contains(name)) {
-              val (bucketsNum, buckets) = bucketMap(name)
-              val scoreNew = 1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
-              name + ":" + scoreNew.toString
-            } else {
-              name + ":" + score.toString
-            }
-          } else {
-            ""
-          }
-        }
-    }.filter(_.nonEmpty)
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_71_originData_20250109.scala

@@ -1,447 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.{JSON, JSONObject}
-import com.aliyun.odps.TableSchema
-import com.aliyun.odps.data.Record
-import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils, env}
-import examples.extractor.RankExtractorFeature_20240530
-import examples.utils.SimilarityUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.util.Random
-
-/*
-   20250109 提取特征
- */
-
-object makedata_recsys_71_originData_20250109 {
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val project = param.getOrElse("project", "loghubods")
-    val table = param.getOrElse("table", "alg_recsys_sample_tmp_20250109")
-    val tablePart = param.getOrElse("tablePart", "64").toInt
-    val beginStr = param.getOrElse("beginStr", "2025010723")
-    val endStr = param.getOrElse("endStr", "2025010723")
-    val whatLabel = param.getOrElse("whatLabel", "is_return_1")
-    val fuSampleRate = param.getOrElse("fuSampleRate", "0.1").toDouble
-    val repartition = param.getOrElse("repartition", "32").toInt
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/71_origin_data/")
-
-    // 2 odps
-    val odpsOps = env.getODPS(sc)
-
-    // 3 循环执行数据生产
-    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
-    for (dt_hh <- timeRange) {
-      val dt = dt_hh.substring(0, 8)
-      val hh = dt_hh.substring(8, 10)
-      val partition = s"dt=$dt,hh=$hh"
-      println("开始执行partiton:" + partition)
-      val odpsData = odpsOps.readTable(project = project,
-          table = table,
-          partition = partition,
-          transfer = func,
-          numPartition = tablePart)
-        .filter(record => {
-          val label = getStringValue(record, whatLabel, "0")
-          "1".equals(label) || new Random().nextDouble() <= fuSampleRate
-        })
-        .mapPartitions(p => {
-          SimilarityUtils.init()
-          p.map(record => {
-            val featureMap = new JSONObject()
-            val metaData = getJsonObject(record, "metafeaturemap")
-            // a 视频特征
-            val b1: JSONObject = getJsonObject(metaData, "alg_vid_feature_all_exp_v2")
-            val b2: JSONObject = getJsonObject(metaData, "alg_vid_feature_all_share")
-            val b3: JSONObject = getJsonObject(metaData, "alg_vid_feature_all_return")
-            val b6: JSONObject = getJsonObject(metaData, "alg_vid_feature_exp2share_v2")
-            val b7: JSONObject = getJsonObject(metaData, "alg_vid_feature_share2return")
-
-            val b8: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_noflow_exp_v2")
-            val b9: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_noflow_root_share_v2")
-            val b10: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_noflow_root_return_v2")
-            val b11: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_flow_exp_v2")
-            val b12: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_flow_root_share_v2")
-            val b13: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_flow_root_return_v2")
-            val b17: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_province_exp_v2")
-            val b18: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_province_root_share_v2")
-            val b19: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_province_root_return_v2")
-
-            val origin_data = List(
-              (b1, b2, b3, "b123"), (b1, b6, b7, "b167"),
-              (b8, b9, b10, "b8910"), (b11, b12, b13, "b111213"),
-              (b17, b18, b19, "b171819")
-            )
-            for ((b_1, b_2, b_3, prefix1) <- origin_data) {
-              for (prefix2 <- List(
-                "1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d"
-              )) {
-                val exp = if (b_1.isEmpty) 0D else b_1.getIntValue("exp_pv_" + prefix2).toDouble
-                val share = if (b_2.isEmpty) 0D else b_2.getIntValue("share_pv_" + prefix2).toDouble
-                val returns = if (b_3.isEmpty) 0D else b_3.getIntValue("return_uv_" + prefix2).toDouble
-                val f1 = RankExtractorFeature_20240530.calDiv(share, exp)
-                val f2 = RankExtractorFeature_20240530.calLog(share)
-                val f3 = RankExtractorFeature_20240530.calDiv(returns, exp)
-                val f4 = RankExtractorFeature_20240530.calLog(returns)
-                val f5 = f3 * f4
-                val f6 = RankExtractorFeature_20240530.calDiv(returns, share)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "STR", f1)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "log(share)", f2)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROV", f3)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "log(return)", f4)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROV*log(return)", f5)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROS", f6)
-              }
-            }
-
-            val video_info: JSONObject = getJsonObject(metaData, "alg_vid_feature_basic_info")
-            featureMap.put("total_time", if (video_info.containsKey("total_time")) video_info.getIntValue("total_time").toDouble else 0D)
-            featureMap.put("bit_rate", if (video_info.containsKey("bit_rate")) video_info.getIntValue("bit_rate").toDouble else 0D)
-
-            val c1: JSONObject = getJsonObject(metaData, "alg_mid_feature_play")
-            if (c1.nonEmpty) {
-              featureMap.put("playcnt_6h", if (c1.containsKey("playcnt_6h")) c1.getIntValue("playcnt_6h").toDouble else 0D)
-              featureMap.put("playcnt_1d", if (c1.containsKey("playcnt_1d")) c1.getIntValue("playcnt_1d").toDouble else 0D)
-              featureMap.put("playcnt_3d", if (c1.containsKey("playcnt_3d")) c1.getIntValue("playcnt_3d").toDouble else 0D)
-              featureMap.put("playcnt_7d", if (c1.containsKey("playcnt_7d")) c1.getIntValue("playcnt_7d").toDouble else 0D)
-            }
-            val c2: JSONObject = getJsonObject(metaData, "alg_mid_feature_share_and_return")
-            if (c2.nonEmpty) {
-              featureMap.put("share_pv_12h", if (c2.containsKey("share_pv_12h")) c2.getIntValue("share_pv_12h").toDouble else 0D)
-              featureMap.put("share_pv_1d", if (c2.containsKey("share_pv_1d")) c2.getIntValue("share_pv_1d").toDouble else 0D)
-              featureMap.put("share_pv_3d", if (c2.containsKey("share_pv_3d")) c2.getIntValue("share_pv_3d").toDouble else 0D)
-              featureMap.put("share_pv_7d", if (c2.containsKey("share_pv_7d")) c2.getIntValue("share_pv_7d").toDouble else 0D)
-              featureMap.put("return_uv_12h", if (c2.containsKey("return_uv_12h")) c2.getIntValue("return_uv_12h").toDouble else 0D)
-              featureMap.put("return_uv_1d", if (c2.containsKey("return_uv_1d")) c2.getIntValue("return_uv_1d").toDouble else 0D)
-              featureMap.put("return_uv_3d", if (c2.containsKey("return_uv_3d")) c2.getIntValue("return_uv_3d").toDouble else 0D)
-              featureMap.put("return_uv_7d", if (c2.containsKey("return_uv_7d")) c2.getIntValue("return_uv_7d").toDouble else 0D)
-            }
-
-            val title = if (video_info.containsKey("title")) video_info.getString("title") else ""
-            if (!title.equals("")) {
-              val c34567List = List(
-                ("c3_feature", getJsonObject(metaData, "alg_mid_feature_play_tags")),
-                ("c4_feature", getJsonObject(metaData, "alg_mid_feature_return_tags")),
-                ("c5_feature", getJsonObject(metaData, "alg_mid_feature_share_tags")),
-                ("c6_feature", getJsonObject(metaData, "alg_mid_feature_feed_exp_share_tags_v2")),
-                ("c7_feature", getJsonObject(metaData, "alg_mid_feature_feed_exp_return_tags_v2"))
-              )
-              for ((key_feature, c34567) <- c34567List) {
-                for (key_time <- List("tags_1d", "tags_3d", "tags_7d")) {
-                  val tags = if (c34567.containsKey(key_time)) c34567.getString(key_time) else ""
-                  if (!tags.equals("")) {
-                    val (f1, f2, f3, f4) = funcC34567ForTagsW2V(tags, title)
-                    featureMap.put(key_feature + "_" + key_time + "_matchnum", f1)
-                    featureMap.put(key_feature + "_" + key_time + "_maxscore", f3)
-                    featureMap.put(key_feature + "_" + key_time + "_avgscore", f4)
-                  }
-                }
-              }
-            }
-
-            val vid = if (record.isNull("vid")) "" else record.getString("vid")
-            if (!vid.equals("")) {
-              val c89List = List(
-                ("c8_feature", getJsonObject(metaData, "alg_mid_feature_sharecf")),
-                ("c9_feature", getJsonObject(metaData, "alg_mid_feature_returncf"))
-              )
-              for ((key_feature, c89) <- c89List) {
-                for (key_action <- List("share", "return")) {
-                  val cfListStr = if (c89.containsKey(key_action)) c89.getString(key_action) else ""
-                  if (!cfListStr.equals("")) {
-                    val cfMap = cfListStr.split(",").map(r => {
-                      val rList = r.split(":")
-                      (rList(0), (rList(1), rList(2), rList(3)))
-                    }).toMap
-                    if (cfMap.contains(vid)) {
-                      val (score, num, rank) = cfMap(vid)
-                      featureMap.put(key_feature + "_" + key_action + "_score", score.toDouble)
-                      featureMap.put(key_feature + "_" + key_action + "_num", num.toDouble)
-                      featureMap.put(key_feature + "_" + key_action + "_rank", 1.0 / rank.toDouble)
-                    }
-                  }
-                }
-              }
-            }
-
-            val d1: JSONObject = getJsonObject(metaData, "alg_recsys_feature_cf_i2i_new_v2")
-            if (d1.nonEmpty) {
-              featureMap.put("d1_exp", if (d1.containsKey("exp")) d1.getString("exp").toDouble else 0D)
-              featureMap.put("d1_return_n", if (d1.containsKey("return_n")) d1.getString("return_n").toDouble else 0D)
-              featureMap.put("d1_rovn", if (d1.containsKey("rovn")) d1.getString("rovn").toDouble else 0D)
-            }
-
-            // ************* new feature *************
-            val shortPeriod = List("1h", "2h", "4h", "6h", "12h", "24h", "7d")
-            val middlePeriod = List("14d", "30d")
-            val longPeriod = List("7d", "35d", "90d", "365d")
-            val vidStatFeat = List(
-              ("b20", shortPeriod, getJsonObject(record, "b20_feature")), // cate2_feature
-              ("b21", shortPeriod, getJsonObject(record, "b21_feature")), // cate1_feature
-              ("b22", shortPeriod, getJsonObject(record, "b22_feature")), // source_feature
-              ("b28", shortPeriod, getJsonObject(record, "b28_feature")), // sence_type_feature
-              ("b29", shortPeriod, getJsonObject(record, "b29_feature")), // alg_videoid_feature
-              ("b23", middlePeriod, getJsonObject(record, "b23_feature")), // cate2_feature_day
-              ("b24", middlePeriod, getJsonObject(record, "b24_feature")), // cate1_feature_day
-              ("b25", middlePeriod, getJsonObject(record, "b25_feature")), // source_feature_day
-              ("b26", longPeriod, getJsonObject(record, "b26_feature")), // unionid_feature_day
-              ("b27", longPeriod, getJsonObject(record, "b27_feature")) // vid_feature_day
-            )
-            for ((featType, featPeriod, featData) <- vidStatFeat) {
-              for (period <- featPeriod) {
-                // val view = if (featData.isEmpty) 0D else featData.getDoubleValue("view_" + period)
-                val share = if (featData.isEmpty) 0D else featData.getDoubleValue("share_" + period)
-                val return_ = if (featData.isEmpty) 0D else featData.getDoubleValue("return_" + period)
-                val view_hasreturn = if (featData.isEmpty) 0D else featData.getDoubleValue("view_hasreturn_" + period)
-                val share_hasreturn = if (featData.isEmpty) 0D else featData.getDoubleValue("share_hasreturn_" + period)
-                val ros = if (featData.isEmpty) 0D else featData.getDoubleValue("ros_" + period)
-                val rov = if (featData.isEmpty) 0D else featData.getDoubleValue("rov_" + period)
-                val r_cnt = if (featData.isEmpty) 0D else featData.getDoubleValue("r_cnt_" + period)
-                val r_rate = if (featData.isEmpty) 0D else featData.getDoubleValue("r_rate_" + period)
-                val r_cnt4s = if (featData.isEmpty) 0D else featData.getDoubleValue("r_cnt4s_" + period)
-                val str = if (featData.isEmpty) 0D else featData.getDoubleValue("str_" + period)
-                // scale
-                // val view_s = RankExtractorFeature_20240530.calLog(view)
-                val share_s = RankExtractorFeature_20240530.calLog(share)
-                val return_s = RankExtractorFeature_20240530.calLog(return_)
-                val view_hasreturn_s = RankExtractorFeature_20240530.calLog(view_hasreturn)
-                val share_hasreturn_s = RankExtractorFeature_20240530.calLog(share_hasreturn)
-
-                // featureMap.put(featType + "_" + period + "_" + "view", view_s)
-                featureMap.put(featType + "_" + period + "_" + "share", share_s)
-                featureMap.put(featType + "_" + period + "_" + "return", return_s)
-                featureMap.put(featType + "_" + period + "_" + "view_hasreturn", view_hasreturn_s)
-                featureMap.put(featType + "_" + period + "_" + "share_hasreturn", share_hasreturn_s)
-                featureMap.put(featType + "_" + period + "_" + "ros", ros)
-                featureMap.put(featType + "_" + period + "_" + "rov", rov)
-                featureMap.put(featType + "_" + period + "_" + "r_cnt", r_cnt)
-                featureMap.put(featType + "_" + period + "_" + "r_rate", r_rate)
-                featureMap.put(featType + "_" + period + "_" + "r_cnt4s", r_cnt4s)
-                featureMap.put(featType + "_" + period + "_" + "str", str)
-              }
-            }
-
-            // new cf
-            val d2345Data = List(
-              ("d2", "rosn", getJsonObject(record, "d2_feature")),
-              ("d3", "rosn", getJsonObject(record, "d3_feature")),
-              ("d4", "rovn", getJsonObject(record, "d4_feature")),
-              ("d5", "rovn", getJsonObject(record, "d5_feature"))
-            )
-            for ((featType, valType, featData) <- d2345Data) {
-              if (featData.nonEmpty) {
-                val expKey = if (valType.equals("rosn")) "share" else "exp"
-                val exp = if (featData.containsKey(expKey)) featData.getString(expKey).toDouble else 0D
-                val return_n = if (featData.containsKey("return_n")) featData.getString("return_n").toDouble else 0D
-                val value = if (featData.containsKey(valType)) featData.getString(valType).toDouble else 0D
-                // scale
-                val exp_s = RankExtractorFeature_20240530.calLog(exp)
-                val return_n_s = RankExtractorFeature_20240530.calLog(return_n)
-                featureMap.put(featType + "_exp", exp_s)
-                featureMap.put(featType + "_return_n", return_n_s)
-                featureMap.put(featType + "_" + valType, value)
-              }
-            }
-
-            if (!vid.equals("")) {
-              val idScoreObj = getJsonObject(getJsonObject(record, "d6_feature"), "vids", "scores")
-              if (idScoreObj.nonEmpty && idScoreObj.containsKey(vid)) {
-                val score = idScoreObj.getString(vid).toDouble
-                featureMap.put("d6", score)
-              }
-            }
-
-            // head video & rank video
-            val headVideo = getJsonObject(record, "v2_feature")
-            val rankVideo = video_info
-            if (headVideo.nonEmpty && rankVideo.nonEmpty) {
-              val videoAttrs = List("title", "topic", "keywords", "cate1_list", "cate2", "cate2_list", "style", "theme", "user_value")
-              for (attr <- videoAttrs) {
-                val headAttr = if (headVideo.containsKey(attr)) headVideo.getString(attr) else ""
-                val rankAttr = if (rankVideo.containsKey(attr)) rankVideo.getString(attr) else ""
-                if (headAttr.nonEmpty && !headAttr.equals("unknown") && rankAttr.nonEmpty && !rankAttr.equals("unknown")) {
-                  val simScore = SimilarityUtils.word2VecSimilarity(headAttr, rankAttr)
-                  featureMap.put("video_sim_" + attr, simScore)
-                }
-              }
-            }
-
-            /*
-            视频特征： 5*6*5 = 240个
-                      曝光使用pv 分享使用pv 回流使用uv --> 1h 2h 3h 4h 12h 1d 3d 7d
-                      STR log(share) ROV log(return) ROV*log(return) ROS
-                      整体、整体曝光对应、推荐非冷启root、推荐冷启root、分省份root
-            视频基础： 2个   视频时长、比特率
-            用户： 4+8 = 12个
-                      播放次数 --> 6h 1d 3d 7d --> 4个
-                      带回来的分享pv 回流uv --> 12h 1d 3d 7d --> 8个
-            人+vid-title:  5*3*3 = 45
-                      播放点/回流点/分享点/累积分享/累积回流 --> 1d 3d 7d --> 匹配数量 语义最高相似度分 语义平均相似度分 --> 45个
-            人+vid-cf： 2*3*3 = 12
-                      基于分享行为/基于回流行为 -->  “分享cf”+”回流点击cf“ 相似分 相似数量 相似rank的倒数 --> 12个
-            头部视频：  3
-                      曝光 回流 ROVn 3个特征
-            场景：     小时 星期 apptype city province pagesource 机器型号
-            总量：     240+2+12+45+12+3 = 314
-            ---------------------------------------------------------------
-            视频特征：(4*7+3*2+2*4)*10 = 420个
-            CF: 13个
-            视频相似特征: 9个
-             */
-
-            //4 处理label信息。
-            val labels = new JSONObject
-            for (labelKey <- List("is_share", "share_cnt", "is_return_1", "return_1_uv", "is_return_n", "return_n_uv", "is_return_noself", "return_1_uv_noself")) {
-              if (!record.isNull(labelKey)) {
-                labels.put(labelKey, record.getString(labelKey))
-              }
-            }
-            //5 处理log key表头。
-            val apptype = record.getString("apptype")
-            val page = getStringValue(record, "page")
-            val pagesource = getStringValue(record, "pagesource")
-            val recommendpagetype = getStringValue(record, "recommendpagetype")
-            val flowpool = getStringValue(record, "flowpool")
-            val abcode = record.getString("abcode")
-            val mid = record.getString("mid")
-            val level = getStringValue(record, "level", "0")
-            val ts = record.getString("ts")
-            val logKey = (apptype, page, pagesource, recommendpagetype, flowpool, abcode, mid, vid, level, ts).productIterator.mkString(",")
-            val labelKey = labels.toString()
-            val featureKey = truncateDecimal(featureMap).toString()
-            val scoresMap = getSubJson(record, "extend_alg", "scoresMap").toString()
-            //6 拼接数据，保存。
-            logKey + "\t" + labelKey + "\t" + scoresMap + "\t" + featureKey
-          })
-        })
-
-      // 4 保存数据到hdfs
-      val savePartition = dt + hh
-      val hdfsPath = savePath + "/" + savePartition
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        odpsData.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  def func(record: Record, schema: TableSchema): Record = {
-    record
-  }
-
-  def getJsonObject(record: Record, key: String): JSONObject = {
-    val data = if (record.isNull(key)) new JSONObject() else JSON.parseObject(record.getString(key))
-    val data2 = new JSONObject()
-    data.foreach(r => {
-      if (r._2 != null) {
-        data2.put(r._1, r._2)
-      }
-    })
-    data2
-  }
-
-  def getJsonObject(obj: JSONObject, key: String): JSONObject = {
-    if (obj.nonEmpty) {
-      if (obj.containsKey(key)) {
-        val data = obj.getJSONObject(key)
-        val validData = new JSONObject()
-        data.foreach(r => {
-          if (r._2 != null) {
-            validData.put(r._1, r._2)
-          }
-        })
-        return validData
-      }
-    }
-    new JSONObject()
-  }
-
-  def getJsonObject(obj: JSONObject, keyName: String, valueName: String): JSONObject = {
-    val map = new JSONObject()
-    if (obj.nonEmpty) {
-      val keys = if (obj.containsKey(keyName)) obj.getString(keyName) else ""
-      val values = if (obj.containsKey(valueName)) obj.getString(valueName) else ""
-      if (!keys.equals("") && !values.equals("")) {
-        val key_list = keys.split(",")
-        val value_list = values.split(",")
-        if (key_list.length == value_list.length) {
-          for (index <- 0 until key_list.length) {
-            map.put(key_list(index), value_list(index))
-          }
-        }
-      }
-    }
-    map
-  }
-
-  def truncateDecimal(obj: JSONObject, scale: Int = 6): JSONObject = {
-    val data = new JSONObject()
-    for (key <- obj.keySet()) {
-      try {
-        val value = obj.getDoubleValue(key)
-        if (value == value.floor) {
-          data.put(key, value)
-        } else {
-          val newValue = BigDecimal(value).setScale(scale, BigDecimal.RoundingMode.HALF_UP).toDouble
-          data.put(key, newValue)
-        }
-      } catch {
-        case e: Exception => System.err.println(e.getMessage)
-      }
-    }
-    data
-  }
-
-  def funcC34567ForTagsW2V(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
-    // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
-    val tagsList = tags.split(",")
-    var d1 = 0.0
-    val d2 = new ArrayBuffer[String]()
-    var d3 = 0.0
-    var d4 = 0.0
-    for (tag <- tagsList) {
-      if (title.contains(tag)) {
-        d1 = d1 + 1.0
-        d2.add(tag)
-      }
-      val score = SimilarityUtils.word2VecSimilarity(tag, title)
-      d3 = if (score > d3) score else d3
-      d4 = d4 + score
-    }
-    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
-    (d1, d2.mkString(","), d3, d4)
-  }
-
-  def getSubJson(record: Record, key1: String, key2: String): JSONObject = {
-    if (!record.isNull(key1)) {
-      val obj = JSON.parseObject(record.getString(key1))
-      if (obj.nonEmpty && obj.containsKey(key2)) {
-        val data = obj.getString(key2)
-        return JSON.parseObject(data.replace("\\", ""))
-      }
-    }
-    new JSONObject()
-  }
-
-  def getStringValue(record: Record, key: String, default: String = ""): String = {
-    if (!record.isNull(key)) {
-      val value = record.getString(key)
-      return value.trim
-    }
-    default
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_71_rov_sample_20250109.scala

@@ -1,195 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.JSON
-import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils}
-import examples.extractor.ExtractorUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.io.Source
-import scala.util.Random
-
-
-object makedata_recsys_71_rov_sample_20250109 {
-  def main(args: Array[String]): Unit = {
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val readPath = param.getOrElse("readPath", "/dw/recommend/model/71_origin_data/")
-    val beginStr = param.getOrElse("beginStr", "20250103")
-    val endStr = param.getOrElse("endStr", "20250103")
-    val whatApps = param.getOrElse("whatApps", "0,3,4,21,17").split(",").toSet
-    val whatLabel = param.getOrElse("whatLabel", "is_return_1")
-    val fuSampleRate = param.getOrElse("fuSampleRate", "1.0").toDouble
-    val notUseBucket = param.getOrElse("notUseBucket", "0").toInt
-    val featureNameFile = param.getOrElse("featureName", "20241209_recsys_rov_name.txt")
-    val featureBucketFile = param.getOrElse("featureBucket", "20241209_recsys_rov_bucket.txt")
-    val repartition = param.getOrElse("repartition", "100").toInt
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/71_recsys_rov_train_data/")
-
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    val loader = getClass.getClassLoader
-    val featureNameSet = loadUseFeatureNames(loader, featureNameFile)
-    val featureBucketMap = loadUseFeatureBuckets(loader, notUseBucket, featureBucketFile)
-    val bucketsMap_br = sc.broadcast(featureBucketMap)
-
-    val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
-    for (date <- dateRange) {
-      println("开始执行:" + date)
-      val data = sc.textFile(readPath + "/" + date + "*").map(r => {
-          // logKey + "\t" + labelKey + "\t" + scoresMap + "\t" + featureKey
-          val rList = r.split("\t")
-          val logKey = rList(0)
-          val labelKey = rList(1)
-          val scoresMap = rList(2)
-          val featData = rList(3)
-          (logKey, labelKey, scoresMap, featData)
-        })
-        .filter {
-          case (logKey, labelKey, scoresMap, featData) =>
-            validData(logKey, whatApps)
-        }.filter {
-          case (logKey, labelKey, scoresMap, featData) =>
-            val label = parseLabel(labelKey, whatLabel)
-            "1".equals(label) || new Random().nextDouble() <= fuSampleRate
-        }
-        .map {
-          case (logKey, labelKey, scoresMap, featData) =>
-            val label = parseLabel(labelKey, whatLabel)
-            val features = parseFeature(featData)
-            (logKey, label, scoresMap, features)
-        }
-        .mapPartitions(row => {
-          val result = new ArrayBuffer[String]()
-          val bucketsMap = bucketsMap_br.value
-          row.foreach {
-            case (logKey, label, scoresMap, features) =>
-              val featuresBucket = bucketFeature(featureNameSet, bucketsMap, features)
-              result.add(logKey + "\t" + label + "\t" + scoresMap + "\t" + featuresBucket.mkString("\t"))
-          }
-          result.iterator
-        })
-
-      // 4 保存数据到hdfs
-      val hdfsPath = savePath + "/" + date
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        data.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  private def loadFileData(loader: ClassLoader, nameFile: String): String = {
-    val resourceUrlBucket = loader.getResource(nameFile)
-    val data =
-      if (resourceUrlBucket != null) {
-        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
-        Source.fromURL(resourceUrlBucket).close()
-        buckets
-      } else {
-        ""
-      }
-    data
-  }
-
-  private def loadUseFeatureNames(loader: ClassLoader, nameFile: String): Set[String] = {
-    val names = loadFileData(loader, nameFile)
-    println(names)
-    names.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .toSet
-  }
-
-  private def loadUseFeatureBuckets(loader: ClassLoader, notUseBucket: Int, nameFile: String): Map[String, (Double, Array[Double])] = {
-    if (notUseBucket > 0) {
-      return Map[String, (Double, Array[Double])]()
-    }
-    val buckets = loadFileData(loader, nameFile)
-    println(buckets)
-    buckets.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .map(r => {
-        val rList = r.split("\t")
-        (rList(0), (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
-      }).toMap
-  }
-
-  private def recommendFlow(flowPool: String): Boolean = {
-    if (flowPool.isEmpty || flowPool.endsWith("#1")) {
-      return true
-    }
-    false
-  }
-
-  private def validData(logKey: String, whatApps: Set[String]): Boolean = {
-    // apptype, page, pagesource, recommendpagetype, flowpool, abcode, mid, vid, level, ts
-    val cells = logKey.split(",")
-    val apptype = cells(0)
-    val page = cells(1)
-    //val pagesource = cells(2)
-    val recommendpagetype = cells(3)
-    val flowpool = cells(4)
-    if (whatApps.contains(apptype)) {
-      if (recommendFlow(flowpool)) {
-        if (page.equals("详情后沉浸页")) {
-          return true
-        } else if (page.equals("回流后沉浸页&内页feed")) {
-          //if (recommendpagetype.endsWith("pages/user-videos-share-recommend-detail")) {
-          return true
-          //}
-        } else if (page.equals("首页feed")) {
-          return true
-        }
-      }
-    }
-    false
-  }
-
-  private def parseLabel(data: String, key: String, default: String = "0"): String = {
-    JSON.parseObject(data).getOrDefault(key, default).toString
-  }
-
-  private def parseFeature(data: String): scala.collection.mutable.Map[String, Double] = {
-    val features = scala.collection.mutable.Map[String, Double]()
-    if (data.nonEmpty) {
-      val obj = JSON.parseObject(data)
-      obj.foreach(r => {
-        features.put(r._1, obj.getDoubleValue(r._1))
-      })
-    }
-    features
-  }
-
-  private def bucketFeature(nameSet: Set[String], bucketMap: Map[String, (Double, Array[Double])], features: scala.collection.mutable.Map[String, Double]): Iterable[String] = {
-    features.map {
-      case (name, score) =>
-        if (!nameSet.contains(name)) {
-          ""
-        } else {
-          if (score > 1E-8) {
-            if (bucketMap.contains(name)) {
-              val (bucketsNum, buckets) = bucketMap(name)
-              val scoreNew = 1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
-              name + ":" + scoreNew.toString
-            } else {
-              name + ":" + score.toString
-            }
-          } else {
-            ""
-          }
-        }
-    }.filter(_.nonEmpty)
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_72_originData_20250109.scala

@@ -1,492 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.{JSON, JSONObject}
-import com.aliyun.odps.TableSchema
-import com.aliyun.odps.data.Record
-import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils, env}
-import examples.extractor.RankExtractorFeature_20240530
-import examples.utils.SimilarityUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import java.util.Calendar
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.util.Random
-
-/*
-   20250109 提取特征
- */
-
-object makedata_recsys_72_originData_20250109 {
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val project = param.getOrElse("project", "loghubods")
-    val table = param.getOrElse("table", "alg_recsys_sample_tmp_20250109")
-    val tablePart = param.getOrElse("tablePart", "64").toInt
-    val beginStr = param.getOrElse("beginStr", "2025010723")
-    val endStr = param.getOrElse("endStr", "2025010723")
-    val whatLabel = param.getOrElse("whatLabel", "is_return_1")
-    val fuSampleRate = param.getOrElse("fuSampleRate", "0.1").toDouble
-    val repartition = param.getOrElse("repartition", "32").toInt
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/72_origin_data/")
-
-    // 2 odps
-    val odpsOps = env.getODPS(sc)
-
-    // 3 循环执行数据生产
-    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
-    for (dt_hh <- timeRange) {
-      val dt = dt_hh.substring(0, 8)
-      val hh = dt_hh.substring(8, 10)
-      val partition = s"dt=$dt,hh=$hh"
-      println("开始执行partiton:" + partition)
-      val odpsData = odpsOps.readTable(project = project,
-          table = table,
-          partition = partition,
-          transfer = func,
-          numPartition = tablePart)
-        .filter(record => {
-          val label = getStringValue(record, whatLabel, "0")
-          "1".equals(label) || new Random().nextDouble() <= fuSampleRate
-        })
-        .mapPartitions(p => {
-          SimilarityUtils.init()
-          p.map(record => {
-            val featureMap = new JSONObject()
-            val metaData = getJsonObject(record, "metafeaturemap")
-            // a 视频特征
-            val b1: JSONObject = getJsonObject(metaData, "alg_vid_feature_all_exp_v2", record, "b1_feature")
-            val b2: JSONObject = getJsonObject(metaData, "alg_vid_feature_all_share", record, "b2_feature")
-            val b3: JSONObject = getJsonObject(metaData, "alg_vid_feature_all_return", record, "b3_feature")
-            val b6: JSONObject = getJsonObject(metaData, "alg_vid_feature_exp2share_v2", record, "b6_feature")
-            val b7: JSONObject = getJsonObject(metaData, "alg_vid_feature_share2return", record, "b7_feature")
-
-            val b8: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_noflow_exp_v2", record, "b8_feature")
-            val b9: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_noflow_root_share_v2", record, "b9_feature")
-            val b10: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_noflow_root_return_v2", record, "b10_feature")
-            val b11: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_flow_exp_v2", record, "b11_feature")
-            val b12: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_flow_root_share_v2", record, "b12_feature")
-            val b13: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_flow_root_return_v2", record, "b13_feature")
-            val b17: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_province_exp_v2", record, "b17_feature")
-            val b18: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_province_root_share_v2", record, "b18_feature")
-            val b19: JSONObject = getJsonObject(metaData, "alg_vid_feature_feed_province_root_return_v2", record, "b19_feature")
-
-            val origin_data = List(
-              (b1, b2, b3, "b123"), (b1, b6, b7, "b167"),
-              (b8, b9, b10, "b8910"), (b11, b12, b13, "b111213"),
-              (b17, b18, b19, "b171819")
-            )
-            for ((b_1, b_2, b_3, prefix1) <- origin_data) {
-              for (prefix2 <- List(
-                "1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d"
-              )) {
-                val exp = if (b_1.isEmpty) 0D else b_1.getIntValue("exp_pv_" + prefix2).toDouble
-                val share = if (b_2.isEmpty) 0D else b_2.getIntValue("share_pv_" + prefix2).toDouble
-                val returns = if (b_3.isEmpty) 0D else b_3.getIntValue("return_uv_" + prefix2).toDouble
-                val f1 = RankExtractorFeature_20240530.calDiv(share, exp)
-                val f2 = RankExtractorFeature_20240530.calLog(share)
-                val f3 = RankExtractorFeature_20240530.calDiv(returns, exp)
-                val f4 = RankExtractorFeature_20240530.calLog(returns)
-                val f5 = f3 * f4
-                val f6 = RankExtractorFeature_20240530.calDiv(returns, share)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "STR", f1)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "log(share)", f2)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROV", f3)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "log(return)", f4)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROV*log(return)", f5)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROS", f6)
-              }
-            }
-
-            val video_info: JSONObject = getJsonObject(metaData, "alg_vid_feature_basic_info", record, "t_v_info_feature")
-            featureMap.put("total_time", if (video_info.containsKey("total_time")) video_info.getIntValue("total_time").toDouble else 0D)
-            featureMap.put("bit_rate", if (video_info.containsKey("bit_rate")) video_info.getIntValue("bit_rate").toDouble else 0D)
-
-            val c1: JSONObject = getJsonObject(metaData, "alg_mid_feature_play", record, "c1_feature")
-            if (c1.nonEmpty) {
-              featureMap.put("playcnt_6h", if (c1.containsKey("playcnt_6h")) c1.getIntValue("playcnt_6h").toDouble else 0D)
-              featureMap.put("playcnt_1d", if (c1.containsKey("playcnt_1d")) c1.getIntValue("playcnt_1d").toDouble else 0D)
-              featureMap.put("playcnt_3d", if (c1.containsKey("playcnt_3d")) c1.getIntValue("playcnt_3d").toDouble else 0D)
-              featureMap.put("playcnt_7d", if (c1.containsKey("playcnt_7d")) c1.getIntValue("playcnt_7d").toDouble else 0D)
-            }
-            val c2: JSONObject = getJsonObject(metaData, "alg_mid_feature_share_and_return", record, "c2_feature")
-            if (c2.nonEmpty) {
-              featureMap.put("share_pv_12h", if (c2.containsKey("share_pv_12h")) c2.getIntValue("share_pv_12h").toDouble else 0D)
-              featureMap.put("share_pv_1d", if (c2.containsKey("share_pv_1d")) c2.getIntValue("share_pv_1d").toDouble else 0D)
-              featureMap.put("share_pv_3d", if (c2.containsKey("share_pv_3d")) c2.getIntValue("share_pv_3d").toDouble else 0D)
-              featureMap.put("share_pv_7d", if (c2.containsKey("share_pv_7d")) c2.getIntValue("share_pv_7d").toDouble else 0D)
-              featureMap.put("return_uv_12h", if (c2.containsKey("return_uv_12h")) c2.getIntValue("return_uv_12h").toDouble else 0D)
-              featureMap.put("return_uv_1d", if (c2.containsKey("return_uv_1d")) c2.getIntValue("return_uv_1d").toDouble else 0D)
-              featureMap.put("return_uv_3d", if (c2.containsKey("return_uv_3d")) c2.getIntValue("return_uv_3d").toDouble else 0D)
-              featureMap.put("return_uv_7d", if (c2.containsKey("return_uv_7d")) c2.getIntValue("return_uv_7d").toDouble else 0D)
-            }
-
-            val title = if (video_info.containsKey("title")) video_info.getString("title") else ""
-            if (!title.equals("")) {
-              val c34567List = List(
-                ("c3_feature", getJsonObject(metaData, "alg_mid_feature_play_tags")),
-                ("c4_feature", getJsonObject(metaData, "alg_mid_feature_return_tags")),
-                ("c5_feature", getJsonObject(metaData, "alg_mid_feature_share_tags")),
-                ("c6_feature", getJsonObject(metaData, "alg_mid_feature_feed_exp_share_tags_v2")),
-                ("c7_feature", getJsonObject(metaData, "alg_mid_feature_feed_exp_return_tags_v2"))
-              )
-              for ((key_feature, c34567) <- c34567List) {
-                for (key_time <- List("tags_1d", "tags_3d", "tags_7d")) {
-                  val tags = if (c34567.containsKey(key_time)) c34567.getString(key_time) else ""
-                  if (!tags.equals("")) {
-                    val (f1, f2, f3, f4) = funcC34567ForTagsW2V(tags, title)
-                    featureMap.put(key_feature + "_" + key_time + "_matchnum", f1)
-                    featureMap.put(key_feature + "_" + key_time + "_maxscore", f3)
-                    featureMap.put(key_feature + "_" + key_time + "_avgscore", f4)
-                  }
-                }
-              }
-            }
-
-            val vid = if (record.isNull("vid")) "" else record.getString("vid")
-            if (!vid.equals("")) {
-              val c89List = List(
-                ("c8_feature", getJsonObject(metaData, "alg_mid_feature_sharecf")),
-                ("c9_feature", getJsonObject(metaData, "alg_mid_feature_returncf"))
-              )
-              for ((key_feature, c89) <- c89List) {
-                for (key_action <- List("share", "return")) {
-                  val cfListStr = if (c89.containsKey(key_action)) c89.getString(key_action) else ""
-                  if (!cfListStr.equals("")) {
-                    val cfMap = cfListStr.split(",").map(r => {
-                      val rList = r.split(":")
-                      (rList(0), (rList(1), rList(2), rList(3)))
-                    }).toMap
-                    if (cfMap.contains(vid)) {
-                      val (score, num, rank) = cfMap(vid)
-                      featureMap.put(key_feature + "_" + key_action + "_score", score.toDouble)
-                      featureMap.put(key_feature + "_" + key_action + "_num", num.toDouble)
-                      featureMap.put(key_feature + "_" + key_action + "_rank", 1.0 / rank.toDouble)
-                    }
-                  }
-                }
-              }
-            }
-
-            val d1: JSONObject = getJsonObject(metaData, "alg_recsys_feature_cf_i2i_new_v2", record, "d1_feature")
-            if (d1.nonEmpty) {
-              featureMap.put("d1_exp", if (d1.containsKey("exp")) d1.getString("exp").toDouble else 0D)
-              featureMap.put("d1_return_n", if (d1.containsKey("return_n")) d1.getString("return_n").toDouble else 0D)
-              featureMap.put("d1_rovn", if (d1.containsKey("rovn")) d1.getString("rovn").toDouble else 0D)
-            }
-
-            // ************* new feature *************
-            val shortPeriod = List("1h", "2h", "4h", "6h", "12h", "24h", "7d")
-            val middlePeriod = List("14d", "30d")
-            val longPeriod = List("7d", "35d", "90d", "365d")
-            val vidStatFeat = List(
-              ("b20", shortPeriod, getJsonObject(record, "b20_feature")), // cate2_feature
-              ("b21", shortPeriod, getJsonObject(record, "b21_feature")), // cate1_feature
-              ("b22", shortPeriod, getJsonObject(record, "b22_feature")), // source_feature
-              ("b28", shortPeriod, getJsonObject(record, "b28_feature")), // sence_type_feature
-              ("b29", shortPeriod, getJsonObject(record, "b29_feature")), // alg_videoid_feature
-              ("b23", middlePeriod, getJsonObject(record, "b23_feature")), // cate2_feature_day
-              ("b24", middlePeriod, getJsonObject(record, "b24_feature")), // cate1_feature_day
-              ("b25", middlePeriod, getJsonObject(record, "b25_feature")), // source_feature_day
-              ("b26", longPeriod, getJsonObject(record, "b26_feature")), // unionid_feature_day
-              ("b27", longPeriod, getJsonObject(record, "b27_feature")) // vid_feature_day
-            )
-            for ((featType, featPeriod, featData) <- vidStatFeat) {
-              for (period <- featPeriod) {
-                // val view = if (featData.isEmpty) 0D else featData.getDoubleValue("view_" + period)
-                val share = if (featData.isEmpty) 0D else featData.getDoubleValue("share_" + period)
-                val return_ = if (featData.isEmpty) 0D else featData.getDoubleValue("return_" + period)
-                val view_hasreturn = if (featData.isEmpty) 0D else featData.getDoubleValue("view_hasreturn_" + period)
-                val share_hasreturn = if (featData.isEmpty) 0D else featData.getDoubleValue("share_hasreturn_" + period)
-                val ros = if (featData.isEmpty) 0D else featData.getDoubleValue("ros_" + period)
-                val rov = if (featData.isEmpty) 0D else featData.getDoubleValue("rov_" + period)
-                val r_cnt = if (featData.isEmpty) 0D else featData.getDoubleValue("r_cnt_" + period)
-                val r_rate = if (featData.isEmpty) 0D else featData.getDoubleValue("r_rate_" + period)
-                val r_cnt4s = if (featData.isEmpty) 0D else featData.getDoubleValue("r_cnt4s_" + period)
-                val str = if (featData.isEmpty) 0D else featData.getDoubleValue("str_" + period)
-                // scale
-                // val view_s = RankExtractorFeature_20240530.calLog(view)
-                val share_s = RankExtractorFeature_20240530.calLog(share)
-                val return_s = RankExtractorFeature_20240530.calLog(return_)
-                val view_hasreturn_s = RankExtractorFeature_20240530.calLog(view_hasreturn)
-                val share_hasreturn_s = RankExtractorFeature_20240530.calLog(share_hasreturn)
-
-                // featureMap.put(featType + "_" + period + "_" + "view", view_s)
-                featureMap.put(featType + "_" + period + "_" + "share", share_s)
-                featureMap.put(featType + "_" + period + "_" + "return", return_s)
-                featureMap.put(featType + "_" + period + "_" + "view_hasreturn", view_hasreturn_s)
-                featureMap.put(featType + "_" + period + "_" + "share_hasreturn", share_hasreturn_s)
-                featureMap.put(featType + "_" + period + "_" + "ros", ros)
-                featureMap.put(featType + "_" + period + "_" + "rov", rov)
-                featureMap.put(featType + "_" + period + "_" + "r_cnt", r_cnt)
-                featureMap.put(featType + "_" + period + "_" + "r_rate", r_rate)
-                featureMap.put(featType + "_" + period + "_" + "r_cnt4s", r_cnt4s)
-                featureMap.put(featType + "_" + period + "_" + "str", str)
-              }
-            }
-
-            // new cf
-            val d2345Data = List(
-              ("d2", "rosn", getJsonObject(record, "d2_feature")),
-              ("d3", "rosn", getJsonObject(record, "d3_feature")),
-              ("d4", "rovn", getJsonObject(record, "d4_feature")),
-              ("d5", "rovn", getJsonObject(record, "d5_feature"))
-            )
-            for ((featType, valType, featData) <- d2345Data) {
-              if (featData.nonEmpty) {
-                val expKey = if (valType.equals("rosn")) "share" else "exp"
-                val exp = if (featData.containsKey(expKey)) featData.getString(expKey).toDouble else 0D
-                val return_n = if (featData.containsKey("return_n")) featData.getString("return_n").toDouble else 0D
-                val value = if (featData.containsKey(valType)) featData.getString(valType).toDouble else 0D
-                // scale
-                val exp_s = RankExtractorFeature_20240530.calLog(exp)
-                val return_n_s = RankExtractorFeature_20240530.calLog(return_n)
-                featureMap.put(featType + "_exp", exp_s)
-                featureMap.put(featType + "_return_n", return_n_s)
-                featureMap.put(featType + "_" + valType, value)
-              }
-            }
-
-            if (!vid.equals("")) {
-              val idScoreObj = getJsonObject(getJsonObject(record, "d6_feature"), "vids", "scores")
-              if (idScoreObj.nonEmpty && idScoreObj.containsKey(vid)) {
-                val score = idScoreObj.getString(vid).toDouble
-                featureMap.put("d6", score)
-              }
-            }
-
-            // head video & rank video
-            val headVideo = getJsonObject(record, "v2_feature")
-            val rankVideo = video_info
-            if (headVideo.nonEmpty && rankVideo.nonEmpty) {
-              val videoAttrs = List("title", "topic", "keywords", "cate1_list", "cate2", "cate2_list", "style", "theme", "user_value")
-              for (attr <- videoAttrs) {
-                val headAttr = if (headVideo.containsKey(attr)) headVideo.getString(attr) else ""
-                val rankAttr = if (rankVideo.containsKey(attr)) rankVideo.getString(attr) else ""
-                if (headAttr.nonEmpty && !headAttr.equals("unknown") && rankAttr.nonEmpty && !rankAttr.equals("unknown")) {
-                  val simScore = SimilarityUtils.word2VecSimilarity(headAttr, rankAttr)
-                  featureMap.put("video_sim_" + attr, simScore)
-                }
-              }
-            }
-
-            // time
-            val ts = record.getString("ts")
-            val calendar = tsToCalendar(ts)
-            if (null != calendar) {
-              val week = calendar.get(Calendar.DAY_OF_WEEK)
-              val hour = calendar.get(Calendar.HOUR_OF_DAY) + 1
-              featureMap.put("week", week)
-              featureMap.put("hour", hour)
-            }
-
-            /*
-            视频特征： 5*6*5 = 240个
-                      曝光使用pv 分享使用pv 回流使用uv --> 1h 2h 3h 4h 12h 1d 3d 7d
-                      STR log(share) ROV log(return) ROV*log(return) ROS
-                      整体、整体曝光对应、推荐非冷启root、推荐冷启root、分省份root
-            视频基础： 2个   视频时长、比特率
-            用户： 4+8 = 12个
-                      播放次数 --> 6h 1d 3d 7d --> 4个
-                      带回来的分享pv 回流uv --> 12h 1d 3d 7d --> 8个
-            人+vid-title:  5*3*3 = 45
-                      播放点/回流点/分享点/累积分享/累积回流 --> 1d 3d 7d --> 匹配数量 语义最高相似度分 语义平均相似度分 --> 45个
-            人+vid-cf： 2*3*3 = 12
-                      基于分享行为/基于回流行为 -->  “分享cf”+”回流点击cf“ 相似分 相似数量 相似rank的倒数 --> 12个
-            头部视频：  3
-                      曝光 回流 ROVn 3个特征
-            场景：     小时 星期 apptype city province pagesource 机器型号
-            总量：     240+2+12+45+12+3 = 314
-            ---------------------------------------------------------------
-            视频特征：(4*7+3*2+2*4)*10 = 420个
-            CF: 13个
-            视频相似特征: 9个
-             */
-
-            //4 处理label信息。
-            val labels = new JSONObject
-            for (labelKey <- List("is_share", "share_cnt", "is_return_1", "return_1_uv", "is_return_n", "return_n_uv", "is_return_noself", "return_1_uv_noself")) {
-              if (!record.isNull(labelKey)) {
-                labels.put(labelKey, record.getString(labelKey))
-              }
-            }
-            //5 处理log key表头。
-            val apptype = record.getString("apptype")
-            val page = getStringValue(record, "page")
-            val pagesource = getStringValue(record, "pagesource")
-            val recommendpagetype = getStringValue(record, "recommendpagetype")
-            val flowpool = getStringValue(record, "flowpool")
-            val abcode = record.getString("abcode")
-            val mid = record.getString("mid")
-            val level = getStringValue(record, "level", "0")
-            val logKey = (apptype, page, pagesource, recommendpagetype, flowpool, abcode, mid, vid, level, ts).productIterator.mkString(",")
-            val labelKey = labels.toString()
-            val featureKey = truncateDecimal(featureMap).toString()
-            val scoresMap = getSubJson(record, "extend_alg", "scoresMap").toString()
-            //6 拼接数据，保存。
-            logKey + "\t" + labelKey + "\t" + scoresMap + "\t" + featureKey
-          })
-        })
-
-      // 4 保存数据到hdfs
-      val savePartition = dt + hh
-      val hdfsPath = savePath + "/" + savePartition
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        odpsData.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  def func(record: Record, schema: TableSchema): Record = {
-    record
-  }
-
-  def getJsonObject(record: Record, key: String): JSONObject = {
-    val data = if (record.isNull(key)) new JSONObject() else JSON.parseObject(record.getString(key))
-    val data2 = new JSONObject()
-    data.foreach(r => {
-      if (r._2 != null) {
-        data2.put(r._1, r._2)
-      }
-    })
-    data2
-  }
-
-  def getJsonObject(obj: JSONObject, key: String): JSONObject = {
-    if (obj.nonEmpty) {
-      if (obj.containsKey(key)) {
-        val data = obj.getJSONObject(key)
-        val validData = new JSONObject()
-        data.foreach(r => {
-          if (r._2 != null) {
-            validData.put(r._1, r._2)
-          }
-        })
-        return validData
-      }
-    }
-    new JSONObject()
-  }
-
-  def getJsonObject(obj: JSONObject, onlineKey: String, record: Record, offlineKey: String): JSONObject = {
-    var data: JSONObject = null
-    if (obj.nonEmpty) {
-      if (obj.containsKey(onlineKey)) {
-        data = obj.getJSONObject(onlineKey)
-      }
-    } else {
-      if (!record.isNull(offlineKey)) {
-        data = JSON.parseObject(record.getString(offlineKey))
-      }
-    }
-    if (null != data && data.nonEmpty) {
-      val validData = new JSONObject()
-      data.foreach(r => {
-        if (r._2 != null) {
-          validData.put(r._1, r._2)
-        }
-      })
-      return validData
-    }
-    new JSONObject()
-  }
-
-  def getJsonObject(obj: JSONObject, keyName: String, valueName: String): JSONObject = {
-    val map = new JSONObject()
-    if (obj.nonEmpty) {
-      val keys = if (obj.containsKey(keyName)) obj.getString(keyName) else ""
-      val values = if (obj.containsKey(valueName)) obj.getString(valueName) else ""
-      if (!keys.equals("") && !values.equals("")) {
-        val key_list = keys.split(",")
-        val value_list = values.split(",")
-        if (key_list.length == value_list.length) {
-          for (index <- 0 until key_list.length) {
-            map.put(key_list(index), value_list(index))
-          }
-        }
-      }
-    }
-    map
-  }
-
-  def truncateDecimal(obj: JSONObject, scale: Int = 6): JSONObject = {
-    val data = new JSONObject()
-    for (key <- obj.keySet()) {
-      try {
-        val value = obj.getDoubleValue(key)
-        if (value == value.floor) {
-          data.put(key, value)
-        } else {
-          val newValue = BigDecimal(value).setScale(scale, BigDecimal.RoundingMode.HALF_UP).toDouble
-          data.put(key, newValue)
-        }
-      } catch {
-        case e: Exception => System.err.println(e.getMessage)
-      }
-    }
-    data
-  }
-
-  def funcC34567ForTagsW2V(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
-    // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
-    val tagsList = tags.split(",")
-    var d1 = 0.0
-    val d2 = new ArrayBuffer[String]()
-    var d3 = 0.0
-    var d4 = 0.0
-    for (tag <- tagsList) {
-      if (title.contains(tag)) {
-        d1 = d1 + 1.0
-        d2.add(tag)
-      }
-      val score = SimilarityUtils.word2VecSimilarity(tag, title)
-      d3 = if (score > d3) score else d3
-      d4 = d4 + score
-    }
-    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
-    (d1, d2.mkString(","), d3, d4)
-  }
-
-  def getSubJson(record: Record, key1: String, key2: String): JSONObject = {
-    if (!record.isNull(key1)) {
-      val obj = JSON.parseObject(record.getString(key1))
-      if (obj.nonEmpty && obj.containsKey(key2)) {
-        val data = obj.getString(key2)
-        return JSON.parseObject(data.replace("\\", ""))
-      }
-    }
-    new JSONObject()
-  }
-
-  def getStringValue(record: Record, key: String, default: String = ""): String = {
-    if (!record.isNull(key)) {
-      val value = record.getString(key)
-      return value.trim
-    }
-    default
-  }
-
-  def tsToCalendar(ts: String): Calendar = {
-    if (null != ts && ts.nonEmpty) {
-      val ms = ts.toLong * 1000L
-      if (ms > 1107658247000L && ms < 2685495047000L) {
-        val calendar = Calendar.getInstance
-        calendar.setTimeInMillis(ms)
-        return calendar
-      }
-    }
-    null
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_81_nor_sample_20250217.scala

@@ -1,192 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.JSON
-import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils}
-import examples.extractor.ExtractorUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.io.Source
-import scala.util.Random
-
-object makedata_recsys_81_nor_sample_20250217 {
-  def main(args: Array[String]): Unit = {
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val readPath = param.getOrElse("readPath", "/dw/recommend/model/81_origin_data/")
-    val beginStr = param.getOrElse("beginStr", "20250118")
-    val endStr = param.getOrElse("endStr", "20250118")
-    val whatApps = param.getOrElse("whatApps", "0,4,5,21,3,6").split(",").toSet
-    val whatLabel = param.getOrElse("whatLabel", "return_n_uv_noself")
-    val fuSampleRate = param.getOrElse("fuSampleRate", "-1.0").toDouble
-    val notUseBucket = param.getOrElse("notUseBucket", "0").toInt
-    val featureNameFile = param.getOrElse("featureName", "20241209_recsys_nor_name.txt")
-    val featureBucketFile = param.getOrElse("featureBucket", "20241209_recsys_nor_bucket.txt")
-    val repartition = param.getOrElse("repartition", "100").toInt
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/81_recsys_nor_train_data/")
-
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    val loader = getClass.getClassLoader
-    val featureNameSet = loadUseFeatureNames(loader, featureNameFile)
-    val featureBucketMap = loadUseFeatureBuckets(loader, notUseBucket, featureBucketFile)
-    val bucketsMap_br = sc.broadcast(featureBucketMap)
-
-    val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
-    for (date <- dateRange) {
-      println("开始执行:" + date)
-      val data = sc.textFile(readPath + "/" + date + "*").map(r => {
-          // logKey + "\t" + labelKey + "\t" + scoresMap + "\t" + featureKey
-          val rList = r.split("\t")
-          val logKey = rList(0)
-          val labelKey = rList(1)
-          val scoresMap = rList(2)
-          val featData = rList(3)
-          (logKey, labelKey, scoresMap, featData)
-        })
-        .filter {
-          case (logKey, labelKey, scoresMap, featData) =>
-            validData(logKey, whatApps)
-        }.filter {
-          case (logKey, labelKey, scoresMap, featData) =>
-            val label = parseLabel(labelKey, whatLabel).toDouble
-            label > 0 || new Random().nextDouble() <= fuSampleRate
-        }
-        .map {
-          case (logKey, labelKey, scoresMap, featData) =>
-            val label = parseLabel(labelKey, whatLabel).toDouble
-            val features = parseFeature(featData)
-            (logKey, label, scoresMap, features)
-        }
-        .mapPartitions(row => {
-          val result = new ArrayBuffer[String]()
-          val bucketsMap = bucketsMap_br.value
-          row.foreach {
-            case (logKey, label, scoresMap, features) =>
-              val featuresBucket = bucketFeature(featureNameSet, bucketsMap, features)
-              result.add(logKey + "\t" + label + "\t" + scoresMap + "\t" + featuresBucket.mkString("\t"))
-          }
-          result.iterator
-        })
-
-      // 4 保存数据到hdfs
-      val hdfsPath = savePath + "/" + date
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        data.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  private def loadFileData(loader: ClassLoader, nameFile: String): String = {
-    val resourceUrlBucket = loader.getResource(nameFile)
-    val data =
-      if (resourceUrlBucket != null) {
-        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
-        Source.fromURL(resourceUrlBucket).close()
-        buckets
-      } else {
-        ""
-      }
-    data
-  }
-
-  private def loadUseFeatureNames(loader: ClassLoader, nameFile: String): Set[String] = {
-    val names = loadFileData(loader, nameFile)
-    println(names)
-    names.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .toSet
-  }
-
-  private def loadUseFeatureBuckets(loader: ClassLoader, notUseBucket: Int, nameFile: String): Map[String, (Double, Array[Double])] = {
-    if (notUseBucket > 0) {
-      return Map[String, (Double, Array[Double])]()
-    }
-    val buckets = loadFileData(loader, nameFile)
-    println(buckets)
-    buckets.split("\n")
-      .map(r => r.replace(" ", "").replaceAll("\n", ""))
-      .filter(r => r.nonEmpty)
-      .map(r => {
-        val rList = r.split("\t")
-        (rList(0), (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
-      }).toMap
-  }
-
-  private def recommendFlow(flowPool: String): Boolean = {
-    if (flowPool.isEmpty || flowPool.endsWith("#1")) {
-      return true
-    }
-    false
-  }
-
-  private def validData(logKey: String, whatApps: Set[String]): Boolean = {
-    // apptype, page, pagesource, recommendpagetype, flowpool, abcode, mid, vid, level, ts
-    val cells = logKey.split(",")
-    val apptype = cells(0)
-    val page = cells(1)
-    //val pagesource = cells(2)
-    val recommendpagetype = cells(3)
-    val flowpool = cells(4)
-    if (whatApps.contains(apptype)) {
-      if (recommendFlow(flowpool)) {
-        if (page.equals("详情后沉浸页")) {
-          return true
-        } else if (page.equals("回流后沉浸页&内页feed")) {
-          return true
-        } else if (page.equals("首页feed")) {
-          return true
-        }
-      }
-    }
-    false
-  }
-
-  private def parseLabel(data: String, key: String, default: String = "0"): String = {
-    JSON.parseObject(data).getOrDefault(key, default).toString
-  }
-
-  private def parseFeature(data: String): scala.collection.mutable.Map[String, Double] = {
-    val features = scala.collection.mutable.Map[String, Double]()
-    if (data.nonEmpty) {
-      val obj = JSON.parseObject(data)
-      obj.foreach(r => {
-        features.put(r._1, obj.getDoubleValue(r._1))
-      })
-    }
-    features
-  }
-
-  private def bucketFeature(nameSet: Set[String], bucketMap: Map[String, (Double, Array[Double])], features: scala.collection.mutable.Map[String, Double]): Iterable[String] = {
-    features.map {
-      case (name, score) =>
-        if (!nameSet.contains(name)) {
-          ""
-        } else {
-          if (score > 1E-8) {
-            if (bucketMap.contains(name)) {
-              val (bucketsNum, buckets) = bucketMap(name)
-              val scoreNew = 1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
-              name + ":" + scoreNew.toString
-            } else {
-              name + ":" + score.toString
-            }
-          } else {
-            ""
-          }
-        }
-    }.filter(_.nonEmpty)
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_81_originData_20250217.scala

@@ -1,490 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.alibaba.fastjson.{JSON, JSONObject}
-import com.aliyun.odps.TableSchema
-import com.aliyun.odps.data.Record
-import com.aliyun.odps.spark.examples.myUtils._
-import examples.extractor.RankExtractorFeature_20240530
-import examples.utils.SimilarityUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.sql.SparkSession
-
-import java.util.Calendar
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.util.Random
-
-/*
-   20250217 提取特征
- */
-
-object makedata_recsys_81_originData_20250217 {
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    // 1 读取参数
-    val param = ParamUtils.parseArgs(args)
-    val project = param.getOrElse("project", "loghubods")
-    val table = param.getOrElse("table", "alg_recsys_sample_tmp_20250109")
-    val tablePart = param.getOrElse("tablePart", "64").toInt
-    val beginStr = param.getOrElse("beginStr", "2025011800")
-    val endStr = param.getOrElse("endStr", "2025011800")
-    val whatLabel = param.getOrElse("whatLabel", "is_return_n_noself")
-    val fuSampleRate = param.getOrElse("fuSampleRate", "0.1").toDouble
-    val smoothPlus = param.getOrElse("smoothPlus", "5").toDouble
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/81_origin_data/")
-    val repartition = param.getOrElse("repartition", "32").toInt
-
-    // 2 odps
-    val odpsOps = env.getODPS(sc)
-
-    // 3 循环执行数据生产
-    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
-    for (dt_hh <- timeRange) {
-      val dt = dt_hh.substring(0, 8)
-      val hh = dt_hh.substring(8, 10)
-      val partition = s"dt=$dt,hh=$hh"
-      println("开始执行partiton:" + partition)
-      val odpsData = odpsOps.readTable(project = project,
-          table = table,
-          partition = partition,
-          transfer = func,
-          numPartition = tablePart)
-        .filter(record => {
-          val label = getStringValue(record, whatLabel, "0")
-          "1".equals(label) || new Random().nextDouble() <= fuSampleRate
-        })
-        .mapPartitions(p => {
-          SimilarityUtils.init()
-          p.map(record => {
-            val featureMap = new JSONObject()
-
-            // a 视频特征
-            val b1: JSONObject = getJsonObject(record, "b1_feature")
-            val b2: JSONObject = getJsonObject(record, "b2_feature")
-            val b3: JSONObject = getJsonObject(record, "b3_feature")
-            val b6: JSONObject = getJsonObject(record, "b6_feature")
-            val b7: JSONObject = getJsonObject(record, "b7_feature")
-
-            val b8: JSONObject = getJsonObject(record, "b8_feature")
-            val b9: JSONObject = getJsonObject(record, "b9_feature")
-            val b10: JSONObject = getJsonObject(record, "b10_feature")
-            val b11: JSONObject = getJsonObject(record, "b11_feature")
-            val b12: JSONObject = getJsonObject(record, "b12_feature")
-            val b13: JSONObject = getJsonObject(record, "b13_feature")
-            val b17: JSONObject = getJsonObject(record, "b17_feature")
-            val b18: JSONObject = getJsonObject(record, "b18_feature")
-            val b19: JSONObject = getJsonObject(record, "b19_feature")
-
-            val origin_data = List(
-              (b1, b2, b3, "b123"), (b1, b6, b7, "b167"),
-              (b8, b9, b10, "b8910"), (b11, b12, b13, "b111213"),
-              (b17, b18, b19, "b171819")
-            )
-            for ((b_1, b_2, b_3, prefix1) <- origin_data) {
-              for (prefix2 <- List(
-                "1h", "2h", "3h", "4h", "12h", "1d", "3d", "7d"
-              )) {
-                val exp = if (b_1.isEmpty) 0D else b_1.getIntValue("exp_pv_" + prefix2).toDouble
-                val share = if (b_2.isEmpty) 0D else b_2.getIntValue("share_pv_" + prefix2).toDouble
-                val returns = if (b_3.isEmpty) 0D else b_3.getIntValue("return_uv_" + prefix2).toDouble
-                val f0 = RankExtractorFeature_20240530.calLog(exp)
-                val f1 = RankExtractorFeature_20240530.calDiv(share, exp)
-                val f2 = RankExtractorFeature_20240530.calLog(share)
-                val f3 = RankExtractorFeature_20240530.calDiv(returns, exp)
-                val f4 = RankExtractorFeature_20240530.calLog(returns)
-                val f5 = f3 * f4
-                val f6 = RankExtractorFeature_20240530.calDiv(returns, share)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "EXP", f0)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "STR", f1)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "log(share)", f2)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROV", f3)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "log(return)", f4)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROV*log(return)", f5)
-                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ROS", f6)
-              }
-            }
-
-            val video_info: JSONObject = getJsonObject(record, "t_v_info_feature")
-            featureMap.put("total_time", if (video_info.containsKey("total_time")) video_info.getIntValue("total_time").toDouble else 0D)
-            featureMap.put("bit_rate", if (video_info.containsKey("bit_rate")) video_info.getIntValue("bit_rate").toDouble else 0D)
-
-            val c1: JSONObject = getJsonObject(record, "c1_feature")
-            if (c1.nonEmpty) {
-              featureMap.put("playcnt_6h", if (c1.containsKey("playcnt_6h")) c1.getIntValue("playcnt_6h").toDouble else 0D)
-              featureMap.put("playcnt_1d", if (c1.containsKey("playcnt_1d")) c1.getIntValue("playcnt_1d").toDouble else 0D)
-              featureMap.put("playcnt_3d", if (c1.containsKey("playcnt_3d")) c1.getIntValue("playcnt_3d").toDouble else 0D)
-              featureMap.put("playcnt_7d", if (c1.containsKey("playcnt_7d")) c1.getIntValue("playcnt_7d").toDouble else 0D)
-            }
-            val c2: JSONObject = getJsonObject(record, "c2_feature")
-            if (c2.nonEmpty) {
-              featureMap.put("share_pv_12h", if (c2.containsKey("share_pv_12h")) c2.getIntValue("share_pv_12h").toDouble else 0D)
-              featureMap.put("share_pv_1d", if (c2.containsKey("share_pv_1d")) c2.getIntValue("share_pv_1d").toDouble else 0D)
-              featureMap.put("share_pv_3d", if (c2.containsKey("share_pv_3d")) c2.getIntValue("share_pv_3d").toDouble else 0D)
-              featureMap.put("share_pv_7d", if (c2.containsKey("share_pv_7d")) c2.getIntValue("share_pv_7d").toDouble else 0D)
-              featureMap.put("return_uv_12h", if (c2.containsKey("return_uv_12h")) c2.getIntValue("return_uv_12h").toDouble else 0D)
-              featureMap.put("return_uv_1d", if (c2.containsKey("return_uv_1d")) c2.getIntValue("return_uv_1d").toDouble else 0D)
-              featureMap.put("return_uv_3d", if (c2.containsKey("return_uv_3d")) c2.getIntValue("return_uv_3d").toDouble else 0D)
-              featureMap.put("return_uv_7d", if (c2.containsKey("return_uv_7d")) c2.getIntValue("return_uv_7d").toDouble else 0D)
-            }
-
-            val title = if (video_info.containsKey("title")) video_info.getString("title") else ""
-            if (!title.equals("")) {
-              for (key_feature <- List("c3_feature", "c4_feature", "c5_feature", "c6_feature", "c7_feature")) {
-                val c34567: JSONObject = if (record.isNull(key_feature)) new JSONObject() else
-                  JSON.parseObject(record.getString(key_feature))
-                for (key_time <- List("tags_1d", "tags_3d", "tags_7d")) {
-                  val tags = if (c34567.containsKey(key_time)) c34567.getString(key_time) else ""
-                  if (!tags.equals("")) {
-                    val (f1, f2, f3, f4) = funcC34567ForTagsW2V(tags, title)
-                    featureMap.put(key_feature + "_" + key_time + "_matchnum", f1)
-                    featureMap.put(key_feature + "_" + key_time + "_maxscore", f3)
-                    featureMap.put(key_feature + "_" + key_time + "_avgscore", f4)
-                  }
-                }
-              }
-            }
-
-            val vid = if (record.isNull("vid")) "" else record.getString("vid")
-            if (!vid.equals("")) {
-              for (key_feature <- List("c8_feature", "c9_feature")) {
-                val c89: JSONObject = if (record.isNull(key_feature)) new JSONObject() else
-                  JSON.parseObject(record.getString(key_feature))
-                for (key_action <- List("share", "return")) {
-                  val cfListStr = if (c89.containsKey(key_action)) c89.getString(key_action) else ""
-                  if (!cfListStr.equals("")) {
-                    val cfMap = cfListStr.split(",").map(r => {
-                      val rList = r.split(":")
-                      (rList(0), (rList(1), rList(2), rList(3)))
-                    }).toMap
-                    if (cfMap.contains(vid)) {
-                      val (score, num, rank) = cfMap(vid)
-                      featureMap.put(key_feature + "_" + key_action + "_score", score.toDouble)
-                      featureMap.put(key_feature + "_" + key_action + "_num", num.toDouble)
-                      featureMap.put(key_feature + "_" + key_action + "_rank", 1.0 / rank.toDouble)
-                    }
-                  }
-                }
-              }
-            }
-
-            val d1: JSONObject = getJsonObject(record, "d1_feature")
-            if (d1.nonEmpty) {
-              featureMap.put("d1_exp", if (d1.containsKey("exp")) d1.getString("exp").toDouble else 0D)
-              featureMap.put("d1_return_n", if (d1.containsKey("return_n")) d1.getString("return_n").toDouble else 0D)
-              featureMap.put("d1_rovn", if (d1.containsKey("rovn")) d1.getString("rovn").toDouble else 0D)
-            }
-
-            // ************* new feature *************
-            val shortPeriod = List("1h", "2h", "4h", "6h", "12h", "24h", "7d")
-            val middlePeriod = List("14d", "30d")
-            val longPeriod = List("7d", "35d", "90d", "365d")
-            val vidStatFeat = List(
-              ("b20", shortPeriod, getJsonObject(record, "b20_feature")), // cate2_feature
-              ("b21", shortPeriod, getJsonObject(record, "b21_feature")), // cate1_feature
-              ("b22", shortPeriod, getJsonObject(record, "b22_feature")), // source_feature
-              ("b28", shortPeriod, getJsonObject(record, "b28_feature")), // sence_type_feature
-              ("b29", shortPeriod, getJsonObject(record, "b29_feature")), // alg_videoid_feature
-              ("b23", middlePeriod, getJsonObject(record, "b23_feature")), // cate2_feature_day
-              ("b24", middlePeriod, getJsonObject(record, "b24_feature")), // cate1_feature_day
-              ("b25", middlePeriod, getJsonObject(record, "b25_feature")), // source_feature_day
-              ("b26", longPeriod, getJsonObject(record, "b26_feature")), // unionid_feature_day
-              ("b27", longPeriod, getJsonObject(record, "b27_feature")) // vid_feature_day
-            )
-            for ((featType, featPeriod, featData) <- vidStatFeat) {
-              for (period <- featPeriod) {
-                val view = if (featData.isEmpty) 0D else featData.getDoubleValue("view_" + period)
-                val share = if (featData.isEmpty) 0D else featData.getDoubleValue("share_" + period)
-                val return_ = if (featData.isEmpty) 0D else featData.getDoubleValue("return_" + period)
-                val view_hasreturn = if (featData.isEmpty) 0D else featData.getDoubleValue("view_hasreturn_" + period)
-                val share_hasreturn = if (featData.isEmpty) 0D else featData.getDoubleValue("share_hasreturn_" + period)
-                var ros = if (featData.isEmpty) 0D else featData.getDoubleValue("ros_" + period)
-                var rov = if (featData.isEmpty) 0D else featData.getDoubleValue("rov_" + period)
-                var r_cnt = if (featData.isEmpty) 0D else featData.getDoubleValue("r_cnt_" + period)
-                var r_rate = if (featData.isEmpty) 0D else featData.getDoubleValue("r_rate_" + period)
-                var r_cnt4s = if (featData.isEmpty) 0D else featData.getDoubleValue("r_cnt4s_" + period)
-                var str = if (featData.isEmpty) 0D else featData.getDoubleValue("str_" + period)
-                var s2r_rate = 0D
-                if (smoothPlus > 0) {
-                  ros = RankExtractorFeature_20240530.smoothDiv(return_, share, smoothPlus)
-                  rov = RankExtractorFeature_20240530.smoothDiv(return_, view, smoothPlus)
-                  r_cnt = RankExtractorFeature_20240530.smoothDiv(return_, view_hasreturn, smoothPlus)
-                  r_rate = RankExtractorFeature_20240530.smoothDiv(view_hasreturn, view, smoothPlus)
-                  r_cnt4s = RankExtractorFeature_20240530.smoothDiv(return_, share_hasreturn, smoothPlus)
-                  str = RankExtractorFeature_20240530.smoothDiv(share, view, smoothPlus)
-                  s2r_rate = RankExtractorFeature_20240530.smoothDiv(share_hasreturn, share, smoothPlus)
-                }
-
-                // scale
-                val view_s = RankExtractorFeature_20240530.calLog(view)
-                val share_s = RankExtractorFeature_20240530.calLog(share)
-                val return_s = RankExtractorFeature_20240530.calLog(return_)
-                val view_hasreturn_s = RankExtractorFeature_20240530.calLog(view_hasreturn)
-                val share_hasreturn_s = RankExtractorFeature_20240530.calLog(share_hasreturn)
-
-                featureMap.put(featType + "_" + period + "_" + "view", view_s)
-                featureMap.put(featType + "_" + period + "_" + "share", share_s)
-                featureMap.put(featType + "_" + period + "_" + "return", return_s)
-                featureMap.put(featType + "_" + period + "_" + "view_hasreturn", view_hasreturn_s)
-                featureMap.put(featType + "_" + period + "_" + "share_hasreturn", share_hasreturn_s)
-                featureMap.put(featType + "_" + period + "_" + "ros", ros)
-                featureMap.put(featType + "_" + period + "_" + "rov", rov)
-                featureMap.put(featType + "_" + period + "_" + "r_cnt", r_cnt)
-                featureMap.put(featType + "_" + period + "_" + "r_rate", r_rate)
-                featureMap.put(featType + "_" + period + "_" + "r_cnt4s", r_cnt4s)
-                featureMap.put(featType + "_" + period + "_" + "str", str)
-                featureMap.put(featType + "_" + period + "_" + "s2r_rate", s2r_rate)
-              }
-            }
-
-            // new cf
-            val d2345Data = List(
-              ("d2", "rosn", getJsonObject(record, "d2_feature")),
-              ("d3", "rosn", getJsonObject(record, "d3_feature")),
-              ("d4", "rovn", getJsonObject(record, "d4_feature")),
-              ("d5", "rovn", getJsonObject(record, "d5_feature"))
-            )
-            for ((featType, valType, featData) <- d2345Data) {
-              if (featData.nonEmpty) {
-                val exp = if (featData.containsKey("exp")) featData.getString("exp").toDouble else 0D
-                val return_n = if (featData.containsKey("return_n")) featData.getString("return_n").toDouble else 0D
-                val value = if (featData.containsKey(valType)) featData.getString(valType).toDouble else 0D
-                // scale
-                val exp_s = RankExtractorFeature_20240530.calLog(exp)
-                val return_n_s = RankExtractorFeature_20240530.calLog(return_n)
-                featureMap.put(featType + "_exp", exp_s)
-                featureMap.put(featType + "_return_n", return_n_s)
-                featureMap.put(featType + "_" + valType, value)
-              }
-            }
-
-            if (!vid.equals("")) {
-              val idScoreObj = getJsonObject(getJsonObject(record, "d6_feature"), "vids", "scores")
-              if (idScoreObj.nonEmpty && idScoreObj.containsKey(vid)) {
-                val score = idScoreObj.getString(vid).toDouble
-                featureMap.put("d6", score)
-              }
-            }
-
-            // head video & rank video
-            val headVideo = getJsonObject(record, "v2_feature")
-            val rankVideo = getJsonObject(record, "v1_feature")
-            if (headVideo.nonEmpty && rankVideo.nonEmpty) {
-              val videoAttrs = List("title", "topic", "keywords", "cate1_list", "cate2", "cate2_list", "style", "theme", "user_value")
-              for (attr <- videoAttrs) {
-                val headAttr = if (headVideo.containsKey(attr)) headVideo.getString(attr) else ""
-                val rankAttr = if (rankVideo.containsKey(attr)) rankVideo.getString(attr) else ""
-                if (!headAttr.equals("") && !rankAttr.equals("")) {
-                  val simScore = SimilarityUtils.word2VecSimilarity(headAttr, rankAttr)
-                  featureMap.put("video_sim_" + attr, simScore)
-                }
-              }
-            }
-
-            // time
-            val ts = record.getString("ts")
-
-            // category
-            if (rankVideo.nonEmpty) {
-              val attrList = List(FeatureUtils.cate1Attr, FeatureUtils.cate2Attr, FeatureUtils.festive1Attr)
-              for (attr <- attrList) {
-                val attrVal = if (rankVideo.containsKey(attr)) rankVideo.getString(attr) else ""
-                val attrId = FeatureUtils.getAttrId(attr, attrVal)
-                if (attrId > 0) {
-                  val key = "%s@%s@%d".format("r", attr, attrId)
-                  featureMap.put(key, 1)
-                }
-              }
-            }
-
-            // createTime
-            if (null != ts && ts.nonEmpty && rankVideo.containsKey("gmt_create_timestamp")) {
-              val currentMs = ts.toLong * 1000L
-              val createMs = rankVideo.getString("gmt_create_timestamp").toLong
-              val createTime = FeatureUtils.getTimeDiff(currentMs, createMs)
-              featureMap.put("createTime", 1 - createTime)
-            }
-
-            // week & hour
-            val calendar = tsToCalendar(ts)
-            if (null != calendar) {
-              val week = calendar.get(Calendar.DAY_OF_WEEK)
-              val hour = calendar.get(Calendar.HOUR_OF_DAY) + 1
-              featureMap.put("week", week)
-              featureMap.put("hour", hour)
-            }
-
-            /*
-            视频特征： 5*6*5 = 240个
-                      曝光使用pv 分享使用pv 回流使用uv --> 1h 2h 3h 4h 12h 1d 3d 7d
-                      STR log(share) ROV log(return) ROV*log(return) ROS
-                      整体、整体曝光对应、推荐非冷启root、推荐冷启root、分省份root
-            视频基础： 2个   视频时长、比特率
-            用户： 4+8 = 12个
-                      播放次数 --> 6h 1d 3d 7d --> 4个
-                      带回来的分享pv 回流uv --> 12h 1d 3d 7d --> 8个
-            人+vid-title:  5*3*3 = 45
-                      播放点/回流点/分享点/累积分享/累积回流 --> 1d 3d 7d --> 匹配数量 语义最高相似度分 语义平均相似度分 --> 45个
-            人+vid-cf： 2*3*3 = 12
-                      基于分享行为/基于回流行为 -->  “分享cf”+”回流点击cf“ 相似分 相似数量 相似rank的倒数 --> 12个
-            头部视频：  3
-                      曝光 回流 ROVn 3个特征
-            场景：     小时 星期 apptype city province pagesource 机器型号
-            总量：     240+2+12+45+12+3 = 314
-            ---------------------------------------------------------------
-            视频特征：(4*7+3*2+2*4)*10 = 420个
-            CF: 13个
-            视频相似特征: 9个
-             */
-
-
-            //4 处理label信息。
-            val labels = new JSONObject
-            for (labelKey <- List("is_share", "share_cnt",
-              "is_return_1", "return_1_uv",
-              "is_return_n", "return_n_uv",
-              "is_return_noself", "return_1_uv_noself",
-              "is_return_n_noself", "return_n_uv_noself"
-            )) {
-              if (!record.isNull(labelKey)) {
-                labels.put(labelKey, record.getString(labelKey))
-              }
-            }
-            //5 处理log key表头。
-            val apptype = record.getString("apptype")
-            val page = getStringValue(record, "page")
-            val pagesource = getStringValue(record, "pagesource")
-            val recommendpagetype = getStringValue(record, "recommendpagetype")
-            val flowpool = getStringValue(record, "flowpool")
-            val abcode = record.getString("abcode")
-            val mid = record.getString("mid")
-            val level = getStringValue(record, "level", "0")
-            val logKey = (apptype, page, pagesource, recommendpagetype, flowpool, abcode, mid, vid, level, ts).productIterator.mkString(",")
-            val labelKey = labels.toString()
-            val featureKey = truncateDecimal(featureMap).toString()
-            val scoresMap = getSubJson(record, "extend_alg", "scoresMap").toString()
-            //6 拼接数据，保存。
-            logKey + "\t" + labelKey + "\t" + scoresMap + "\t" + featureKey
-          })
-        })
-
-      // 4 保存数据到hdfs
-      val savePartition = dt + hh
-      val hdfsPath = savePath + "/" + savePartition
-      if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
-        println("删除路径并开始数据写入:" + hdfsPath)
-        MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        odpsData.repartition(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
-      } else {
-        println("路径不合法，无法写入:" + hdfsPath)
-      }
-    }
-  }
-
-  def func(record: Record, schema: TableSchema): Record = {
-    record
-  }
-
-  def getJsonObject(record: Record, key: String): JSONObject = {
-    val data = if (record.isNull(key)) new JSONObject() else JSON.parseObject(record.getString(key))
-    val data2 = new JSONObject()
-    data.foreach(r => {
-      if (r._2 != null) {
-        data2.put(r._1, r._2)
-      }
-    })
-    data2
-  }
-
-  def getJsonObject(obj: JSONObject, keyName: String, valueName: String): JSONObject = {
-    val map = new JSONObject()
-    if (obj.nonEmpty) {
-      val keys = if (obj.containsKey(keyName)) obj.getString(keyName) else ""
-      val values = if (obj.containsKey(valueName)) obj.getString(valueName) else ""
-      if (!keys.equals("") && !values.equals("")) {
-        val key_list = keys.split(",")
-        val value_list = values.split(",")
-        if (key_list.length == value_list.length) {
-          for (index <- 0 until key_list.length) {
-            map.put(key_list(index), value_list(index))
-          }
-        }
-      }
-    }
-    map
-  }
-
-  def truncateDecimal(obj: JSONObject, scale: Int = 6): JSONObject = {
-    val data = new JSONObject()
-    for (key <- obj.keySet()) {
-      try {
-        val value = obj.getDoubleValue(key)
-        if (value == value.floor) {
-          data.put(key, value)
-        } else {
-          val newValue = BigDecimal(value).setScale(scale, BigDecimal.RoundingMode.HALF_UP).toDouble
-          data.put(key, newValue)
-        }
-      } catch {
-        case e: Exception => System.err.println(e.getMessage)
-      }
-    }
-    data
-  }
-
-
-  def funcC34567ForTagsW2V(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
-    // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
-    val tagsList = tags.split(",")
-    var d1 = 0.0
-    val d2 = new ArrayBuffer[String]()
-    var d3 = 0.0
-    var d4 = 0.0
-    for (tag <- tagsList) {
-      if (title.contains(tag)) {
-        d1 = d1 + 1.0
-        d2.add(tag)
-      }
-      val score = SimilarityUtils.word2VecSimilarity(tag, title)
-      d3 = if (score > d3) score else d3
-      d4 = d4 + score
-    }
-    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
-    (d1, d2.mkString(","), d3, d4)
-  }
-
-  def getSubJson(record: Record, key1: String, key2: String): JSONObject = {
-    if (!record.isNull(key1)) {
-      val obj = JSON.parseObject(record.getString(key1))
-      if (obj.nonEmpty && obj.containsKey(key2)) {
-        val data = obj.getString(key2)
-        return JSON.parseObject(data.replace("\\", ""))
-      }
-    }
-    new JSONObject()
-  }
-
-  def tsToCalendar(ts: String): Calendar = {
-    if (null != ts && ts.nonEmpty) {
-      val ms = ts.toLong * 1000L
-      if (ms > 1107658247000L && ms < 2685495047000L) {
-        val calendar = Calendar.getInstance
-        calendar.setTimeInMillis(ms)
-        return calendar
-      }
-    }
-    null
-  }
-
-  def getStringValue(record: Record, key: String, default: String = ""): String = {
-    if (!record.isNull(key)) {
-      val value = record.getString(key)
-      return value.trim
-    }
-    default
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_82_originData_20250221.scala

@@ -1,155 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_recsys_r_rate
-
-import com.aliyun.odps.data.Record
-import com.aliyun.odps.spark.examples.myUtils._
-import examples.utils.SimilarityUtils
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.SparkSession
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-
-/*
-   20250221 提取特征
- */
-
-object makedata_recsys_82_originData_20250221 {
-  private val labelNames = List(
-    "is_share", "share_cnt",
-    "is_return_1", "return_1_uv",
-    "is_return_n", "return_n_uv",
-    "is_return_noself", "return_1_uv_noself",
-    "is_return_n_noself", "return_n_uv_noself"
-  )
-
-  private def parseVideoRdd(videoRdd: RDD[Record]): RDD[(String, java.util.Map[String, String])] = {
-    videoRdd
-      .map(record => {
-        val vid = record.getString("vid")
-        val feature = ConvertUtils.getRecordCol(record, "feature")
-        feature.put("vid", vid)
-        (vid, feature)
-      })
-      .reduceByKey((a, b) => if (a.size() > b.size()) a else b)
-  }
-
-  private def getVidMidRdd(logRdd: RDD[java.util.Map[String, String]]): RDD[(String, String)] = {
-    logRdd
-      .map(raw => {
-        val mid = raw.getOrElse("mid", "")
-        val c9 = raw.getOrElse("c9_feature", "")
-        (mid, c9)
-      })
-      .filter(_._1.nonEmpty)
-      .reduceByKey((a, b) => a)
-      .flatMap(raw => {
-        val result = new ArrayBuffer[(String, String)]
-        for (hVid <- ConvertUtils.getVidList(raw._2)) {
-          result += ((hVid, raw._1)) // (vid, mid)
-        }
-        result
-      })
-  }
-
-  private def getMidSeqRdd(vidMidRdd: RDD[(String, String)], videoRdd: RDD[(String, java.util.Map[String, String])]): RDD[(String, List[java.util.Map[String, String]])] = {
-    vidMidRdd
-      .join(videoRdd)
-      .map(raw => {
-        (raw._2._1, raw._2._2)
-      })
-      .groupByKey()
-      .map(raw => {
-        (raw._1, raw._2.toList)
-      })
-  }
-
-  private def joinMidSeq(logRdd: RDD[java.util.Map[String, String]], midSeqRdd: RDD[(String, List[java.util.Map[String, String]])]): RDD[(java.util.Map[String, String], List[java.util.Map[String, String]])] = {
-    logRdd
-      .map(raw => {
-        val mid = raw.getOrElse("mid", "")
-        (mid, raw)
-      })
-      .leftOuterJoin(midSeqRdd)
-      .map(raw => {
-        if (raw._2._2.isDefined) {
-          (raw._2._1, raw._2._2.get)
-        } else {
-          (raw._2._1, List())
-        }
-      })
-  }
-
-  private def getFeature(rdd: RDD[(java.util.Map[String, String], List[java.util.Map[String, String]])]): RDD[String] = {
-    rdd.mapPartitions(partition => {
-      SimilarityUtils.init()
-      partition.map(raw => {
-        val record = raw._1
-        val videoSeq = raw._2
-        val logKey = DataUtils.getLogKey(record)
-        val labels = DataUtils.getLabels(labelNames, record).toString
-        val features = ConvertUtils.getFeature(record, videoSeq, 6).toString
-        val scoresMap = DataUtils.getSubJson(record, "extend_alg", "scoresMap").toString
-        logKey + "\t" + labels + "\t" + scoresMap + "\t" + features
-      })
-    })
-  }
-
-  def main(args: Array[String]): Unit = {
-    val spark = SparkSession
-      .builder()
-      .appName(this.getClass.getName)
-      .getOrCreate()
-    val sc = spark.sparkContext
-
-    // 1. 解析参数
-    val param = ParamUtils.parseArgs(args)
-    val project = param.getOrElse("project", "loghubods")
-    val table = param.getOrElse("table", "dwd_recsys_alg_sample_all_20250212")
-    val tablePart = param.getOrElse("tablePart", "64").toInt
-    val beginStr = param.getOrElse("beginStr", "2025022317")
-    val endStr = param.getOrElse("endStr", "2025022317")
-    val whatLabel = param.getOrElse("whatLabel", "is_return_n_noself")
-    val fuSampleRate = param.getOrElse("fuSampleRate", "0.1").toDouble
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/82_origin_data/")
-    val repartition = param.getOrElse("repartition", "32").toInt
-
-    // 2. 数据处理
-    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
-    for (dt_hh <- timeRange) {
-      // a. 分区
-      val dt = dt_hh.substring(0, 8)
-      val hh = dt_hh.substring(8, 10)
-      val partition = "dt=%s,hh=%s".format(dt, hh)
-      println("开始执行partition:" + partition)
-
-      // b. 加载视频信息
-      val originVideo = DataUtils.getODPSData(sc, project, "alg_recsys_feature_user_behavior_video", partition, tablePart)
-      val uniqVideo = parseVideoRdd(originVideo)
-
-      // c. 加载样本数据
-      val odpsData = DataUtils.getODPSData(sc, project, table, partition, tablePart)
-
-      // d. 样本重采样
-      val resampleData = DataUtils.resample(whatLabel, fuSampleRate, odpsData)
-        .map(record => {
-          ConvertUtils.record2Map(record)
-        })
-
-      // e. get vid mid rdd
-      val vidMidRdd = getVidMidRdd(resampleData)
-
-      // f. get mid seq rdd
-      val midSeqRdd = getMidSeqRdd(vidMidRdd, uniqVideo)
-
-      // g. 历史行为关联video
-      val seqSampleData = joinMidSeq(resampleData, midSeqRdd)
-
-      // h. 特征转换
-      val featureData = getFeature(seqSampleData)
-
-      // i. 保存数据
-      val hdfsPath = "%s/%s%s".format(savePath, dt, hh)
-      DataUtils.saveData(featureData, hdfsPath, repartition)
-    }
-  }
-}

src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_82_nor_sample_20250221.scala → src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys_r_rate/makedata_recsys_83_nor_sample_20250317.scala

@@ -11,7 +11,7 @@ import scala.collection.mutable.ArrayBuffer
 import scala.io.Source
 import scala.util.Random
 
-object makedata_recsys_82_nor_sample_20250221 {
+object makedata_recsys_83_nor_sample_20250317 {
   val hoursMap = Map(
     "hour@1" -> 1.0,
     "hour@2" -> 2.0,
@@ -42,7 +42,7 @@ object makedata_recsys_82_nor_sample_20250221 {
   def main(args: Array[String]): Unit = {
     // 1 读取参数
     val param = ParamUtils.parseArgs(args)
-    val readPath = param.getOrElse("readPath", "/dw/recommend/model/82_origin_data/")
+    val readPath = param.getOrElse("readPath", "/dw/recommend/model/83_origin_data/")
     val beginStr = param.getOrElse("beginStr", "20250221")
     val endStr = param.getOrElse("endStr", "20250221")
     val whatApps = param.getOrElse("whatApps", "0,4,5,21,3,6").split(",").toSet
@@ -52,7 +52,7 @@ object makedata_recsys_82_nor_sample_20250221 {
     val featureNameFile = param.getOrElse("featureName", "20241209_recsys_nor_name.txt")
     val featureBucketFile = param.getOrElse("featureBucket", "20241209_recsys_nor_bucket.txt")
     val repartition = param.getOrElse("repartition", "100").toInt
-    val savePath = param.getOrElse("savePath", "/dw/recommend/model/82_recsys_nor_train_data/")
+    val savePath = param.getOrElse("savePath", "/dw/recommend/model/83_recsys_nor_train_data/")
 
     val spark = SparkSession
       .builder()

src/main/scala/com/aliyun/odps/spark/examples/myUtils/ConvertUtils.java

@@ -11,95 +11,6 @@ import java.math.RoundingMode;
 import java.util.*;
 
 public class ConvertUtils {
-    public static JSONObject getFeature(Map<String, String> record, List<Map<String, String>> videoSeq, int scale) {
-        Map<String, Double> featMap = new HashMap<>();
-
-        // origin info
-        String ts = record.get("ts");
-        long currentMs = Long.parseLong(ts) * 1000;
-        String vid = record.get("vid");
-        Map<String, String> headInfo = getRecordCol(record, "v2_feature");
-        Map<String, String> rankInfo = getRecordCol(record, "v1_feature");
-        Map<String, Map<String, String>> userOriginInfo = getUserOriginInfo(record);
-        Map<String, Map<String, Map<String, String>>> videoOriginInfo = getVideoOriginInfo(record);
-
-        // parse info
-        Map<String, String> c9Map = userOriginInfo.get("alg_recsys_feature_user_share_return_stat");
-        UserShareReturnProfile userProfile = JSON.parseObject(toJson(c9Map), UserShareReturnProfile.class);
-        Map<String, Map<String, String>> historyVideoMap = list2Map(videoSeq);
-
-        Map<String, Map<String, String[]>> c7Map = FeatureTransform.parseUCFScore(userOriginInfo.get("alg_mid_feature_sharecf"));
-        Map<String, Map<String, String[]>> c8Map = FeatureTransform.parseUCFScore(userOriginInfo.get("alg_mid_feature_returncf"));
-
-        // context feature
-        FeatureTransform.getContextFeature(currentMs, featMap);
-
-        // head video feature
-        FeatureTransform.getVideoBaseFeature("h", currentMs, headInfo, featMap);
-
-        // user feature
-        FeatureTransform.getUserFeature(userOriginInfo, featMap);
-        FeatureTransform.getUserProfileFeature(userProfile, record, featMap);
-
-        // user & video feature
-        FeatureTransform.getUserTagsCrossVideoFeature("c5", rankInfo, userOriginInfo.get("alg_mid_feature_return_tags"), featMap);
-        FeatureTransform.getUserTagsCrossVideoFeature("c6", rankInfo, userOriginInfo.get("alg_mid_feature_share_tags"), featMap);
-        FeatureTransform.getUserCFFeature("c7", vid, c7Map, featMap);
-        FeatureTransform.getUserCFFeature("c8", vid, c8Map, featMap);
-
-        // rank video feature
-        FeatureTransform.getVideoBaseFeature("r", currentMs, rankInfo, featMap);
-        FeatureTransform.getVideoFeature(vid, videoOriginInfo, featMap);
-
-        // head&rank cross feature
-        FeatureTransform.getHeadRankVideoCrossFeature(headInfo, rankInfo, featMap);
-
-        // user profile & rank cross
-        FeatureTransform.getProfileVideoCrossFeature(currentMs, userProfile, rankInfo, historyVideoMap, featMap);
-
-        return filterAndTruncate(featMap, scale);
-    }
-
-    private static Map<String, Map<String, String>> getUserOriginInfo(Map<String, String> record) {
-        Map<String, Map<String, String>> map = new HashMap<>();
-        map.put("mid_global_feature_20250212", getRecordCol(record, "c1_feature"));
-        map.put("mid_u2u_friend_index_feature_20250212", getRecordCol(record, "c4_feature"));
-        map.put("alg_mid_feature_return_tags", getRecordCol(record, "c5_feature"));
-        map.put("alg_mid_feature_share_tags", getRecordCol(record, "c6_feature"));
-        map.put("alg_mid_feature_sharecf", getRecordCol(record, "c7_feature"));
-        map.put("alg_mid_feature_returncf", getRecordCol(record, "c8_feature"));
-        map.put("alg_recsys_feature_user_share_return_stat", getRecordCol(record, "c9_feature"));
-        return map;
-    }
-
-    private static Map<String, Map<String, Map<String, String>>> getVideoOriginInfo(Map<String, String> record) {
-        Map<String, Map<String, String>> map = new HashMap<>();
-        map.put("alg_vid_global_feature_20250212", getRecordCol(record, "b1_feature"));
-        map.put("alg_vid_recommend_exp_feature_20250212", getRecordCol(record, "b2_feature"));
-        map.put("alg_vid_recommend_flowpool_exp_feature_20250212", getRecordCol(record, "b3_feature"));
-        map.put("alg_vid_apptype_recommend_exp_feature_20250212", getRecordCol(record, "b4_feature"));
-        map.put("alg_vid_province_recommend_exp_feature_20250212", getRecordCol(record, "b5_feature"));
-        map.put("alg_vid_brand_recommend_exp_feature_20250212", getRecordCol(record, "b6_feature"));
-        map.put("alg_vid_hotsencetype_recommend_exp_feature_20250212", getRecordCol(record, "b7_feature"));
-        map.put("alg_merge_cate1_recommend_exp_feature_20250212", getRecordCol(record, "b8_feature"));
-        map.put("alg_merge_cate2_recommend_exp_feature_20250212", getRecordCol(record, "b9_feature"));
-        map.put("alg_channel_recommend_exp_feature_20250212", getRecordCol(record, "b10_feature"));
-        map.put("alg_festive_recommend_exp_feature_20250212", getRecordCol(record, "b11_feature"));
-        map.put("alg_vid_long_period_recommend_exp_feature_20250212", getRecordCol(record, "b12_feature"));
-        map.put("alg_video_unionid_recommend_exp_feature_20250212", getRecordCol(record, "b13_feature"));
-        map.put("scene_type_vid_cf_feature_20250212", getRecordCol(record, "d1_feature"));
-        map.put("vid_click_cf_feature_20250212", getRecordCol(record, "d2_feature"));
-        map.put("alg_recsys_feature_cf_i2i_v2", getRecordCol(record, "d3_feature"));
-        // 特征
-        map.put("mid_merge_cate1_feature_20250212", getRecordCol(record, "c2_feature"));
-        map.put("mid_merge_cate2_feature_20250212", getRecordCol(record, "c3_feature"));
-
-        Map<String, Map<String, Map<String, String>>> allMap = new HashMap<>();
-        String vid = record.get("vid");
-        allMap.put(vid, map);
-        return allMap;
-    }
-
     public static Set<String> getVidList(String data) {
         Set<String> vidList = new HashSet<>();
         if (null != data && !data.isEmpty()) {

src/main/scala/com/aliyun/odps/spark/examples/myUtils/FeatureTransform.java

@@ -9,7 +9,6 @@ public class FeatureTransform {
     private static final int seqLastN = 2;
     private static final double smoothPlus = 5.0;
     private static final List<String> c1Periods = Arrays.asList("72h", "168h");
-    private static final List<String> c4Periods = Arrays.asList("72h", "168h");
     private static final List<String> b1Periods = Arrays.asList("1h", "3h", "24h", "72h", "168h");
     private static final List<String> b2Periods = Arrays.asList("1h", "3h", "24h");
     private static final List<String> b3Periods = Arrays.asList("24h", "168h");
@@ -22,7 +21,6 @@ public class FeatureTransform {
     private static final List<String> b10Periods = Arrays.asList("1h", "12h");
     private static final List<String> b11Periods = Arrays.asList("12h", "168h");
     private static final List<String> b13Periods = Arrays.asList("24h", "168h");
-    private static final List<String> dayPeriods = Arrays.asList("7d", "14d", "30d", "60d");
     private static final List<String> videoCateAttrs = Arrays.asList(FeatureUtils.cate1Attr, FeatureUtils.cate2Attr, FeatureUtils.festive1Attr);
     private static final List<String> videoSimAttrs = Arrays.asList("title", "cate2", "cate2_list", "keywords");
     private static final List<String> hVideoSimAttrs = Arrays.asList("title");
@@ -40,10 +38,6 @@ public class FeatureTransform {
 
     public static void getUserFeature(Map<String, Map<String, String>> userOriginInfo, Map<String, Double> featMap) {
         oneTypeStatFeature("c1", "return_1_uv", c1Periods, userOriginInfo.get("mid_global_feature_20250212"), featMap);
-        Map<String, String> c4Map = userOriginInfo.get("mid_u2u_friend_index_feature_20250212");
-//        for (String calType : Arrays.asList("avg_", "max_", "min_")) {
-//            getRateStatFeature("c4", calType, c4Periods, c4Map, featMap);
-//        }
     }
 
     public static void getUserProfileFeature(UserShareReturnProfile profile, Map<String, String> userInfo, Map<String, Double> featMap) {
@@ -135,10 +129,6 @@ public class FeatureTransform {
         headVideoCFD1Feature("d1", videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("scene_type_vid_cf_feature_20250212"), featMap);
         headVideoCFD2Feature("d2", videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("vid_click_cf_feature_20250212"), featMap);
         headVideoCFD3Feature("d3", videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("alg_recsys_feature_cf_i2i_v2"), featMap);
-
-        // 特殊mid * cate
-        //oneTypeStatFeature("c2", "return_n_uv", c1Periods, videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("mid_merge_cate1_feature_20250212"), featMap);
-        //oneTypeStatFeature("c3", "return_n_uv", c1Periods, videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("mid_merge_cate2_feature_20250212"), featMap);
     }
 
     public static void getVideoBaseFeature(String prefix, long currentMs, Map<String, String> videoInfo, Map<String, Double> featMap) {
@@ -171,7 +161,7 @@ public class FeatureTransform {
     }
 
     public static void getHeadRankVideoCrossFeature(Map<String, String> headInfo, Map<String, String> rankInfo, Map<String, Double> featMap) {
-        getTwoVideoCrossFeature("hr_sim", FeatureTransform.videoSimAttrs, headInfo, rankInfo, featMap);
+        getTwoVideoCrossFeature("hr_sim", videoSimAttrs, headInfo, rankInfo, featMap);
     }
 
     public static void getProfileVideoCrossFeature(long currentMs, UserShareReturnProfile profile, Map<String, String> rankVideo, Map<String, Map<String, String>> hVideoMap, Map<String, Double> featMap) {
@@ -215,7 +205,6 @@ public class FeatureTransform {
                         }
                         if (null != hVideoMap && hVideoMap.containsKey(vid)) {
                             Map<String, String> hVideo = hVideoMap.get(vid);
-                            getVideoCateFeature(baseKey, hVideo, featMap);
                             getTwoVideoCrossFeature(baseKey, hVideoSimAttrs, hVideo, rankVideo, featMap);
                         }
                     }
@@ -342,29 +331,6 @@ public class FeatureTransform {
         return allScoresMap;
     }
 
-    private static void getRateStatFeature(String prefix, String calType, List<String> periods, Map<String, String> infoMap, Map<String, Double> featMap) {
-        if (null == infoMap || infoMap.isEmpty()) {
-            return;
-        }
-        for (String period : periods) {
-            double str_one = getOneInfo(calType + "str_one_" + period, infoMap);
-            double ros_one = getOneInfo(calType + "ros_one_" + period, infoMap);
-            double str = getOneInfo(calType + "str_" + period, infoMap);
-            double ros = getOneInfo(calType + "ros_" + period, infoMap);
-            double str_plus = getOneInfo(calType + "str_plus_" + period, infoMap);
-            double ros_minus = getOneInfo(calType + "ros_minus_" + period, infoMap);
-            double rovn = getOneInfo(calType + "rovn_" + period, infoMap);
-
-            featMap.put(prefix + "_" + period + "_" + calType + "str_one", str_one);
-            featMap.put(prefix + "_" + period + "_" + calType + "ros_one", ros_one);
-            featMap.put(prefix + "_" + period + "_" + calType + "str", str);
-            featMap.put(prefix + "_" + period + "_" + calType + "ros", ros);
-            featMap.put(prefix + "_" + period + "_" + calType + "str_plus", str_plus);
-            featMap.put(prefix + "_" + period + "_" + calType + "ros_minus", ros_minus);
-            featMap.put(prefix + "_" + period + "_" + calType + "rovn", rovn);
-        }
-    }
-
     private static void oneTypeStatFeature(String prefix, String uvPrefix, List<String> periods, Map<String, String> infoMap, Map<String, Double> featMap) {
         if (null == infoMap || infoMap.isEmpty()) {
             return;

src/main/scala/com/aliyun/odps/spark/examples/myUtils/FeatureTransformV2.java

@@ -10,7 +10,6 @@ public class FeatureTransformV2 {
     private static final double smoothPlus = 5.0;
     private static final double log1Scale = 10.0;
     private static final List<String> c1Periods = Arrays.asList("72h", "168h");
-    private static final List<String> c4Periods = Arrays.asList("72h", "168h");
     private static final List<String> b0Periods = Arrays.asList("1h", "3h", "6h", "12h");
     private static final List<String> b1Periods = Arrays.asList("1h", "3h", "24h", "72h", "168h");
     private static final List<String> b2Periods = Arrays.asList("1h", "3h", "24h");
@@ -24,7 +23,6 @@ public class FeatureTransformV2 {
     private static final List<String> b10Periods = Arrays.asList("1h", "12h");
     private static final List<String> b11Periods = Arrays.asList("12h", "168h");
     private static final List<String> b13Periods = Arrays.asList("24h", "168h");
-    private static final List<String> dayPeriods = Arrays.asList("7d", "14d", "30d", "60d");
     private static final List<String> videoCateAttrs = Arrays.asList(FeatureUtils.cate1Attr, FeatureUtils.cate2Attr, FeatureUtils.festive1Attr, FeatureUtils.channelAttr);
     private static final List<String> videoSimAttrs = Arrays.asList("title", "cate2", "cate2_list", "keywords");
     private static final List<String> hVideoSimAttrs = Arrays.asList("title");
@@ -53,10 +51,6 @@ public class FeatureTransformV2 {
 
     public static void getUserFeature(Map<String, Map<String, String>> userOriginInfo, Map<String, Double> featMap) {
         oneTypeStatFeature("c1", "return_1_uv", c1Periods, userOriginInfo.get("mid_global_feature_20250212"), featMap);
-        Map<String, String> c4Map = userOriginInfo.get("mid_u2u_friend_index_feature_20250212");
-//        for (String calType : Arrays.asList("avg_", "max_", "min_")) {
-//            getRateStatFeature("c4", calType, c4Periods, c4Map, featMap);
-//        }
     }
 
     public static void getUserProfileFeature(UserShareReturnProfile profile, Map<String, String> userInfo, Map<String, Double> featMap) {
@@ -160,10 +154,6 @@ public class FeatureTransformV2 {
         headVideoCFD1Feature("d1", videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("scene_type_vid_cf_feature_20250212"), featMap);
         headVideoCFD2Feature("d2", videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("vid_click_cf_feature_20250212"), featMap);
         headVideoCFD3Feature("d3", videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("alg_recsys_feature_cf_i2i_v2"), featMap);
-
-        // 特殊mid * cate
-        //oneTypeStatFeature("c2", "return_n_uv", c1Periods, videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("mid_merge_cate1_feature_20250212"), featMap);
-        //oneTypeStatFeature("c3", "return_n_uv", c1Periods, videoOriginInfo.getOrDefault(vid, new HashMap<>()).get("mid_merge_cate2_feature_20250212"), featMap);
     }
 
     public static void getVideoBaseFeature(String prefix, long currentMs, Map<String, String> videoInfo, Map<String, Double> featMap) {
@@ -251,7 +241,6 @@ public class FeatureTransformV2 {
                         }
                         if (null != hVideoMap && hVideoMap.containsKey(vid)) {
                             Map<String, String> hVideo = hVideoMap.get(vid);
-                            //getVideoCateFeature(baseKey, hVideo, featMap);
                             getTwoVideoCrossFeature(baseKey, hVideoSimAttrs, hVideo, rankVideo, featMap);
                         }
                     }
@@ -389,29 +378,6 @@ public class FeatureTransformV2 {
         return allScoresMap;
     }
 
-    private static void getRateStatFeature(String prefix, String calType, List<String> periods, Map<String, String> infoMap, Map<String, Double> featMap) {
-        if (null == infoMap || infoMap.isEmpty()) {
-            return;
-        }
-        for (String period : periods) {
-            double str_one = getOneInfo(calType + "str_one_" + period, infoMap);
-            double ros_one = getOneInfo(calType + "ros_one_" + period, infoMap);
-            double str = getOneInfo(calType + "str_" + period, infoMap);
-            double ros = getOneInfo(calType + "ros_" + period, infoMap);
-            double str_plus = getOneInfo(calType + "str_plus_" + period, infoMap);
-            double ros_minus = getOneInfo(calType + "ros_minus_" + period, infoMap);
-            double rovn = getOneInfo(calType + "rovn_" + period, infoMap);
-
-            featMap.put(prefix + "_" + period + "@" + calType + "str_one", str_one);
-            featMap.put(prefix + "_" + period + "@" + calType + "ros_one", ros_one);
-            featMap.put(prefix + "_" + period + "@" + calType + "str", str);
-            featMap.put(prefix + "_" + period + "@" + calType + "ros", ros);
-            featMap.put(prefix + "_" + period + "@" + calType + "str_plus", str_plus);
-            featMap.put(prefix + "_" + period + "@" + calType + "ros_minus", ros_minus);
-            featMap.put(prefix + "_" + period + "@" + calType + "rovn", rovn);
-        }
-    }
-
     private static void oneTypeStatFeature(String prefix, String uvPrefix, List<String> periods, Map<String, String> infoMap, Map<String, Double> featMap) {
         if (null == infoMap || infoMap.isEmpty()) {
             return;