|
@@ -4,8 +4,8 @@ import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUt
|
|
|
import examples.extractor.ExtractorUtils
|
|
|
import org.apache.hadoop.io.compress.GzipCodec
|
|
|
import org.apache.spark.sql.SparkSession
|
|
|
+import com.alibaba.fastjson.{JSON, JSONObject}
|
|
|
|
|
|
-import com.alibaba.fastjson.JSON
|
|
|
import scala.collection.JavaConversions._
|
|
|
import scala.collection.mutable.ArrayBuffer
|
|
|
import scala.io.Source
|
|
@@ -76,32 +76,37 @@ object makedata_16_bucketData_20240609 {
|
|
|
val features = rList(2).split(",").map(_.toDouble)
|
|
|
val allFeature: JSONObject = if (rList(3).equals("\\\\N")) new JSONObject() else
|
|
|
JSON.parseObject(rList(3))
|
|
|
- (logKey, labelKey, features, allFeatureKey)
|
|
|
+ (logKey, labelKey, features, allFeature)
|
|
|
})
|
|
|
.filter{
|
|
|
- case (logKey, labelKey, features) =>
|
|
|
+ case (logKey, labelKey, features, allFeature) =>
|
|
|
val logKeyList = logKey.split(",")
|
|
|
val apptype = logKeyList(0)
|
|
|
val pagesource = logKeyList(1)
|
|
|
Set("0", "4", "5", "21", "3", "6").contains(apptype) && pagesource.endsWith("recommend")
|
|
|
}
|
|
|
.map{
|
|
|
- case (logKey, labelKey, features) =>
|
|
|
+ case (logKey, labelKey, features, allFeature) =>
|
|
|
val label = JSON.parseObject(labelKey).getOrDefault("is_return", "0").toString
|
|
|
- (label, features)
|
|
|
+ (label, features, allFeature)
|
|
|
}
|
|
|
.mapPartitions(row => {
|
|
|
val result = new ArrayBuffer[String]()
|
|
|
val contentList = contentList_br.value
|
|
|
val bucketsMap = bucketsMap_br.value
|
|
|
row.foreach{
|
|
|
- case (label, features) =>
|
|
|
+ case (label, features, allFeature) =>
|
|
|
val featuresBucket = contentList.indices.map(i =>{
|
|
|
val featureName = contentList(i)
|
|
|
val score = features(i)
|
|
|
|
|
|
if (featureName.startsWith("c")) {
|
|
|
- allFeature.getOrDefault(featureName, "0").toString
|
|
|
+ val scoreNew = allFeature.getOrDefault(featureName, "").toString
|
|
|
+ if (scoreNew.equals("")) {
|
|
|
+ ""
|
|
|
+ } else {
|
|
|
+ featureName + ":" + scoreNew.toString
|
|
|
+ }
|
|
|
} else {
|
|
|
if (score > 1E-8) {
|
|
|
val (bucketNum, buckets) = bucketsMap(featureName)
|