|
@@ -127,14 +127,15 @@ object pred_recsys_61_xgb_nor_hdfsfile_20241209 {
|
|
|
|
|
|
def getClipData(spark: SparkSession, df: DataFrame): DataFrame = {
|
|
|
import spark.implicits._
|
|
|
- df.select("label", "prediction").rdd
|
|
|
+ df.select("label", "prediction", "logKey").rdd
|
|
|
.map(row => {
|
|
|
val label = row.getAs[Double]("label")
|
|
|
val prediction = row.getAs[Double]("prediction")
|
|
|
+ val logKey = row.getAs[String]("logKey")
|
|
|
if (prediction < 1E-8) {
|
|
|
- (label, prediction, 0d)
|
|
|
+ (label, prediction, 0d, logKey)
|
|
|
} else {
|
|
|
- (label, prediction, prediction)
|
|
|
+ (label, prediction, prediction, logKey)
|
|
|
}
|
|
|
}
|
|
|
).toDF("label", "prediction", "clipPrediction")
|