|
@@ -20,6 +20,7 @@ object stat_origin_feature {
|
|
|
val featureIndex = param.getOrElse("featureIndex", "2").toInt
|
|
val featureIndex = param.getOrElse("featureIndex", "2").toInt
|
|
|
val repartition = param.getOrElse("repartition", "10").toInt
|
|
val repartition = param.getOrElse("repartition", "10").toInt
|
|
|
val savePath = param.getOrElse("savePath", "/dw/recommend/model/832_recsys_analysis_data")
|
|
val savePath = param.getOrElse("savePath", "/dw/recommend/model/832_recsys_analysis_data")
|
|
|
|
|
+ val subPath = param.getOrElse("subPath", year)
|
|
|
|
|
|
|
|
val spark = SparkSession
|
|
val spark = SparkSession
|
|
|
.builder()
|
|
.builder()
|
|
@@ -53,7 +54,7 @@ object stat_origin_feature {
|
|
|
})
|
|
})
|
|
|
|
|
|
|
|
// 3. 保存数据到hdfs
|
|
// 3. 保存数据到hdfs
|
|
|
- val hdfsPath = savePath + "/" + year
|
|
|
|
|
|
|
+ val hdfsPath = savePath + "/" + subPath
|
|
|
if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
|
|
if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
|
|
|
println("删除路径并开始数据写入:" + hdfsPath)
|
|
println("删除路径并开始数据写入:" + hdfsPath)
|
|
|
MyHdfsUtils.delete_hdfs_path(hdfsPath)
|
|
MyHdfsUtils.delete_hdfs_path(hdfsPath)
|