all

src/main/scala/com/aliyun/odps/spark/examples/makedata/makedata_07_strData.scala

@@ -30,6 +30,7 @@ object makedata_07_strData {
     val readPath = param.getOrElse("readPath", "/dw/recommend/model/00_sample_data/")
     val savePath = param.getOrElse("savePath", "/dw/recommend/model/01_str_data/")
     val featureVersion =  param.getOrElse("featureVersion", "v2")
+    val ifRepart = param.getOrElse("ifRepart", "0").toInt
 
 
 
@@ -180,7 +181,12 @@ object makedata_07_strData {
       if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")){
         println("删除路径并开始数据写入:" + hdfsPath)
         MyHdfsUtils.delete_hdfs_path(hdfsPath)
-        data.repartition(100).saveAsTextFile(hdfsPath, classOf[GzipCodec])
+        if (ifRepart == 0){
+          data.saveAsTextFile(hdfsPath, classOf[GzipCodec])
+        }else{
+          data.repartition(ifRepart).saveAsTextFile(hdfsPath, classOf[GzipCodec])
+        }
+
       }else{
         println("路径不合法，无法写入:" + hdfsPath)
       }