Browse Source

add partition

often 5 months ago
parent
commit
a1c9dc402b

+ 2 - 1
src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys/video_dssm_sampler.scala

@@ -309,7 +309,8 @@ object video_dssm_sampler {
       //  .join(broadcast(categoryData), col("vid_right") === categoryData("vid"), "left")
 
       // 保存结果到HDFS
-      result.write
+      val resultWithDt = result.withColumn("dt", lit(s"$dt"))
+      resultWithDt.write
         .mode("overwrite")
         .partitionBy("dt")
         .parquet(s"$outputPath/dt=$dt")