Browse Source

add partition

often 5 months ago
parent
commit
b9363e2be9

+ 4 - 2
src/main/scala/com/aliyun/odps/spark/examples/makedata_recsys/video_dssm_sampler.scala

@@ -90,7 +90,9 @@ object video_dssm_sampler {
     (record.getString("category1"), record.getString("feature"))  // 或 category2 取决于表
   }
 
-
+  def funcL2CatStatFeatures(record: Record, schema: TableSchema): (String, String) = {
+    (record.getString("category2"), record.getString("feature"))  // 或 category2 取决于表
+  }
 
   def generateNegativeSamples(spark: SparkSession, dt: String, outputPath: String): Unit = {
     val stats = ProcessingStats()
@@ -209,7 +211,7 @@ object video_dssm_sampler {
           project = "loghubods",
           table = "t_vid_l2_cat_stat_feature",
           partition = s"dt='$dt'",
-          transfer = funcCatStatFeatures,
+          transfer = funcL2CatStatFeatures,
           numPartition = CONFIG("shuffle.partitions").toInt
         )
         val schema = StructType(Array(