|
@@ -119,6 +119,7 @@ object video_dssm_sampler {
|
|
|
transfer = funcPositive,
|
|
|
numPartition = CONFIG("shuffle.partitions").toInt
|
|
|
).persist(StorageLevel.MEMORY_AND_DISK)
|
|
|
+ logger.info(s"11111111111111 $dt")
|
|
|
val positivePairs = spark.createDataFrame(rdd, schema).persist(StorageLevel.MEMORY_AND_DISK)
|
|
|
stats.positiveSamplesCount = positivePairs.count()
|
|
|
logger.info(s"start read vid list for date $dt")
|