|
@@ -23,11 +23,14 @@ public class VideoCleanExecutor {
|
|
|
|
|
|
JavaRDD<String> multiFileRDD = jsc.textFile("/dyp/oss/video_clean/part-00000");
|
|
|
|
|
|
+ log.info("count {}", multiFileRDD.count())
|
|
|
+
|
|
|
multiFileRDD.repartition(100).foreachPartition(s -> {
|
|
|
List<String> objectNames = new ArrayList<>();
|
|
|
if (s.hasNext()) {
|
|
|
String[] data = StringUtils.split(s.next(), "\t");
|
|
|
objectNames.add(data[2]);
|
|
|
+ log.info("data[2] {}", data[2]);
|
|
|
//objectNames.add(data[3]);
|
|
|
new OSSService().transToDeepColdArchive("art-pubbucket", objectNames);
|
|
|
}
|