|
@@ -24,16 +24,15 @@ public class VideoCleanExecutor {
|
|
|
JavaRDD<String> multiFileRDD = jsc.textFile("/dyp/oss/video_clean/part-00000");
|
|
|
|
|
|
log.info("count {}", multiFileRDD.count());
|
|
|
-
|
|
|
+ OSSService ossService = new OSSService();
|
|
|
multiFileRDD.repartition(100).foreachPartition(s -> {
|
|
|
List<String> objectNames = new ArrayList<>();
|
|
|
if (s.hasNext()) {
|
|
|
String[] data = StringUtils.split(s.next(), "\t");
|
|
|
objectNames.add(data[2]);
|
|
|
log.info("data[2] {}", data[2]);
|
|
|
- //objectNames.add(data[3]);
|
|
|
}
|
|
|
- new OSSService().transToDeepColdArchive("art-pubbucket", objectNames);
|
|
|
+ ossService.transToDeepColdArchive("art-pubbucket", objectNames);
|
|
|
});
|
|
|
}
|
|
|
|