|
@@ -16,12 +16,14 @@ import java.util.List;
|
|
@Slf4j
|
|
@Slf4j
|
|
public class VideoCleanExecutor {
|
|
public class VideoCleanExecutor {
|
|
public static void main(String[] args) {
|
|
public static void main(String[] args) {
|
|
|
|
+ String file = args[0];
|
|
|
|
+ log.info("hdfs file {}", file);
|
|
SparkConf sparkConf = new SparkConf()
|
|
SparkConf sparkConf = new SparkConf()
|
|
//.setMaster("local")
|
|
//.setMaster("local")
|
|
.setAppName("VideoCleanExecutor");
|
|
.setAppName("VideoCleanExecutor");
|
|
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
|
|
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
|
|
|
|
|
|
- JavaRDD<String> multiFileRDD = jsc.textFile("/dyp/oss/video_clean/part-00001");
|
|
|
|
|
|
+ JavaRDD<String> multiFileRDD = jsc.textFile(file);
|
|
|
|
|
|
log.info("count {}", multiFileRDD.count());
|
|
log.info("count {}", multiFileRDD.count());
|
|
OSSService ossService = new OSSService();
|
|
OSSService ossService = new OSSService();
|