|
@@ -15,12 +15,15 @@ import java.util.Map;
|
|
|
@Slf4j
|
|
|
public class VideoCleanODPSToHDFS {
|
|
|
public static void main(String[] args) {
|
|
|
+
|
|
|
SparkConf sparkConf = new SparkConf()
|
|
|
+ .setMaster("local")
|
|
|
.setAppName("VideoCleanODPSToHDFS");
|
|
|
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
|
|
|
|
|
|
ODPSService odpsService = new ODPSService();
|
|
|
- JavaRDD<Map<String, String>> record = odpsService.read(jsc, "loghubods", "not_active_videos", null, 90);
|
|
|
+ JavaRDD<Map<String, String>> record = odpsService.read(jsc, "loghubods", "not_active_videos_di", "dt=20240617",
|
|
|
+ 90);
|
|
|
|
|
|
JavaRDD<String> data = record.map(r -> {
|
|
|
StringBuilder sb = new StringBuilder();
|