oss delete

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/VideoCleanExecutor.java

@@ -0,0 +1,38 @@
+package com.tzld.piaoquan.recommend.feature.produce;
+
+import com.tzld.piaoquan.recommend.feature.produce.service.OSSService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author dyp
+ */
+@Slf4j
+public class VideoCleanExecutor {
+    public static void main(String[] args) {
+        SparkConf sparkConf = new SparkConf()
+                //.setMaster("local")
+                .setAppName("VideoCleanODPSToHDFS");
+        JavaSparkContext jsc = new JavaSparkContext(sparkConf);
+
+        JavaRDD<String> multiFileRDD = jsc.textFile("/dyp/oss/video_clean/part-00000");
+
+        OSSService ossService = new OSSService();
+        multiFileRDD.repartition(100).foreachPartition(s -> {
+            List<String> objectNames = new ArrayList<>();
+            if (s.hasNext()) {
+                String[] data = StringUtils.split(s.next(), "\t");
+                objectNames.add(data[2]);
+                //objectNames.add(data[3]);
+                ossService.transToDeepColdArchive("art-pubbucket", objectNames);
+            }
+        });
+    }
+
+}

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/VideoCleanODPSToHDFS.java

@@ -49,7 +49,7 @@ public class VideoCleanODPSToHDFS {
             return sb.toString();
         });
 
-        data.coalesce(1000).saveAsTextFile(path);
+        data.coalesce(90).saveAsTextFile(path);
     }
 
 }