dssm train

recommend-model-produce/src/main/java/com/tzld/piaoquan/recommend/model/produce/i2i/I2IDSSMPredict.java

@@ -33,14 +33,6 @@ public class I2IDSSMPredict {
         String file = argMap.get("path");
         int repartition = NumberUtils.toInt(argMap.get("repartition"), 64);
 
-        // 将处理后的数据写入新的文件，使用Gzip压缩
-        String outputPath = "hdfs:/dyp/vec2";
-        try {
-            hdfsService.deleteOnExit(outputPath);
-        } catch (Exception e) {
-            log.error("deleteOnExit error outputPath {}", outputPath, e);
-        }
-
         // 加载模型
         SparkSession spark = SparkSession.builder()
                 .appName("I2IDSSMInfer")
@@ -124,7 +116,13 @@ public class I2IDSSMPredict {
                 }
             };
         });
-
+        // 将处理后的数据写入新的文件，使用Gzip压缩
+        String outputPath = "hdfs:/dyp/vec2";
+        try {
+            hdfsService.deleteIfExist(outputPath);
+        } catch (Exception e) {
+            log.error("deleteOnExit error outputPath {}", outputPath, e);
+        }
         processedRdd.coalesce(repartition).saveAsTextFile(outputPath, GzipCodec.class);
     }
 

recommend-model-produce/src/main/java/com/tzld/piaoquan/recommend/model/produce/service/HDFSService.java

@@ -31,8 +31,12 @@ public class HDFSService implements Serializable {
         return true;
     }
 
-    public boolean deleteOnExit(String path) throws IOException {
-        return fSystem.deleteOnExit(new Path(path));
+    public boolean deleteIfExist(String path) throws IOException {
+        Path p = new Path(path);
+        if (fSystem.exists(p)) {
+            return fSystem.delete(p, true);
+        }
+        return false;
     }
 
 }