dssm train

recommend-model-produce/src/main/java/com/tzld/piaoquan/recommend/model/produce/i2i/I2IDSSMPredict.java

@@ -14,7 +14,6 @@ import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.SparkSession;
 
-import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
 
@@ -33,6 +32,14 @@ public class I2IDSSMPredict {
         String file = argMap.get("path");
         int repartition = NumberUtils.toInt(argMap.get("repartition"), 64);
 
+        // 将处理后的数据写入新的文件，使用Gzip压缩
+        String outputPath = "hdfs:/dyp/vec2";
+        try {
+            hdfsService.deleteOnExit(outputPath);
+        } catch (Exception e) {
+            log.error("deleteOnExit error outputPath {}", outputPath, e);
+        }
+
         // 加载模型
         SparkSession spark = SparkSession.builder()
                 .appName("I2IDSSMInfer")
@@ -116,13 +123,7 @@ public class I2IDSSMPredict {
                 }
             };
         });
-        // 将处理后的数据写入新的文件，使用Gzip压缩
-        String outputPath = "hdfs:/dyp/vec2";
-        try {
-            hdfsService.deleteOnExit(outputPath);
-        } catch (Exception e) {
-            log.error("deleteOnExit error outputPath {}", outputPath, e);
-        }
+
         processedRdd.coalesce(repartition).saveAsTextFile(outputPath, GzipCodec.class);
     }
 

recommend-model-produce/src/main/java/com/tzld/piaoquan/recommend/model/produce/service/HDFSService.java

@@ -35,8 +35,4 @@ public class HDFSService implements Serializable {
         return fSystem.deleteOnExit(new Path(path));
     }
 
-    public void download(String path) {
-        // fSystem
-    }
-
 }