dssm train

recommend-model-produce/src/main/java/com/tzld/piaoquan/recommend/model/produce/i2i/I2IDSSMPredict.java

@@ -15,7 +15,7 @@ import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.SparkSession;
 
-import java.io.File;
+import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
 
@@ -27,7 +27,7 @@ public class I2IDSSMPredict {
 
     private static HDFSService hdfsService = new HDFSService();
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws IOException {
         CMDService cmd = new CMDService();
         Map<String, String> argMap = cmd.parse(args);
         String file = argMap.get("path");
@@ -38,6 +38,7 @@ public class I2IDSSMPredict {
 
         String gzPath2 = "/root/recommend-model/model.tar.gz";
         ossService2.download(bucketName2, gzPath2, objectName2);
+        hdfsService.copyFromLocalFile(gzPath2, "/dyp/dssm");
         // 加载模型
         SparkSession spark = SparkSession.builder()
                 .appName("I2IDSSMInfer")
@@ -49,14 +50,15 @@ public class I2IDSSMPredict {
         // 定义处理数据的函数
         JavaRDD<String> processedRdd = rdd.mapPartitions(lines -> {
             System.loadLibrary("paddle_inference");
-            String bucketName = "art-recommend";
-            String objectName = "dyp/dssm.tar.gz";
-            OSSService ossService = new OSSService();
-
-            String gzPath = "/root/recommend-model/model.tar.gz";
-            ossService.download(bucketName, gzPath, objectName);
+//            String bucketName = "art-recommend";
+//            String objectName = "dyp/dssm.tar.gz";
+//            OSSService ossService = new OSSService();
+//
+//            String gzPath = "/root/recommend-model/model.tar.gz";
+//            ossService.download(bucketName, gzPath, objectName);
             String modelDir = "/root/recommend-model";
-            CompressUtil.decompressGzFile(gzPath, modelDir);
+            hdfsService.copyFromLocalFile(gzPath2, "/dyp/dssm");
+            CompressUtil.decompressGzFile(gzPath2, modelDir);
 
             String modelFile = modelDir + "/dssm.pdmodel";
             String paramFile = modelDir + "/dssm.pdiparams";

recommend-model-produce/src/main/java/com/tzld/piaoquan/recommend/model/produce/service/HDFSService.java

@@ -39,4 +39,11 @@ public class HDFSService implements Serializable {
         return false;
     }
 
+    public boolean copyFromLocalFile(String local, String hdfs) throws IOException {
+        Path localP = new Path(local);
+        Path hdfsP = new Path(hdfs);
+        fSystem.copyFromLocalFile(localP, hdfsP);
+        return false;
+    }
+
 }