dssm train

recommend-model-produce/src/main/python/models/dssm/milvus_data_process.py

@@ -1,9 +1,9 @@
 import os
 import sys
 
-__dir__ = os.path.dirname(os.path.abspath(__file__))
+__dir__ = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 #sys.path.append(__dir__)
-sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
+sys.path.append(os.path.join(__dir__,"tools"))
 
 import json
 from concurrent.futures import ThreadPoolExecutor
@@ -58,8 +58,8 @@ def main():
 
     max_workers = 2
     split_file_list = [
-        ['/dyp/vec/vec-part-00017.gz'],
-        ['/dyp/vec/vec-part-00018.gz']
+        ['/dyp/vec/part-00017.gz'],
+        ['/dyp/vec/part-00018.gz']
     ]
     future_list = []
     with ThreadPoolExecutor(max_workers=max_workers) as executor:

recommend-model-produce/src/main/python/tools/inferv2.py

@@ -71,7 +71,7 @@ def thread_task(name, file_list, model_file, params_file):
         output_file = f"/app/vec-{file_name}.json"
         write_results(results, output_file)
         compress.compress_file_tar(output_file, f"{output_file}.tar.gz")
-        hdfs_client.upload(f"{output_file}.tar.gz", f"/dyp/vec/{file_name}.gz", multi_processes=1, overwrite=True):
+        hdfs_client.upload(f"{output_file}.tar.gz", f"/dyp/vec/{file_name}.gz", multi_processes=1, overwrite=True)
         results=[]
         print(f"Thread {name}: ending file:{file_path} {i}/{count}")