dssm train

recommend-model-produce/src/main/python/models/dssm/milvus_data_process.py

@@ -28,17 +28,17 @@ def process_file(file_path):
             vid, vec = sample_values
             result.append({
                 "vid":vid,
-                "vec":vec
+                "vec":json.loads(vec)
             })
         else :
+            # 过滤PaxHeader
             print(f"{sample_values}")
     return result
 
-def write_results(results, output_file):
+def write_results(data, output_file):
     """将结果写入文件"""
     with open(output_file, 'w') as json_file:
-        for s in results:
-            json_file.write(s + "\n")
+            json_file.write(data)
 
 def thread_task(name, file_list):
     """线程任务"""

recommend-model-produce/src/main/python/tools/utils/compress.py

@@ -22,6 +22,7 @@ def compress_file_tar(file_path, output_filename):
     # 创建一个 tarfile 对象，使用 'w:gz' 模式表示写入 gzip 压缩的 tar 包
     with tarfile.open(output_filename, "w:gz") as tar:
         # 将文件添加到 tar 包中，arcname 指定在 tar 包中的相对路径
+        tar.format = tarfile.GNU_FORMAT
         tar.add(file_path, arcname=file_path)
 
 def compress_tar(folder_path, output_filename):