dssm train

recommend-model-produce/src/main/python/tools/inferv2.py

@@ -40,9 +40,6 @@ def process_file(file_path, model_file, params_file):
         left_features = [float(x) for x in left_features_str.split(',')]
         input_data[vid] = left_features
 
-
-    i=0
-    count=len(input_data)
     result = []
     for k, v in input_data.items():
         v2 = np.array([v], dtype=np.float32)
@@ -52,10 +49,6 @@ def process_file(file_path, model_file, params_file):
         output_handle = predictor.get_output_handle(predictor.get_output_names()[0])
         output_data = output_handle.copy_to_cpu()
         result.append(k + "\t" + str(output_data.tolist()[0]))
-        i=i+1
-        if i % 1000 == 0:
-            print(f"Thread {name}: write batch {i}/{count}")
-
     return result
 
 def write_results(results, output_file):
@@ -68,10 +61,16 @@ def thread_task(name, file_list, model_file, params_file):
     """线程任务"""
     print(f"Thread {name}: starting file_list:{file_list}")
     results = []
+    i=0
     for file_path in file_list:
+        i=i+1
+        count=len(file_list)
+        print(f"Thread {name}: starting file:{file_path} {i}/{count}")
         results.extend(process_file(file_path, model_file, params_file))
         output_file = f"/app/data_{os.path.basename(file_path)}.json"
         write_results(results, output_file)
+        results=[]
+        print(f"Thread {name}: ending file:{file_path} {i}/{count}")
     
     print(f"Thread {name}: finishing")