|
@@ -69,7 +69,7 @@ def thread_task(name, file_list, model_file, params_file):
|
|
|
results.extend(process_file(file_path, model_file, params_file))
|
|
results.extend(process_file(file_path, model_file, params_file))
|
|
|
output_file = f"/app/data_{os.path.basename(file_path)}.json"
|
|
output_file = f"/app/data_{os.path.basename(file_path)}.json"
|
|
|
write_results(results, output_file)
|
|
write_results(results, output_file)
|
|
|
- client.upload(output_file, f"/dyc/vec/data_{os.path.basename(file_path)}.json", multi_processes=1, overwrite=True):
|
|
|
|
|
|
|
+ hdfs_client.upload(output_file, f"/dyp/vec/data_{os.path.basename(file_path)}.json", multi_processes=1, overwrite=True):
|
|
|
results=[]
|
|
results=[]
|
|
|
print(f"Thread {name}: ending file:{file_path} {i}/{count}")
|
|
print(f"Thread {name}: ending file:{file_path} {i}/{count}")
|
|
|
|
|
|
丁云鹏