|
@@ -80,7 +80,10 @@ def get_infer_reader(input_var, config):
|
|
|
|
|
|
def get_file_list(data_path, config):
|
|
|
logger.info("data_path: {}".format(data_path))
|
|
|
- dirs,file_list = hdfs_client.ls_dir(data_path)
|
|
|
+ try:
|
|
|
+ dirs,file_list = hdfs_client.ls_dir(data_path)
|
|
|
+ except Exception as e:
|
|
|
+ print(f"Error occurred: {e}")
|
|
|
# 如果配置中指定了分割文件列表
|
|
|
if config.get("runner.split_file_list"):
|
|
|
logger.info("Split file list for worker {}".format(dist.get_rank()))
|