|
@@ -88,8 +88,9 @@ def get_file_list(data_path, config, file_extensions=['.gz']):
|
|
|
if file_extensions and not any(file.endswith(ext) for ext in file_extensions):
|
|
|
continue
|
|
|
all_files.append(file)
|
|
|
-
|
|
|
- #print(sub_dirs,all_files)
|
|
|
+
|
|
|
+ all_files = all_files + file_list
|
|
|
+ print(sub_dirs,all_files)
|
|
|
# 如果配置中指定了分割文件列表
|
|
|
if config.get("runner.split_file_list"):
|
|
|
logger.info("Split file list for worker {}".format(dist.get_rank()))
|