Browse Source

read hdfs

丁云鹏 5 months ago
parent
commit
d0a7913663

+ 1 - 1
recommend-model-produce/src/main/python/tools/utils/static_ps/reader_helper_hdfs.py

@@ -83,7 +83,7 @@ def get_file_list(data_path, config):
     # 如果配置中指定了分割文件列表
     if config.get("runner.split_file_list"):
         logger.info("Split file list for worker {}".format(dist.get_rank()))
-        file_list = get_file_shard(file_list)
+        file_list = fleet.util.get_file_shard(file_list)
     logger.info("File list: {}".format(file_list))
 
     base_url = f'{configs["fs.default.name"]}'