Browse Source

read hdfs

丁云鹏 5 months ago
parent
commit
63cfc4d2c7

+ 1 - 2
recommend-model-produce/src/main/python/tools/utils/static_ps/reader_helper_hdfs.py

@@ -43,8 +43,6 @@ def get_reader(input_var, config):
     train_data_path = config.get("runner.train_data_dir")
     train_data_path = config.get("runner.train_data_dir")
     assert train_data_path != ""
     assert train_data_path != ""
 
 
-    train_data_path = os.path.join(config["config_abs_dir"], train_data_path)
-
     assert reader_type in [
     assert reader_type in [
         "QueueDataset", "DataLoader", "RecDataset", "InmemoryDataset", None
         "QueueDataset", "DataLoader", "RecDataset", "InmemoryDataset", None
     ]
     ]
@@ -81,6 +79,7 @@ def get_infer_reader(input_var, config):
 
 
 
 
 def get_file_list(data_path, config):
 def get_file_list(data_path, config):
+    logger.info("data_path: {}".format(data_path))
     dirs,file_list = hdfs_client.ls_dir(data_path)
     dirs,file_list = hdfs_client.ls_dir(data_path)
     # 如果配置中指定了分割文件列表
     # 如果配置中指定了分割文件列表
     if config.get("runner.split_file_list"):
     if config.get("runner.split_file_list"):