read hdfs

recommend-model-produce/src/main/python/models/dssm/config_ps_hdfs.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 runner:
-  train_data_dir: "hdfs:/dyp"
+  train_data_dir: "/dyp"
   train_reader_path: "bq_reader_train"  # importlib format
   train_batch_size: 8
   model_save_path: "output_model_dssm"

recommend-model-produce/src/main/python/tools/utils/static_ps/reader_helper_hdfs.py

@@ -84,12 +84,7 @@ def get_file_list(data_path, config):
     cmd = "test -e {}".format(data_path)
     ret, out = hdfs_client._run_cmd(cmd, redirect_stderr=True, retry_times=1)
     logger.info("ret: {} out: {}".format(ret, out))
-
-
-    try:
-        dirs,file_list = hdfs_client.ls_dir(data_path)
-    except Exception as e:
-        print(f"Error occurred: {e}")
+    dirs,file_list = hdfs_client.ls_dir(data_path)
     # 如果配置中指定了分割文件列表
     if config.get("runner.split_file_list"):
         logger.info("Split file list for worker {}".format(dist.get_rank()))