Browse Source

read hdfs

丁云鹏 5 months ago
parent
commit
ae6bc78bc9

+ 7 - 1
recommend-model-produce/src/main/python/tools/utils/static_ps/reader_helper_hdfs.py

@@ -79,7 +79,13 @@ def get_infer_reader(input_var, config):
 
 
 def get_file_list(data_path, config):
-    logger.info("data_path: {} exists: {}".format(data_path, hdfs_client.is_exist(data_path)))
+    logger.info("data_path: {}".format(data_path))
+
+    cmd = "hadoop fs -test -e {}".format(data_path)
+    ret, out = self._run_cmd(cmd, redirect_stderr=True, retry_times=1)
+    logger.info("ret: {} out: {}".format(ret, out))
+
+
     try:
         dirs,file_list = hdfs_client.ls_dir(data_path)
     except Exception as e: