read hdfs

recommend-model-produce/src/main/python/tools/utils/my_hdfs_client.py

@@ -1,8 +1,27 @@
 from paddle.distributed.fleet.utils import HDFSClient
+import os
 
+class ExecuteError(Exception):
+    pass
 class MyHDFSClient(HDFSClient):
-    def _run_safe_cmd(self, cmd, redirect_stderr=False, retry_times=5):
-            cmdStr = " ".join(cmd)
-            ret, lines = super()._run_safe_cmd(cmdStr, redirect_stderr, retry_times)
-            log.info("cmdStr: {} ret: {} lines: {}".format(cmdStr,ret,lines))
-            return ret, lines
+    def _ls_dir(self, fs_path):
+        cmd = "-ls " + fs_path
+        ret, lines = super()._run_cmd(cmd)
+
+        if ret != 0:
+            raise ExecuteError(cmd)
+
+        dirs = []
+        files = []
+        for line in lines:
+            arr = line.split()
+            if len(arr) != 8:
+                continue
+
+            p = os.path.basename(arr[7])
+            if arr[0][0] == 'd':
+                dirs.append(p)
+            else:
+                files.append(p)
+
+        return dirs, files