Browse Source

read hdfs

丁云鹏 5 months ago
parent
commit
e276324449

+ 2 - 0
recommend-model-produce/src/main/python/tools/utils/static_ps/reader_helper_hdfs.py

@@ -232,6 +232,8 @@ class QueueDatset(object):
             pipe_command=self.pipe_command,
             pipe_command=self.pipe_command,
             batch_size=self.batch_size,
             batch_size=self.batch_size,
             thread_num=self.thread_num,
             thread_num=self.thread_num,
+            fs_name=configs["fs.default.name"],  # 设置 HDFS 名称
+            fs_ugi=configs["hadoop.job.ugi"],  # 设置 HDFS 用户组信息
             download_cmd=""
             download_cmd=""
             )
             )
         print("dataset get_reader thread_num:", self.thread_num)
         print("dataset get_reader thread_num:", self.thread_num)