|
@@ -232,6 +232,8 @@ class QueueDatset(object):
|
|
pipe_command=self.pipe_command,
|
|
pipe_command=self.pipe_command,
|
|
batch_size=self.batch_size,
|
|
batch_size=self.batch_size,
|
|
thread_num=self.thread_num,
|
|
thread_num=self.thread_num,
|
|
|
|
+ fs_name=configs["fs.default.name"], # 设置 HDFS 名称
|
|
|
|
+ fs_ugi=configs["hadoop.job.ugi"], # 设置 HDFS 用户组信息
|
|
download_cmd=""
|
|
download_cmd=""
|
|
)
|
|
)
|
|
print("dataset get_reader thread_num:", self.thread_num)
|
|
print("dataset get_reader thread_num:", self.thread_num)
|