You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zi...@apache.org on 2024/11/15 04:13:40 UTC
(dolphinscheduler) branch dev updated: [Fix-16331] Fix the resource file cannot be displayed hierarchically (#16799)
This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 854098bb44 [Fix-16331] Fix the resource file cannot be displayed hierarchically (#16799)
854098bb44 is described below
commit 854098bb44bebf0cb5acfa50c249358f42fa3297
Author: xiangzihao <46...@qq.com>
AuthorDate: Fri Nov 15 12:13:33 2024 +0800
[Fix-16331] Fix the resource file cannot be displayed hierarchically (#16799)
---
.../plugin/storage/hdfs/HdfsStorageOperator.java | 12 ++++++------
.../plugin/storage/hdfs/LocalStorageOperatorTest.java | 12 ++++++++----
2 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
index d3a2cad7ee..0d21b769a9 100644
--- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
+++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
@@ -32,9 +32,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
@@ -235,10 +233,12 @@ public class HdfsStorageOperator extends AbstractStorageOperator implements Clos
if (!fs.exists(path)) {
continue;
}
- RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(path, true);
- while (remoteIterator.hasNext()) {
- LocatedFileStatus locatedFileStatus = remoteIterator.next();
- result.add(transformFileStatusToResourceMetadata(locatedFileStatus));
+ FileStatus[] fileStatuses = fs.listStatus(path);
+ for (FileStatus fileStatus : fileStatuses) {
+ if (fileStatus.isDirectory()) {
+ foldersToFetch.addLast(fileStatus.getPath().toString());
+ }
+ result.add(transformFileStatusToResourceMetadata(fileStatus));
}
}
return result;
diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
index 0a9b943d5f..5730f46dbd 100644
--- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
+++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
@@ -296,10 +296,14 @@ class LocalStorageOperatorTest {
String resourceFileAbsolutePath = "file:" + baseDir;
List<StorageEntity> storageEntities =
storageOperator.listFileStorageEntityRecursively(resourceFileAbsolutePath);
- assertThat(storageEntities.size()).isEqualTo(1);
-
- StorageEntity storageEntity2 = storageEntities.get(0);
- assertThat(storageEntity2.getFullName()).isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql"));
+ assertThat(storageEntities.size()).isEqualTo(3);
+
+ StorageEntity storageEntity2 = storageEntities.stream()
+ .filter(storageEntity -> storageEntity.getFileName().equals("demo.sql"))
+ .findFirst()
+ .get();
+ assertThat(storageEntity2.getFullName())
+ .isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql"));
assertThat(storageEntity2.getFileName()).isEqualTo("demo.sql");
assertThat(storageEntity2.getPfullName()).isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory"));
assertThat(storageEntity2.isDirectory()).isFalse();