You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ro...@apache.org on 2015/03/13 15:00:44 UTC
falcon git commit: FALCON-1094 getAllFilesRecursivelyHDFS without
recursive call. Contributed by Ruslan Ostafiychuk
Repository: falcon
Updated Branches:
refs/heads/master 334669d84 -> 1d7ca192b
FALCON-1094 getAllFilesRecursivelyHDFS without recursive call. Contributed by Ruslan Ostafiychuk
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/1d7ca192
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/1d7ca192
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/1d7ca192
Branch: refs/heads/master
Commit: 1d7ca192b8ba88078e1716f4dc0ab81707c38144
Parents: 334669d
Author: Ruslan Ostafiychuk <ro...@apache.org>
Authored: Fri Mar 13 15:59:24 2015 +0200
Committer: Ruslan Ostafiychuk <ro...@apache.org>
Committed: Fri Mar 13 15:59:24 2015 +0200
----------------------------------------------------------------------
falcon-regression/CHANGES.txt | 2 ++
.../falcon/regression/core/util/HadoopUtil.java | 22 ++++++++------------
2 files changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/1d7ca192/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 4b38ad6..3e32d78 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -60,6 +60,8 @@ Trunk (Unreleased)
via Samarth Gupta)
IMPROVEMENTS
+ FALCON-1094 getAllFilesRecursivelyHDFS without recursive call (Ruslan Ostafiychuk)
+
FALCON-1093 Tag all new tests added to falcon-regression (Paul Isaychuk via Ruslan Ostafiychuk)
FALCON-1089 ProcessInstanceStatusTest improvement (Paul Isaychuk via Ruslan Ostafiychuk)
http://git-wip-us.apache.org/repos/asf/falcon/blob/1d7ca192/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
index 7122274..e2c3ae2 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
@@ -24,7 +24,9 @@ import org.apache.falcon.regression.core.helpers.ColoHelper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.log4j.Logger;
import java.io.File;
@@ -114,23 +116,17 @@ public final class HadoopUtil {
public static List<Path> getAllFilesRecursivelyHDFS(
FileSystem fs, Path location) throws IOException {
List<Path> returnList = new ArrayList<Path>();
- FileStatus[] stats;
+ RemoteIterator<LocatedFileStatus> remoteIterator;
try {
- stats = fs.listStatus(location);
+ remoteIterator = fs.listFiles(location, true);
} catch (FileNotFoundException e) {
- e.printStackTrace();
- return new ArrayList<Path>();
- }
- if (stats == null) {
+ LOGGER.info("Path '" + location + "' is not found on " + fs.getUri());
return returnList;
}
- for (FileStatus stat : stats) {
- if (!isDir(stat)) {
- if (!stat.getPath().toUri().toString().contains("_SUCCESS")) {
- returnList.add(stat.getPath());
- }
- } else {
- returnList.addAll(getAllFilesRecursivelyHDFS(fs, stat.getPath()));
+ while(remoteIterator.hasNext()) {
+ Path path = remoteIterator.next().getPath();
+ if (!path.toUri().toString().contains("_SUCCESS")) {
+ returnList.add(path);
}
}
return returnList;