You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/07/06 22:57:47 UTC
[25/48] hadoop git commit: HADOOP-10798. globStatus() should always
return a sorted list of files (cmccabe)
HADOOP-10798. globStatus() should always return a sorted list of files (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52d4f05a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52d4f05a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52d4f05a
Branch: refs/heads/YARN-2928
Commit: 52d4f05a83024cbe78eff6c58048c9595261a27f
Parents: 0e713f2
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:40:21 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:31:57 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/fs/Globber.java | 11 ++++++++-
.../org/apache/hadoop/fs/TestGlobPaths.java | 25 ++++++++++++++++++++
3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2aab8b4..4d69a229 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -905,6 +905,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/'
(J.Andreina via vinayakumarb)
+ HADOOP-10798. globStatus() should always return a sorted list of files
+ (cmccabe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index 8a8137a..9cb810f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.commons.logging.LogFactory;
@@ -285,6 +286,14 @@ class Globber {
(flattenedPatterns.size() <= 1)) {
return null;
}
- return results.toArray(new FileStatus[0]);
+ /*
+ * In general, the results list will already be sorted, since listStatus
+ * returns results in sorted order for many Hadoop filesystems. However,
+ * not all Hadoop filesystems have this property. So we sort here in order
+ * to get consistent results. See HADOOP-10798 for details.
+ */
+ FileStatus ret[] = results.toArray(new FileStatus[0]);
+ Arrays.sort(ret);
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52d4f05a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
index 50e2e5b..afd8fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
@@ -21,9 +21,11 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.UUID;
import java.util.regex.Pattern;
+import com.google.common.collect.Ordering;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -1284,4 +1286,27 @@ public class TestGlobPaths {
public void testNonTerminalGlobsOnFC() throws Exception {
testOnFileContext(new TestNonTerminalGlobs(true));
}
+
+ @Test
+ public void testLocalFilesystem() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ String localTmp = System.getProperty("java.io.tmpdir");
+ Path base = new Path(new Path(localTmp), UUID.randomUUID().toString());
+ Assert.assertTrue(fs.mkdirs(base));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "e")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "c")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "a")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "d")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "b")));
+ fs.deleteOnExit(base);
+ FileStatus[] status = fs.globStatus(new Path(base, "*"));
+ ArrayList list = new ArrayList();
+ for (FileStatus f: status) {
+ list.add(f.getPath().toString());
+ }
+ boolean sorted = Ordering.natural().isOrdered(list);
+ Assert.assertTrue(sorted);
+ }
}
+