You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2013/08/20 22:19:08 UTC
svn commit: r1515955 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/ src/test/java/org/apache/hadoop/fs/
Author: wang
Date: Tue Aug 20 20:19:07 2013
New Revision: 1515955
URL: http://svn.apache.org/r1515955
Log:
HADOOP-9877. Fix listing of snapshot directories in globStatus. (Binglin Chang via Andrew Wang)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1515955&r1=1515954&r2=1515955&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Aug 20 20:19:07 2013
@@ -349,6 +349,8 @@ Release 2.3.0 - UNRELEASED
HADOOP-9865. FileContext#globStatus has a regression with respect to
relative path. (Chuan Lin via Colin Patrick McCabe)
+ HADOOP-9877. Fix listing of snapshot directories in globStatus.
+ (Binglin Chang via Andrew Wang)
Release 2.1.1-beta - UNRELEASED
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java?rev=1515955&r1=1515954&r2=1515955&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java Tue Aug 20 20:19:07 2013
@@ -62,6 +62,18 @@ class Globber {
}
}
+ private FileStatus getFileLinkStatus(Path path) {
+ try {
+ if (fs != null) {
+ return fs.getFileLinkStatus(path);
+ } else {
+ return fc.getFileLinkStatus(path);
+ }
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
private FileStatus[] listStatus(Path path) {
try {
if (fs != null) {
@@ -122,6 +134,18 @@ class Globber {
return authority ;
}
+ /**
+ * The glob filter builds a regexp per path component. If the component
+ * does not contain a shell metachar, then it falls back to appending the
+ * raw string to the list of built up paths. This raw path needs to have
+ * the quoting removed. Ie. convert all occurrences of "\X" to "X"
+ * @param name of the path component
+ * @return the unquoted path component
+ */
+ private static String unquotePathComponent(String name) {
+ return name.replaceAll("\\\\(.)", "$1");
+ }
+
public FileStatus[] glob() throws IOException {
// First we get the scheme and authority of the pattern that was passed
// in.
@@ -176,14 +200,30 @@ class Globber {
resolvedCandidate.isDirectory() == false) {
continue;
}
- FileStatus[] children = listStatus(candidate.getPath());
- for (FileStatus child : children) {
- // Set the child path based on the parent path.
- // This keeps the symlinks in our path.
- child.setPath(new Path(candidate.getPath(),
- child.getPath().getName()));
- if (globFilter.accept(child.getPath())) {
- newCandidates.add(child);
+ // For components without pattern, we get its FileStatus directly
+ // using getFileLinkStatus for two reasons:
+ // 1. It should be faster to only get FileStatus needed rather than
+ // get all children.
+ // 2. Some special filesystem directories (e.g. HDFS snapshot
+ // directories) are not returned by listStatus, but do exist if
+ // checked explicitly via getFileLinkStatus.
+ if (globFilter.hasPattern()) {
+ FileStatus[] children = listStatus(candidate.getPath());
+ for (FileStatus child : children) {
+ // Set the child path based on the parent path.
+ // This keeps the symlinks in our path.
+ child.setPath(new Path(candidate.getPath(),
+ child.getPath().getName()));
+ if (globFilter.accept(child.getPath())) {
+ newCandidates.add(child);
+ }
+ }
+ } else {
+ Path p = new Path(candidate.getPath(), unquotePathComponent(component));
+ FileStatus s = getFileLinkStatus(p);
+ if (s != null) {
+ s.setPath(p);
+ newCandidates.add(s);
}
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java?rev=1515955&r1=1515954&r2=1515955&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java Tue Aug 20 20:19:07 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Options.Rena
import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.After;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -632,6 +633,20 @@ public abstract class FileContextMainOpe
filteredPaths));
}
+ protected Path getHiddenPathForTest() {
+ return null;
+ }
+
+ @Test
+ public void testGlobStatusFilterWithHiddenPathTrivialFilter()
+ throws Exception {
+ Path hidden = getHiddenPathForTest();
+ Assume.assumeNotNull(hidden);
+ FileStatus[] filteredPaths = fc.util().globStatus(hidden, DEFAULT_FILTER);
+ Assert.assertNotNull(filteredPaths);
+ Assert.assertEquals(1, filteredPaths.length);
+ }
+
@Test
public void testWriteReadAndDeleteEmptyFile() throws Exception {
writeReadAndDelete(0);
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java?rev=1515955&r1=1515954&r2=1515955&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java Tue Aug 20 20:19:07 2013
@@ -517,6 +517,26 @@ public class TestFsShellReturnCode {
}
return stat;
}
+
+ @Override
+ public FileStatus getFileLinkStatus(Path p) throws IOException {
+ String f = makeQualified(p).toString();
+ FileStatus stat = super.getFileLinkStatus(p);
+
+ stat.getPermission();
+ if (owners.containsKey(f)) {
+ stat.setOwner("STUB-"+owners.get(f));
+ } else {
+ stat.setOwner("REAL-"+stat.getOwner());
+ }
+ if (groups.containsKey(f)) {
+ stat.setGroup("STUB-"+groups.get(f));
+ } else {
+ stat.setGroup("REAL-"+stat.getGroup());
+ }
+ return stat;
+ }
+
}
static class MyFsShell extends FsShell {