You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2012/05/05 00:14:11 UTC
svn commit: r1334231 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/fs/viewfs/
src/test/java/org/apache/hadoop/fs/viewfs/
Author: atm
Date: Fri May 4 22:14:10 2012
New Revision: 1334231
URL: http://svn.apache.org/viewvc?rev=1334231&view=rev
Log:
HADOOP-8349. ViewFS doesn't work when the root of a file system is mounted. Contributed by Aaron T. Myers.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1334231&r1=1334230&r2=1334231&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri May 4 22:14:10 2012
@@ -423,6 +423,8 @@ Release 2.0.0 - UNRELEASED
HADOOP-8355. SPNEGO filter throws/logs exception when authentication fails (tucu)
+ HADOOP-8349. ViewFS doesn't work when the root of a file system is mounted. (atm)
+
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1334231&r1=1334230&r2=1334231&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Fri May 4 22:14:10 2012
@@ -346,7 +346,7 @@ public abstract class AbstractFileSystem
path);
} else {
throw new InvalidPathException(
- "Path without scheme with non-null autorhrity:" + path);
+ "Path without scheme with non-null authority:" + path);
}
}
String thisScheme = this.getUri().getScheme();
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1334231&r1=1334230&r2=1334231&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Fri May 4 22:14:10 2012
@@ -223,6 +223,13 @@ public class Path implements Comparable
return isUriPathAbsolute();
}
+ /**
+ * @return true if and only if this path represents the root of a file system
+ */
+ public boolean isRoot() {
+ return getParent() == null;
+ }
+
/** Returns the final component of this path.*/
public String getName() {
String path = uri.getPath();
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1334231&r1=1334230&r2=1334231&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Fri May 4 22:14:10 2012
@@ -75,7 +75,8 @@ class ChRootedFileSystem extends FilterF
protected Path fullPath(final Path path) {
super.checkPath(path);
return path.isAbsolute() ?
- new Path(chRootPathPartString + path.toUri().getPath()) :
+ new Path((chRootPathPart.isRoot() ? "" : chRootPathPartString)
+ + path.toUri().getPath()) :
new Path(chRootPathPartString + workingDir.toUri().getPath(), path);
}
@@ -127,7 +128,7 @@ class ChRootedFileSystem extends FilterF
}
String pathPart = p.toUri().getPath();
return (pathPart.length() == chRootPathPartString.length()) ? "" : pathPart
- .substring(chRootPathPartString.length() + 1);
+ .substring(chRootPathPartString.length() + (chRootPathPart.isRoot() ? 0 : 1));
}
@Override
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java?rev=1334231&r1=1334230&r2=1334231&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java Fri May 4 22:14:10 2012
@@ -79,7 +79,8 @@ class ChRootedFs extends AbstractFileSys
*/
protected Path fullPath(final Path path) {
super.checkPath(path);
- return new Path(chRootPathPartString + path.toUri().getPath());
+ return new Path((chRootPathPart.isRoot() ? "" : chRootPathPartString)
+ + path.toUri().getPath());
}
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
@@ -127,7 +128,8 @@ class ChRootedFs extends AbstractFileSys
}
String pathPart = p.toUri().getPath();
return (pathPart.length() == chRootPathPartString.length()) ?
- "" : pathPart.substring(chRootPathPartString.length() + 1);
+ "" : pathPart.substring(chRootPathPartString.length() +
+ (chRootPathPart.isRoot() ? 0 : 1));
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java?rev=1334231&r1=1334230&r2=1334231&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java Fri May 4 22:14:10 2012
@@ -71,11 +71,8 @@ public class ViewFileSystemBaseTest {
@Before
public void setUp() throws Exception {
- targetTestRoot = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
- // In case previous test was killed before cleanup
- fsTarget.delete(targetTestRoot, true);
+ initializeTargetTestRoot();
- fsTarget.mkdirs(targetTestRoot);
// Make user and data dirs - we creates links to them in the mount table
fsTarget.mkdirs(new Path(targetTestRoot,"user"));
fsTarget.mkdirs(new Path(targetTestRoot,"data"));
@@ -99,7 +96,16 @@ public class ViewFileSystemBaseTest {
fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true);
}
+ void initializeTargetTestRoot() throws IOException {
+ targetTestRoot = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
+ // In case previous test was killed before cleanup
+ fsTarget.delete(targetTestRoot, true);
+
+ fsTarget.mkdirs(targetTestRoot);
+ }
+
void setupMountPoints() {
+ ConfigUtil.addLink(conf, "/targetRoot", targetTestRoot.toUri());
ConfigUtil.addLink(conf, "/user", new Path(targetTestRoot,"user").toUri());
ConfigUtil.addLink(conf, "/user2", new Path(targetTestRoot,"user").toUri());
ConfigUtil.addLink(conf, "/data", new Path(targetTestRoot,"data").toUri());
@@ -121,7 +127,7 @@ public class ViewFileSystemBaseTest {
}
int getExpectedMountPoints() {
- return 7;
+ return 8;
}
/**
@@ -166,7 +172,7 @@ public class ViewFileSystemBaseTest {
}
}
}
- Assert.assertEquals(expectedTokenCount / 2, delTokens.size());
+ Assert.assertEquals((expectedTokenCount + 1) / 2, delTokens.size());
}
int getExpectedDelegationTokenCountWithCredentials() {
@@ -309,6 +315,16 @@ public class ViewFileSystemBaseTest {
Assert.assertTrue("Renamed dest should exist as dir in target",
fsTarget.isDirectory(new Path(targetTestRoot,"user/dirFooBar")));
+ // Make a directory under a directory that's mounted from the root of another FS
+ fsView.mkdirs(new Path("/targetRoot/dirFoo"));
+ Assert.assertTrue(fsView.exists(new Path("/targetRoot/dirFoo")));
+ boolean dirFooPresent = false;
+ for (FileStatus fileStatus : fsView.listStatus(new Path("/targetRoot/"))) {
+ if (fileStatus.getPath().getName().equals("dirFoo")) {
+ dirFooPresent = true;
+ }
+ }
+ Assert.assertTrue(dirFooPresent);
}
// rename across mount points that point to same target also fail
@@ -418,7 +434,7 @@ public class ViewFileSystemBaseTest {
}
int getExpectedDirPaths() {
- return 6;
+ return 7;
}
@Test
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java?rev=1334231&r1=1334230&r2=1334231&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java Fri May 4 22:14:10 2012
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.AbstractFile
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestHelper;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.FileContextTestHelper.fileType;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsConstants;
@@ -77,12 +78,8 @@ public class ViewFsBaseTest {
@Before
public void setUp() throws Exception {
-
- targetTestRoot = FileContextTestHelper.getAbsoluteTestRootPath(fcTarget);
- // In case previous test was killed before cleanup
- fcTarget.delete(targetTestRoot, true);
+ initializeTargetTestRoot();
- fcTarget.mkdir(targetTestRoot, FileContext.DEFAULT_PERM, true);
// Make user and data dirs - we creates links to them in the mount table
fcTarget.mkdir(new Path(targetTestRoot,"user"),
FileContext.DEFAULT_PERM, true);
@@ -100,6 +97,7 @@ public class ViewFsBaseTest {
// Set up the defaultMT in the config with our mount point links
conf = new Configuration();
+ ConfigUtil.addLink(conf, "/targetRoot", targetTestRoot.toUri());
ConfigUtil.addLink(conf, "/user",
new Path(targetTestRoot,"user").toUri());
ConfigUtil.addLink(conf, "/user2",
@@ -118,6 +116,14 @@ public class ViewFsBaseTest {
fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
// Also try viewfs://default/ - note authority is name of mount table
}
+
+ void initializeTargetTestRoot() throws IOException {
+ targetTestRoot = FileContextTestHelper.getAbsoluteTestRootPath(fcTarget);
+ // In case previous test was killed before cleanup
+ fcTarget.delete(targetTestRoot, true);
+
+ fcTarget.mkdir(targetTestRoot, FileContext.DEFAULT_PERM, true);
+ }
@After
public void tearDown() throws Exception {
@@ -128,7 +134,11 @@ public class ViewFsBaseTest {
public void testGetMountPoints() {
ViewFs viewfs = (ViewFs) fcView.getDefaultFileSystem();
MountPoint[] mountPoints = viewfs.getMountPoints();
- Assert.assertEquals(7, mountPoints.length);
+ Assert.assertEquals(8, mountPoints.length);
+ }
+
+ int getExpectedDelegationTokenCount() {
+ return 0;
}
/**
@@ -140,7 +150,7 @@ public class ViewFsBaseTest {
public void testGetDelegationTokens() throws IOException {
List<Token<?>> delTokens =
fcView.getDelegationTokens(new Path("/"), "sanjay");
- Assert.assertEquals(0, delTokens.size());
+ Assert.assertEquals(getExpectedDelegationTokenCount(), delTokens.size());
}
@@ -281,6 +291,19 @@ public class ViewFsBaseTest {
Assert.assertTrue("Renamed dest should exist as dir in target",
isDir(fcTarget,new Path(targetTestRoot,"user/dirFooBar")));
+ // Make a directory under a directory that's mounted from the root of another FS
+ fcView.mkdir(new Path("/targetRoot/dirFoo"), FileContext.DEFAULT_PERM, false);
+ Assert.assertTrue(exists(fcView, new Path("/targetRoot/dirFoo")));
+ boolean dirFooPresent = false;
+ RemoteIterator<FileStatus> dirContents = fcView.listStatus(new Path(
+ "/targetRoot/"));
+ while (dirContents.hasNext()) {
+ FileStatus fileStatus = dirContents.next();
+ if (fileStatus.getPath().getName().equals("dirFoo")) {
+ dirFooPresent = true;
+ }
+ }
+ Assert.assertTrue(dirFooPresent);
}
// rename across mount points that point to same target also fail
@@ -358,7 +381,7 @@ public class ViewFsBaseTest {
FileStatus[] dirPaths = fcView.util().listStatus(new Path("/"));
FileStatus fs;
- Assert.assertEquals(6, dirPaths.length);
+ Assert.assertEquals(7, dirPaths.length);
fs = FileContextTestHelper.containsPath(fcView, "/user", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());