You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2013/03/28 00:47:37 UTC
svn commit: r1461901 - in
/hadoop/common/branches/branch-2/hadoop-common-project: ./ hadoop-auth/
hadoop-common/ hadoop-common/src/main/docs/ hadoop-common/src/main/java/
hadoop-common/src/main/java/org/apache/hadoop/fs/
hadoop-common/src/test/core/ ha...
Author: eli
Date: Wed Mar 27 23:47:36 2013
New Revision: 1461901
URL: http://svn.apache.org/r1461901
Log:
HADOOP-9357. Fallback to default authority if not specified in FileContext. Contributed by Andrew Wang
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-auth/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project:r1461898
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-auth/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-auth:r1461898
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common:r1461898
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1461901&r1=1461900&r2=1461901&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Mar 27 23:47:36 2013
@@ -98,6 +98,9 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9125. LdapGroupsMapping threw CommunicationException after some
idle time. (Kai Zheng via atm)
+ HADOOP-9357. Fallback to default authority if not specified in FileContext.
+ (Andrew Wang via eli)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1461898
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1461898
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1461898
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1461901&r1=1461900&r2=1461901&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Wed Mar 27 23:47:36 2013
@@ -254,17 +254,33 @@ public final class FileContext {
}
/*
- * Remove relative part - return "absolute":
- * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
- * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
+ * Resolve a relative path passed from the user.
+ *
+ * Relative paths are resolved against the current working directory
+ * (e.g. "foo/bar" becomes "/<workingDir>/foo/bar").
+ * Fully-qualified URIs (e.g. "hdfs://nn:p/foo/bar") and slash-relative paths
* ("/foo/bar") are returned unchanged.
*
+ * Additionally, we fix malformed URIs that specify a scheme but not an
+ * authority (e.g. "hdfs:///foo/bar"). Per RFC 2395, we remove the scheme
+ * if it matches the default FS, and let the default FS add in the default
+ * scheme and authority later (see {@link #AbstractFileSystem#checkPath}).
+ *
* Applications that use FileContext should use #makeQualified() since
- * they really want a fully qualified URI.
+ * they really want a fully-qualified URI.
* Hence this method is not called makeAbsolute() and
* has been deliberately declared private.
*/
private Path fixRelativePart(Path p) {
+ // Per RFC 2396 5.2, drop schema if there is a scheme but no authority.
+ if (p.hasSchemeAndNoAuthority()) {
+ String scheme = p.toUri().getScheme();
+ if (scheme.equalsIgnoreCase(defaultFS.getUri().getScheme())) {
+ p = new Path(p.toUri().getSchemeSpecificPart());
+ }
+ }
+ // Absolute paths are unchanged. Relative paths are resolved against the
+ // current working directory.
if (p.isUriPathAbsolute()) {
return p;
} else {
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1461901&r1=1461900&r2=1461901&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Wed Mar 27 23:47:36 2013
@@ -202,6 +202,10 @@ public class Path implements Comparable
return (isUriPathAbsolute() &&
uri.getScheme() == null && uri.getAuthority() == null);
}
+
+ public boolean hasSchemeAndNoAuthority() {
+ return uri.getScheme() != null && uri.getAuthority() == null;
+ }
/**
* True if the path component (i.e. directory) of this URI is absolute.
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1461898
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java?rev=1461901&r1=1461900&r2=1461901&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java Wed Mar 27 23:47:36 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.EnumSet;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -1164,6 +1166,40 @@ public abstract class FileContextMainOpe
Assert.assertEquals(fc.getFileStatus(file), fc.getFileLinkStatus(file));
}
}
+
+ /**
+ * Test that URIs with a scheme, no authority, and absolute path component
+ * resolve with the authority of the default FS.
+ */
+ @Test(timeout=30000)
+ public void testAbsolutePathSchemeNoAuthority() throws IOException,
+ URISyntaxException {
+ Path file = getTestRootPath(fc, "test/file");
+ createFile(file);
+ URI uri = file.toUri();
+ URI noAuthorityUri = new URI(uri.getScheme(), null, uri.getPath(),
+ uri.getQuery(), uri.getFragment());
+ Path noAuthority = new Path(noAuthorityUri);
+ Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority));
+ }
+
+ /**
+ * Test that URIs with a scheme, no authority, and relative path component
+ * resolve with the authority of the default FS.
+ */
+ @Test(timeout=30000)
+ public void testRelativePathSchemeNoAuthority() throws IOException,
+ URISyntaxException {
+ Path workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test"));
+ fc.setWorkingDirectory(workDir);
+ Path file = new Path(workDir, "file");
+ createFile(file);
+ URI uri = file.toUri();
+ URI noAuthorityUri = new URI(uri.getScheme() + ":file");
+ System.out.println(noAuthorityUri);
+ Path noAuthority = new Path(noAuthorityUri);
+ Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority));
+ }
protected void createFile(Path path) throws IOException {
FSDataOutputStream out = fc.create(path, EnumSet.of(CREATE),