You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2013/06/06 05:29:43 UTC
svn commit: r1490115 - in
/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/fs/local/
src/main/java/org/apache/hadoop/fs/viewfs/
src/test/java/org/apache/hadoop/...
Author: suresh
Date: Thu Jun 6 03:29:43 2013
New Revision: 1490115
URL: http://svn.apache.org/r1490115
Log:
HADOOP-8957. Merge r1420965 from trunk
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1490115&r1=1490114&r2=1490115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Jun 6 03:29:43 2013
@@ -239,6 +239,9 @@ Release 2.1.0-beta - UNRELEASED
HADOOP-9593. stack trace printed at ERROR for all yarn clients without
hadoop.home set (stevel)
+ HADOOP-8957 AbstractFileSystem#IsValidName should be overridden for
+ embedded file systems like ViewFs (Chris Nauroth via Sanjay Radia)
+
BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS
HADOOP-8924. Hadoop Common creating package-info.java must not depend on
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1490115&r1=1490114&r2=1490115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Thu Jun 6 03:29:43 2013
@@ -85,14 +85,20 @@ public abstract class AbstractFileSystem
}
/**
- * Prohibits names which contain a ".", "..", ":" or "/"
+ * Returns true if the specified string is considered valid in the path part
+ * of a URI by this file system. The default implementation enforces the rules
+ * of HDFS, but subclasses may override this method to implement specific
+ * validation rules for specific file systems.
+ *
+ * @param src String source filename to check, path part of the URI
+ * @return boolean true if the specified string is considered valid
*/
- private static boolean isValidName(String src) {
- // Check for ".." "." ":" "/"
+ public boolean isValidName(String src) {
+ // Prohibit ".." "." and anything containing ":"
StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
while(tokens.hasMoreTokens()) {
String element = tokens.nextToken();
- if (element.equals("target/generated-sources") ||
+ if (element.equals("..") ||
element.equals(".") ||
(element.indexOf(":") >= 0)) {
return false;
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java?rev=1490115&r1=1490114&r2=1490115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java Thu Jun 6 03:29:43 2013
@@ -278,4 +278,9 @@ public abstract class FilterFs extends A
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
return myFs.getDelegationTokens(renewer);
}
+
+ @Override
+ public boolean isValidName(String src) {
+ return myFs.isValidName(src);
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1490115&r1=1490114&r2=1490115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java Thu Jun 6 03:29:43 2013
@@ -158,6 +158,14 @@ public class RawLocalFs extends Delegate
}
}
+ @Override
+ public boolean isValidName(String src) {
+ // Different local file systems have different validation rules. Skip
+ // validation here and just let the OS handle it. This is consistent with
+ // RawLocalFileSystem.
+ return true;
+ }
+
@Override
public Path getLinkTarget(Path f) throws IOException {
/* We should never get here. Valid local links are resolved transparently
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java?rev=1490115&r1=1490114&r2=1490115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java Thu Jun 6 03:29:43 2013
@@ -83,7 +83,12 @@ class ChRootedFs extends AbstractFileSys
return new Path((chRootPathPart.isRoot() ? "" : chRootPathPartString)
+ path.toUri().getPath());
}
-
+
+ @Override
+ public boolean isValidName(String src) {
+ return myFs.isValidName(fullPath(new Path(src)).toUri().toString());
+ }
+
public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
throws URISyntaxException {
super(fs.getUri(), fs.getUri().getScheme(),
@@ -103,7 +108,7 @@ class ChRootedFs extends AbstractFileSys
// scheme:/// and scheme://authority/
myUri = new URI(myFs.getUri().toString() +
(myFs.getUri().getAuthority() == null ? "" : Path.SEPARATOR) +
- chRootPathPart.toString().substring(1));
+ chRootPathPart.toUri().getPath().substring(1));
super.checkPath(theRoot);
}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1490115&r1=1490114&r2=1490115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java Thu Jun 6 03:29:43 2013
@@ -64,6 +64,9 @@ import org.apache.hadoop.util.Time;
@InterfaceAudience.Public
@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
public class ViewFileSystem extends FileSystem {
+
+ private static final Path ROOT_PATH = new Path(Path.SEPARATOR);
+
static AccessControlException readOnlyMountTable(final String operation,
final String p) {
return new AccessControlException(
@@ -99,23 +102,6 @@ public class ViewFileSystem extends File
Path homeDir = null;
/**
- * Prohibits names which contain a ".", "..", ":" or "/"
- */
- private static boolean isValidName(final String src) {
- // Check for ".." "." ":" "/"
- final StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
- while(tokens.hasMoreTokens()) {
- String element = tokens.nextToken();
- if (element.equals("..") ||
- element.equals(".") ||
- (element.indexOf(":") >= 0)) {
- return false;
- }
- }
- return true;
- }
-
- /**
* Make the path Absolute and get the path-part of a pathname.
* Checks that URI matches this file system
* and that the path-part is a valid name.
@@ -126,10 +112,6 @@ public class ViewFileSystem extends File
private String getUriPath(final Path p) {
checkPath(p);
String s = makeAbsolute(p).toUri().getPath();
- if (!isValidName(s)) {
- throw new InvalidPathException("Path part " + s + " from URI" + p
- + " is not a valid filename.");
- }
return s;
}
@@ -689,7 +671,7 @@ public class ViewFileSystem extends File
PERMISSION_RRR, ugi.getUserName(), ugi.getGroupNames()[0],
new Path(theInternalDir.fullPath).makeQualified(
- myUri, null));
+ myUri, ROOT_PATH));
}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1490115&r1=1490114&r2=1490115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java Thu Jun 6 03:29:43 2013
@@ -597,6 +597,12 @@ public class ViewFs extends AbstractFile
return result;
}
+ @Override
+ public boolean isValidName(String src) {
+ // Prefix validated at mount time and rest of path validated by mount target.
+ return true;
+ }
+
/*
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java?rev=1490115&r1=1490114&r2=1490115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java Thu Jun 6 03:29:43 2013
@@ -25,6 +25,7 @@ import java.util.EnumSet;
import static org.apache.hadoop.fs.FileContextTestHelper.*;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestHelper;
@@ -36,6 +37,7 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestChRootedFs {
FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
@@ -308,4 +310,21 @@ public class TestChRootedFs {
fc.getDefaultFileSystem().resolvePath(new Path("/nonExisting"));
}
+ @Test
+ public void testIsValidNameValidInBaseFs() throws Exception {
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
+ Mockito.doReturn(true).when(baseFs).isValidName(Mockito.anyString());
+ Assert.assertTrue(chRootedFs.isValidName("/test"));
+ Mockito.verify(baseFs).isValidName("/chroot/test");
+ }
+
+ @Test
+ public void testIsValidNameInvalidInBaseFs() throws Exception {
+ AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
+ ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
+ Mockito.doReturn(false).when(baseFs).isValidName(Mockito.anyString());
+ Assert.assertFalse(chRootedFs.isValidName("/test"));
+ Mockito.verify(baseFs).isValidName("/chroot/test");
+ }
}