You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/09/29 22:15:09 UTC
svn commit: r451421 - in /lucene/hadoop/trunk: CHANGES.txt
src/java/org/apache/hadoop/dfs/FSNamesystem.java
src/java/org/apache/hadoop/fs/Path.java
src/test/org/apache/hadoop/fs/TestPath.java
Author: cutting
Date: Fri Sep 29 13:15:09 2006
New Revision: 451421
URL: http://svn.apache.org/viewvc?view=rev&rev=451421
Log:
HADOOP-423. Normalize paths containing directories named '.' or '..'. Contributed by Wendy.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestPath.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=451421&r1=451420&r2=451421
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Sep 29 13:15:09 2006
@@ -90,6 +90,11 @@
elsewhere. The web interface is also updated to display killed
tasks. (omalley via cutting)
+23. HADOOP-423. Normalize Paths containing directories named "." and
+ "..", using the standard, unix interpretation. Also add checks in
+ DFS, prohibiting the use of "." or ".." as directory or file
+ names. (Wendy Chien via cutting)
+
Release 0.6.2 (unreleased)
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=451421&r1=451420&r2=451421
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Fri Sep 29 13:15:09 2006
@@ -21,6 +21,7 @@
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.mapred.StatusHttpServer;
+import org.apache.hadoop.fs.Path;
import java.io.*;
import java.net.InetSocketAddress;
@@ -398,6 +399,9 @@
+src+" for "+holder+" at "+clientMachine);
if( isInSafeMode() )
throw new SafeModeException( "Cannot create file" + src, safeMode );
+ if (!isValidName(src.toString())) {
+ throw new IOException("Invalid file name: " + src);
+ }
try {
if (pendingCreates.get(src) != null) {
throw new AlreadyBeingCreatedException(
@@ -730,6 +734,9 @@
NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: " + src + " to " + dst );
if( isInSafeMode() )
throw new SafeModeException( "Cannot rename " + src, safeMode );
+ if (!isValidName(dst.toString())) {
+ throw new IOException("Invalid name: " + dst);
+ }
return dir.renameTo(src, dst);
}
@@ -785,12 +792,42 @@
}
/**
+ * Whether the pathname is valid. Currently prohibits relative paths,
+ * and names which contain a ":" or "/"
+ */
+ private boolean isValidName(String src) {
+
+ // Path must be absolute.
+ if (!src.startsWith(Path.SEPARATOR)) {
+ return false;
+ }
+
+ // Check for ".." "." ":" "/"
+ Enumeration tokens = new StringTokenizer(src, Path.SEPARATOR);
+ ArrayList list = Collections.list(tokens);
+ for (int i = 0; i < list.size(); i++) {
+ String element = (String)list.get(i);
+ if (element.equals("..") ||
+ element.equals(".") ||
+ (element.indexOf(":") >= 0) ||
+ (element.indexOf("/") >= 0)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
* Create all the necessary directories
*/
public boolean mkdirs( String src ) throws IOException {
NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src );
if( isInSafeMode() )
throw new SafeModeException( "Cannot create directory " + src, safeMode );
+ if (!isValidName(src)) {
+ throw new IOException("Invalid directory name: " + src);
+ }
return dir.mkdirs(src);
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java?view=diff&rev=451421&r1=451420&r2=451421
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/Path.java Fri Sep 29 13:15:09 2006
@@ -57,13 +57,15 @@
this.elements = child.elements;
} else {
this.isAbsolute = parent.isAbsolute;
- this.elements = new String[parent.elements.length+child.elements.length];
+ ArrayList list = new ArrayList(parent.elements.length+child.elements.length);
for (int i = 0; i < parent.elements.length; i++) {
- elements[i] = parent.elements[i];
+ list.add(parent.elements[i]);
}
for (int i = 0; i < child.elements.length; i++) {
- elements[i+parent.elements.length] = child.elements[i];
+ list.add(child.elements[i]);
}
+ normalize(list);
+ this.elements = (String[])list.toArray(new String[list.size()]);
}
this.drive = child.drive == null ? parent.drive : child.drive;
}
@@ -82,10 +84,10 @@
// determine whether the path is absolute
this.isAbsolute = pathString.startsWith(SEPARATOR);
-
// tokenize the path into elements
- Enumeration tokens = new StringTokenizer(pathString, SEPARATOR);
+ Enumeration tokens = new StringTokenizer(pathString, SEPARATOR);
ArrayList list = Collections.list(tokens);
+ normalize(list);
this.elements = (String[])list.toArray(new String[list.size()]);
}
@@ -180,5 +182,56 @@
return elements.length;
}
+ /*
+ * Removes '.' and '..'
+ */
+ private void normalize(ArrayList list) {
+ boolean canNormalize = this.isAbsolute;
+ boolean changed = false; // true if we have detected any . or ..
+ int index = 0;
+ int listSize = list.size();
+ for (int i = 0; i < listSize; i++) {
+ // Invariant: (index >= 0) && (index <= i)
+ if (list.get(i).equals(".")) {
+ changed = true;
+ } else {
+ if (canNormalize) {
+ if (list.get(i).equals("..")) {
+ if ((index > 0) && !list.get(index-1).equals("..")) {
+ index--; // effectively deletes the last element currently in list.
+ changed = true;
+ } else { // index == 0
+ // the first element is now going to be '..'
+ canNormalize = false;
+ list.set(index, "..");
+ isAbsolute = false;
+ index++;
+ }
+ } else { // list.get(i) != ".." or "."
+ if (changed) {
+ list.set(index, list.get(i));
+ }
+ index++;
+ }
+ } else { // !canNormalize
+ if (changed) {
+ list.set(index, list.get(i));
+ }
+ index++;
+ if (!list.get(i).equals("..")) {
+ canNormalize = true;
+ }
+ } // else !canNormalize
+ }
+ } // for
+
+ // Remove the junk at the end of the list.
+ for (int j = listSize-1; j >= index; j--) {
+ list.remove(j);
+ }
+
+ }
+
+
}
Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestPath.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestPath.java?view=diff&rev=451421&r1=451420&r2=451421
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestPath.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestPath.java Fri Sep 29 13:15:09 2006
@@ -98,4 +98,42 @@
assertFalse(new Path("/").equals(new Path("/foo")));
}
+ public void testDots() {
+ // Test Path(String)
+ assertEquals(new Path("/foo/bar/baz").toString(), "/foo/bar/baz");
+ assertEquals(new Path("/foo/bar/../baz").toString(), "/foo/baz");
+ assertEquals(new Path("/foo/bar/./baz").toString(), "/foo/bar/baz");
+ assertEquals(new Path("/foo/bar/baz/../../fud").toString(), "/foo/fud");
+ assertEquals(new Path("/foo/bar/baz/.././../fud").toString(), "/foo/fud");
+ assertEquals(new Path("../../foo/bar").toString(), "../../foo/bar");
+ assertEquals(new Path(".././../foo/bar").toString(), "../../foo/bar");
+ assertEquals(new Path("./foo/bar/baz").toString(), "foo/bar/baz");
+ assertEquals(new Path("/foo/bar/../../baz/boo").toString(), "/baz/boo");
+ assertEquals(new Path("/foo/bar/../../../baz").toString(), "../baz");
+ assertEquals(new Path("foo/bar/").toString(), "foo/bar");
+ assertEquals(new Path("foo/bar/../baz").toString(), "foo/baz");
+ assertEquals(new Path("foo/bar/../../baz/boo").toString(), "baz/boo");
+
+
+ // Test Path(Path,Path)
+ assertEquals(new Path("/foo/bar", "baz/boo").toString(), "/foo/bar/baz/boo");
+ assertEquals(new Path("foo/bar/","baz/bud").toString(), "foo/bar/baz/bud");
+
+ assertEquals(new Path("/foo/bar","../../boo/bud").toString(), "/boo/bud");
+ assertEquals(new Path("foo/bar","../../boo/bud").toString(), "boo/bud");
+ assertEquals(new Path("","boo/bud").toString(), "boo/bud");
+
+ assertEquals(new Path("/foo/bar/baz","../../boo/bud").toString(), "/foo/boo/bud");
+ assertEquals(new Path("foo/bar/baz","../../boo/bud").toString(), "foo/boo/bud");
+
+ assertEquals(new Path("/foo/bar/","../../../baz/boo").toString(), "../baz/boo");
+
+ assertEquals(new Path("../../","../../boo/bud").toString(), "../../../../boo/bud");
+ assertEquals(new Path("../../foo","../../../boo/bud").toString(), "../../../../boo/bud");
+ assertEquals(new Path("../../foo/bar","../boo/bud").toString(), "../../foo/boo/bud");
+
+ assertEquals(new Path("foo/bar/baz","../../..").toString(), "");
+ assertEquals(new Path("foo/bar/baz","../../../../..").toString(), "../..");
+ }
+
}