You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/12/11 00:55:36 UTC
svn commit: r1419884 - in /hadoop/common/branches/branch-1: CHANGES.txt
src/core/org/apache/hadoop/fs/FileSystem.java
src/core/org/apache/hadoop/fs/Path.java
src/test/org/apache/hadoop/fs/TestPath.java
Author: suresh
Date: Mon Dec 10 23:55:35 2012
New Revision: 1419884
URL: http://svn.apache.org/viewvc?rev=1419884&view=rev
Log:
HADOOP-8164. Handle paths using back slash as path separator for windows only. Backported by Jing Zhao.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/Path.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestPath.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1419884&r1=1419883&r2=1419884&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Mon Dec 10 23:55:35 2012
@@ -322,6 +322,9 @@ Release 1.2.0 - unreleased
MAPREDUCE-4778. Fair scheduler event log is only written if directory
exists on HDFS. (Sandy Ryza via tomwhite)
+ HADOOP-8164. Handle paths using back slash as path separator for windows
+ only. (Daryn Sharp, backported by Jing Zhao via suresh)
+
Release 1.1.2 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileSystem.java?rev=1419884&r1=1419883&r2=1419884&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileSystem.java Mon Dec 10 23:55:35 2012
@@ -1032,11 +1032,12 @@ public abstract class FileSystem extends
results = listStatus(parentPaths, fp);
hasGlob[0] = true;
} else { // last component does not have a pattern
+ // remove the quoting of metachars in a non-regexp expansion
+ String name = unquotePathComponent(components[components.length - 1]);
// get all the path names
ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length);
for (int i = 0; i < parentPaths.length; i++) {
- parentPaths[i] = new Path(parentPaths[i],
- components[components.length - 1]);
+ parentPaths[i] = new Path(parentPaths[i], name);
if (fp.accept(parentPaths[i])) {
filteredPaths.add(parentPaths[i]);
}
@@ -1079,13 +1080,27 @@ public abstract class FileSystem extends
if (fp.hasPattern()) {
parents = FileUtil.stat2Paths(listStatus(parents, fp));
hasGlob[0] = true;
- } else {
+ } else { // the component does not have a pattern
+ // remove the quoting of metachars in a non-regexp expansion
+ String name = unquotePathComponent(filePattern[level]);
for (int i = 0; i < parents.length; i++) {
- parents[i] = new Path(parents[i], filePattern[level]);
+ parents[i] = new Path(parents[i], name);
}
}
return globPathsLevel(parents, filePattern, level + 1, hasGlob);
}
+
+ /**
+ * The glob filter builds a regexp per path component. If the component
+ * does not contain a shell metachar, then it falls back to appending the
+ * raw string to the list of built up paths. This raw path needs to have
+ * the quoting removed. Ie. convert all occurances of "\X" to "X"
+ * @param name of the path component
+ * @return the unquoted path component
+ */
+ private String unquotePathComponent(String name) {
+ return name.replaceAll("\\\\(.)", "$1");
+ }
/** Return the current user's home directory in this filesystem.
* The default implementation returns "/user/$USER/".
Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/Path.java?rev=1419884&r1=1419883&r2=1419884&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/Path.java Mon Dec 10 23:55:35 2012
@@ -154,7 +154,7 @@ public class Path implements Comparable
if (path.indexOf("//") != -1) {
path = path.replace("//", "/");
}
- if (path.indexOf("\\") != -1) {
+ if (Path.WINDOWS && path.indexOf("\\") != -1) {
path = path.replace("\\", "/");
}
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestPath.java?rev=1419884&r1=1419883&r2=1419884&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestPath.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestPath.java Mon Dec 10 23:55:35 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
@@ -177,5 +178,58 @@ public class TestPath extends TestCase {
assertEquals("foo://bar/fud#boo", new Path(new Path(new URI(
"foo://bar/baz#bud")), new Path(new URI("/fud#boo"))).toString());
}
+
+ public void testGlobEscapeStatus() throws Exception {
+ FileSystem lfs = FileSystem.getLocal(new Configuration());
+ Path testRoot = lfs.makeQualified(new Path(System.getProperty(
+ "test.build.data", "test/build/data"), "testPathGlob"));
+ lfs.delete(testRoot, true);
+ lfs.mkdirs(testRoot);
+ assertTrue(lfs.getFileStatus(testRoot).isDir());
+ lfs.setWorkingDirectory(testRoot);
+
+ // create a couple dirs with file in them
+ Path paths[] = new Path[] { new Path(testRoot, "*/f"),
+ new Path(testRoot, "d1/f"), new Path(testRoot, "d2/f") };
+ Arrays.sort(paths);
+ for (Path p : paths) {
+ lfs.create(p).close();
+ assertTrue(lfs.exists(p));
+ }
+ // try the non-globbed listStatus
+ FileStatus stats[] = lfs.listStatus(new Path(testRoot, "*"));
+ assertEquals(1, stats.length);
+ assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());
+
+ // ensure globStatus with "*" finds all dir contents
+ stats = lfs.globStatus(new Path(testRoot, "*"));
+ Arrays.sort(stats);
+ assertEquals(paths.length, stats.length);
+ for (int i = 0; i < paths.length; i++) {
+ assertEquals(paths[i].getParent(), stats[i].getPath());
+ }
+
+ // ensure that globStatus with an escaped "\*" only finds "*"
+ stats = lfs.globStatus(new Path(testRoot, "\\*"));
+ assertEquals(1, stats.length);
+ assertEquals(new Path(testRoot, "*"), stats[0].getPath());
+
+ // try to glob the inner file for all dirs
+ stats = lfs.globStatus(new Path(testRoot, "*/f"));
+ assertEquals(paths.length, stats.length);
+ for (int i = 0; i < paths.length; i++) {
+ assertEquals(paths[i], stats[i].getPath());
+ }
+
+ // try to get the inner file for only the "*" dir
+ stats = lfs.globStatus(new Path(testRoot, "\\*/f"));
+ assertEquals(1, stats.length);
+ assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());
+
+ // try to glob all the contents of the "*" dir
+ stats = lfs.globStatus(new Path(testRoot, "\\*/*"));
+ assertEquals(1, stats.length);
+ assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());
+ }
}