You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/11/02 05:27:28 UTC
svn commit: r1404873 - in
/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common:
./ src/main/java/org/apache/hadoop/fs/local/
src/test/java/org/apache/hadoop/fs/
Author: suresh
Date: Fri Nov 2 04:27:27 2012
New Revision: 1404873
URL: http://svn.apache.org/viewvc?rev=1404873&view=rev
Log:
HADOOP-8956. FileSystem.primitiveMkdir failures on Windows cause multiple test suites to fail. Contributed by Chris Nauroth.
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt?rev=1404873&r1=1404872&r2=1404873&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt Fri Nov 2 04:27:27 2012
@@ -31,3 +31,6 @@ branch-trunk-win changes - unreleased
HADOOP-8949. Remove FileUtil.CygPathCommand dead code. (Chris Nauroth via
suresh)
+
+ HADOOP-8956. FileSystem.primitiveMkdir failures on Windows cause multiple
+ test suites to fail. (Chris Nauroth via suresh)
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1404873&r1=1404872&r2=1404873&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java Fri Nov 2 04:27:27 2012
@@ -89,9 +89,9 @@ public class RawLocalFs extends Delegate
}
// NB: Use createSymbolicLink in java.nio.file.Path once available
try {
- Shell.execCommand(Shell.LINK_COMMAND, "-s",
+ Shell.execCommand(Shell.getSymlinkCommand(
new URI(target.toString()).getPath(),
- new URI(link.toString()).getPath());
+ new URI(link.toString()).getPath()));
} catch (URISyntaxException x) {
throw new IOException("Invalid symlink path: "+x.getMessage());
} catch (IOException x) {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java?rev=1404873&r1=1404872&r2=1404873&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java Fri Nov 2 04:27:27 2012
@@ -68,7 +68,7 @@ public final class FileContextTestHelper
public static String getAbsoluteTestRootDir(FileContext fc)
throws IOException {
if (absTestRootDir == null) {
- if (TEST_ROOT_DIR.startsWith("/")) {
+ if (new Path(TEST_ROOT_DIR).isAbsolute()) {
absTestRootDir = TEST_ROOT_DIR;
} else {
absTestRootDir = fc.getWorkingDirectory().toString() + "/"
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java?rev=1404873&r1=1404872&r2=1404873&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java Fri Nov 2 04:27:27 2012
@@ -20,9 +20,11 @@ package org.apache.hadoop.fs;
import java.io.*;
import java.util.ArrayList;
+import java.util.regex.Pattern;
import junit.framework.Assert;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -52,6 +54,12 @@ public abstract class FileContextURIBase
private static final String basePath = System.getProperty("test.build.data",
"build/test/data") + "/testContextURI";
private static final Path BASE = new Path(basePath);
+
+ // Matches anything containing <, >, :, ", |, ?, *, or anything that ends with
+ // space or dot.
+ private static final Pattern WIN_INVALID_FILE_NAME_PATTERN = Pattern.compile(
+ "^(.*?[<>\\:\"\\|\\?\\*].*?)|(.*?[ \\.])$");
+
protected FileContext fc1;
protected FileContext fc2;
@@ -81,6 +89,10 @@ public abstract class FileContextURIBase
" ", "^ " };
for (String f : fileNames) {
+ if (!isTestableFileNameOnPlatform(f)) {
+ continue;
+ }
+
// Create a file on fc2's file system using fc1
Path testPath = qualifiedPath(f, fc2);
// Ensure file does not exist
@@ -205,6 +217,10 @@ public abstract class FileContextURIBase
"deleteTest/()&^%$#@!~_+}{><?", " ", "^ " };
for (String f : dirNames) {
+ if (!isTestableFileNameOnPlatform(f)) {
+ continue;
+ }
+
// Create a file on fc2's file system using fc1
Path testPath = qualifiedPath(f, fc2);
// Ensure file does not exist
@@ -374,6 +390,10 @@ public abstract class FileContextURIBase
"deleteTest/()&^%$#@!~_+}{><?", " ", "^ " };
for (String f : dirNames) {
+ if (!isTestableFileNameOnPlatform(f)) {
+ continue;
+ }
+
// Create a file on fc2's file system using fc1
Path testPath = qualifiedPath(f, fc2);
// Ensure file does not exist
@@ -492,6 +512,10 @@ public abstract class FileContextURIBase
ArrayList<Path> testDirs = new ArrayList<Path>();
for (String d : dirs) {
+ if (!isTestableFileNameOnPlatform(d)) {
+ continue;
+ }
+
testDirs.add(qualifiedPath(d, fc2));
}
Assert.assertFalse(exists(fc1, testDirs.get(0)));
@@ -506,15 +530,17 @@ public abstract class FileContextURIBase
Assert.assertEquals(qualifiedPath(hPrefix, fc1), paths[0].getPath());
paths = fc1.util().listStatus(qualifiedPath(hPrefix, fc1));
- Assert.assertEquals(6, paths.length);
- for (int i = 0; i < dirs.length; i++) {
+ Assert.assertEquals(testDirs.size(), paths.length);
+ for (int i = 0; i < testDirs.size(); i++) {
boolean found = false;
for (int j = 0; j < paths.length; j++) {
- if (qualifiedPath(dirs[i],fc1).equals(paths[j].getPath())) {
+ if (qualifiedPath(testDirs.get(i).toString(), fc1).equals(
+ paths[j].getPath())) {
+
found = true;
}
}
- Assert.assertTrue(dirs[i] + " not found", found);
+ Assert.assertTrue(testDirs.get(i) + " not found", found);
}
paths = fc1.util().listStatus(qualifiedPath(dirs[0], fc1));
@@ -539,9 +565,32 @@ public abstract class FileContextURIBase
}
Assert.assertTrue(stat.getPath() + " not found", found);
}
- Assert.assertEquals(6, dirLen);
+ Assert.assertEquals(testDirs.size(), dirLen);
pathsItor = fc1.listStatus(qualifiedPath(dirs[0], fc1));
Assert.assertFalse(pathsItor.hasNext());
}
+
+ /**
+ * Returns true if the argument is a file name that is testable on the platform
+ * currently running the test. This is intended for use by tests so that they
+ * can skip checking file names that aren't supported by the underlying
+ * platform. The current implementation specifically checks for patterns that
+ * are not valid file names on Windows when the tests are running on Windows.
+ *
+ * @param fileName String file name to check
+ * @return boolean true if the argument is valid as a file name
+ */
+ private static boolean isTestableFileNameOnPlatform(String fileName) {
+ boolean valid = true;
+
+ if (Shell.WINDOWS) {
+ // Disallow reserved characters: <, >, :, ", |, ?, *.
+ // Disallow trailing space or period.
+ // See http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
+ valid = !WIN_INVALID_FILE_NAME_PATTERN.matcher(fileName).matches();
+ }
+
+ return valid;
+ }
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java?rev=1404873&r1=1404872&r2=1404873&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java Fri Nov 2 04:27:27 2012
@@ -77,7 +77,7 @@ public final class FileSystemTestHelper
throws IOException {
// NOTE: can't cache because of different filesystems!
//if (absTestRootDir == null)
- if (TEST_ROOT_DIR.startsWith("/")) {
+ if (new Path(TEST_ROOT_DIR).isAbsolute()) {
absTestRootDir = TEST_ROOT_DIR;
} else {
absTestRootDir = fSys.getWorkingDirectory().toString() + "/"
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java?rev=1404873&r1=1404872&r2=1404873&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java Fri Nov 2 04:27:27 2012
@@ -25,6 +25,7 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.AvroTestUtil;
+import org.apache.hadoop.util.Shell;
import junit.framework.TestCase;
@@ -268,6 +269,8 @@ public class TestPath extends TestCase {
}
public void testGlobEscapeStatus() throws Exception {
+ // This test is not meaningful on Windows where * is disallowed in file name.
+ if (Shell.WINDOWS) return;
FileSystem lfs = FileSystem.getLocal(new Configuration());
Path testRoot = lfs.makeQualified(new Path(
System.getProperty("test.build.data","test/build/data"),