You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:40:16 UTC
svn commit: r1077649 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
core/org/apache/hadoop/fs/ test/org/apache/hadoop/fs/
Author: omalley
Date: Fri Mar 4 04:40:16 2011
New Revision: 1077649
URL: http://svn.apache.org/viewvc?rev=1077649&view=rev
Log:
commit dd934cf4864f5a61f6421c69029fa792457b3002
Author: Suresh Srinivas <su...@yahoo-inc.com>
Date: Fri Aug 6 16:43:04 2010 -0700
HADOOP-6899 from https://issues.apache.org/jira/secure/attachment/12451477/6899.y20.patch
+++ b/YAHOO-CHANGES.txt
+ HADOOP-6899. RawLocalFileSystem set working directory does
+ not work for relative names. (suresh)
+
Added:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FSMainOperationsBaseTest.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FileSystemTestHelper.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java?rev=1077649&r1=1077648&r2=1077649&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java Fri Mar 4 04:40:16 2011
@@ -314,4 +314,42 @@ public class Path implements Comparable
}
return new Path(newUri);
}
+
+ /** Returns a qualified path object. */
+ public Path makeQualified(URI defaultUri, Path workingDir ) {
+ Path path = this;
+ if (!isAbsolute()) {
+ path = new Path(workingDir, this);
+ }
+
+ URI pathUri = path.toUri();
+
+ String scheme = pathUri.getScheme();
+ String authority = pathUri.getAuthority();
+ String fragment = pathUri.getFragment();
+
+ if (scheme != null &&
+ (authority != null || defaultUri.getAuthority() == null))
+ return path;
+
+ if (scheme == null) {
+ scheme = defaultUri.getScheme();
+ }
+
+ if (authority == null) {
+ authority = defaultUri.getAuthority();
+ if (authority == null) {
+ authority = "";
+ }
+ }
+
+ URI newUri = null;
+ try {
+ newUri = new URI(scheme, authority ,
+ normalizePath(pathUri.getPath()), null, fragment);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
+ }
+ return new Path(newUri);
+ }
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1077649&r1=1077648&r2=1077649&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/RawLocalFileSystem.java Fri Mar 4 04:40:16 2011
@@ -42,6 +42,14 @@ public class RawLocalFileSystem extends
workingDir = new Path(System.getProperty("user.dir")).makeQualified(this);
}
+ private Path makeAbsolute(Path f) {
+ if (f.isAbsolute()) {
+ return f;
+ } else {
+ return new Path(workingDir, f);
+ }
+ }
+
/** Convert a path to a File. */
public File pathToFile(Path path) {
checkPath(path);
@@ -321,7 +329,9 @@ public class RawLocalFileSystem extends
*/
@Override
public void setWorkingDirectory(Path newDir) {
- workingDir = newDir;
+ workingDir = makeAbsolute(newDir);
+ checkPath(workingDir);
+
}
@Override
Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FSMainOperationsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FSMainOperationsBaseTest.java?rev=1077649&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FSMainOperationsBaseTest.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FSMainOperationsBaseTest.java Fri Mar 4 04:40:16 2011
@@ -0,0 +1,738 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.hadoop.fs.FileSystemTestHelper.*;
+
+/**
+ * <p>
+ * A collection of tests for the {@link FileSystem}.
+ * This test should be used for testing an instance of FileSystem
+ * that has been initialized to a specific default FileSystem such a
+ * LocalFileSystem, HDFS,S3, etc.
+ * </p>
+ * <p>
+ * To test a given {@link FileSystem} implementation create a subclass of this
+ * test and override {@link #setUp()} to initialize the <code>fc</code>
+ * {@link FileSystem} instance variable.
+ *
+ * Since this a junit 4 you can also do a single setup before
+ * the start of any tests.
+ * E.g.
+ * @BeforeClass public static void clusterSetupAtBegining()
+ * @AfterClass public static void ClusterShutdownAtEnd()
+ * </p>
+ */
+public abstract class FSMainOperationsBaseTest {
+
+ private static String TEST_DIR_AAA2 = "test/hadoop2/aaa";
+ private static String TEST_DIR_AAA = "test/hadoop/aaa";
+ private static String TEST_DIR_AXA = "test/hadoop/axa";
+ private static String TEST_DIR_AXX = "test/hadoop/axx";
+ private static int numBlocks = 2;
+
+ static final String LOCAL_FS_ROOT_URI = "file:///tmp/test";
+
+
+ protected static FileSystem fSys;
+
+ final private static PathFilter DEFAULT_FILTER = new PathFilter() {
+ public boolean accept(final Path file) {
+ return true;
+ }
+ };
+
+ //A test filter with returns any path containing a "b"
+ final private static PathFilter TEST_X_FILTER = new PathFilter() {
+ public boolean accept(Path file) {
+ if(file.getName().contains("x") || file.toString().contains("X"))
+ return true;
+ else
+ return false;
+ }
+ };
+
+ private static byte[] data = getFileData(numBlocks,
+ getDefaultBlockSize());
+
+ @Before
+ public void setUp() throws Exception {
+ fSys.mkdirs(getTestRootPath(fSys, "test"));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fSys.delete(new Path(getAbsoluteTestRootPath(fSys), new Path("test")), true);
+ fSys.delete(new Path(LOCAL_FS_ROOT_URI), true);
+ }
+
+
+ protected Path getDefaultWorkingDirectory() throws IOException {
+ return getTestRootPath(fSys,
+ "/user/" + System.getProperty("user.name")).makeQualified(
+ fSys.getUri(), fSys.getWorkingDirectory());
+ }
+
+ protected boolean renameSupported() {
+ return true;
+ }
+
+
+ protected IOException unwrapException(IOException e) {
+ return e;
+ }
+
+ @Test
+ public void testWorkingDirectory() throws Exception {
+
+ // First we cd to our test root
+ Path workDir = new Path(getAbsoluteTestRootPath(fSys), new Path("test"));
+ fSys.setWorkingDirectory(workDir);
+ Assert.assertEquals(workDir, fSys.getWorkingDirectory());
+
+ fSys.setWorkingDirectory(new Path("."));
+ Assert.assertEquals(workDir, fSys.getWorkingDirectory());
+
+ fSys.setWorkingDirectory(new Path(".."));
+ Assert.assertEquals(workDir.getParent(), fSys.getWorkingDirectory());
+
+ // cd using a relative path
+
+ // Go back to our test root
+ workDir = new Path(getAbsoluteTestRootPath(fSys), new Path("test"));
+ fSys.setWorkingDirectory(workDir);
+ Assert.assertEquals(workDir, fSys.getWorkingDirectory());
+
+ Path relativeDir = new Path("existingDir1");
+ Path absoluteDir = new Path(workDir,"existingDir1");
+ fSys.mkdirs(absoluteDir);
+ fSys.setWorkingDirectory(relativeDir);
+ Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());
+ // cd using a absolute path
+ absoluteDir = getTestRootPath(fSys, "test/existingDir2");
+ fSys.mkdirs(absoluteDir);
+ fSys.setWorkingDirectory(absoluteDir);
+ Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());
+
+ // Now open a file relative to the wd we just set above.
+ Path absolutePath = new Path(absoluteDir, "foo");
+ FileSystemTestHelper.createFile(fSys, absolutePath);
+ fSys.open(new Path("foo")).close();
+
+
+ // Now mkdir relative to the dir we cd'ed to
+ fSys.mkdirs(new Path("newDir"));
+ Assert.assertTrue(isDir(fSys, new Path(absoluteDir, "newDir")));
+
+ /**
+ * We cannot test this because FileSystem has never checked for
+ * existence of working dir - fixing this would break compatibility,
+ *
+ absoluteDir = getTestRootPath(fc, "nonexistingPath");
+ try {
+ fc.setWorkingDirectory(absoluteDir);
+ Assert.fail("cd to non existing dir should have failed");
+ } catch (Exception e) {
+ // Exception as expected
+ }
+ */
+
+ // Try a URI
+
+ absoluteDir = new Path(LOCAL_FS_ROOT_URI + "/existingDir");
+ fSys.mkdirs(absoluteDir);
+ fSys.setWorkingDirectory(absoluteDir);
+ Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());
+
+ }
+
+ @Test
+ public void testMkdirs() throws Exception {
+ Path testDir = getTestRootPath(fSys, "test/hadoop");
+ Assert.assertFalse(exists(fSys, testDir));
+ Assert.assertFalse(isFile(fSys, testDir));
+
+ fSys.mkdirs(testDir);
+
+ Assert.assertTrue(exists(fSys, testDir));
+ Assert.assertFalse(isFile(fSys, testDir));
+
+ fSys.mkdirs(testDir);
+
+ Assert.assertTrue(exists(fSys, testDir));
+ Assert.assertFalse(isFile(fSys, testDir));
+
+ Path parentDir = testDir.getParent();
+ Assert.assertTrue(exists(fSys, parentDir));
+ Assert.assertFalse(isFile(fSys, parentDir));
+
+ Path grandparentDir = parentDir.getParent();
+ Assert.assertTrue(exists(fSys, grandparentDir));
+ Assert.assertFalse(isFile(fSys, grandparentDir));
+
+ }
+
+ @Test
+ public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
+ Path testDir = getTestRootPath(fSys, "test/hadoop");
+ Assert.assertFalse(exists(fSys, testDir));
+ fSys.mkdirs(testDir);
+ Assert.assertTrue(exists(fSys, testDir));
+
+ createFile(getTestRootPath(fSys, "test/hadoop/file"));
+
+ Path testSubDir = getTestRootPath(fSys, "test/hadoop/file/subdir");
+ Assert.assertFalse(fSys.mkdirs(testSubDir));
+ Assert.assertFalse(exists(fSys, testSubDir));
+
+ Path testDeepSubDir = getTestRootPath(fSys, "test/hadoop/file/deep/sub/dir");
+ Assert.assertFalse(exists(fSys, testSubDir));
+ Assert.assertFalse(fSys.mkdirs(testDeepSubDir));
+ Assert.assertFalse(exists(fSys, testDeepSubDir));
+
+ }
+
+ @Test
+ public void testGetFileStatusThrowsExceptionForNonExistentFile()
+ throws Exception {
+ try {
+ fSys.getFileStatus(getTestRootPath(fSys, "test/hadoop/file"));
+ Assert.fail("Should throw FileNotFoundException");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+
+ public void testListStatusThrowsExceptionForNonExistentFile()
+ throws Exception {
+ try {
+ fSys.listStatus(getTestRootPath(fSys, "test/hadoop/file"));
+ Assert.fail("Should throw FileNotFoundException");
+ } catch (FileNotFoundException fnfe) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testListStatus() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, "test/hadoop/a"),
+ getTestRootPath(fSys, "test/hadoop/b"),
+ getTestRootPath(fSys, "test/hadoop/c/1"), };
+ Assert.assertFalse(exists(fSys, testDirs[0]));
+
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+
+ // test listStatus that returns an array
+ FileStatus[] paths = fSys.listStatus(getTestRootPath(fSys, "test"));
+ Assert.assertEquals(1, paths.length);
+ Assert.assertEquals(getTestRootPath(fSys, "test/hadoop"), paths[0].getPath());
+
+ paths = fSys.listStatus(getTestRootPath(fSys, "test/hadoop"));
+ Assert.assertEquals(3, paths.length);
+
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/a"),
+ paths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/b"),
+ paths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/c"),
+ paths));
+
+ paths = fSys.listStatus(getTestRootPath(fSys, "test/hadoop/a"));
+ Assert.assertEquals(0, paths.length);
+
+ }
+
+ @Test
+ public void testListStatusFilterWithNoMatches() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA2),
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ // listStatus with filters returns empty correctly
+ FileStatus[] filteredPaths = fSys.listStatus(
+ getTestRootPath(fSys, "test"), TEST_X_FILTER);
+ Assert.assertEquals(0,filteredPaths.length);
+
+ }
+
+ public void testListStatusFilterWithSomeMatches() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AAA2), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ // should return 2 paths ("/test/hadoop/axa" and "/test/hadoop/axx")
+ FileStatus[] filteredPaths = fSys.listStatus(
+ getTestRootPath(fSys, "test/hadoop"), TEST_X_FILTER);
+ Assert.assertEquals(2,filteredPaths.length);
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AXA), filteredPaths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AXX), filteredPaths));
+ }
+
+ @Test
+ public void testGlobStatusWithNoMatchesInPath() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AAA2), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ // should return nothing
+ FileStatus[] paths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop/?"));
+ Assert.assertEquals(0, paths.length);
+ }
+
+ @Test
+ public void testGlobStatusSomeMatchesInDirectories() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AAA2), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ // Should return two items ("/test/hadoop" and "/test/hadoop2")
+ FileStatus[] paths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop*"));
+ Assert.assertEquals(2, paths.length);
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ "test/hadoop"), paths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ "test/hadoop2"), paths));
+ }
+
+ @Test
+ public void testGlobStatusWithMultipleWildCardMatches() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AAA2), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ //Should return all 4 items ("/test/hadoop/aaa", "/test/hadoop/axa"
+ //"/test/hadoop/axx", and "/test/hadoop2/axx")
+ FileStatus[] paths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop*/*"));
+ Assert.assertEquals(4, paths.length);
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA), paths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA), paths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX), paths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA2), paths));
+ }
+
+ @Test
+ public void testGlobStatusWithMultipleMatchesOfSingleChar() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AAA2), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ //Should return only 2 items ("/test/hadoop/axa", "/test/hadoop/axx")
+ FileStatus[] paths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop/ax?"));
+ Assert.assertEquals(2, paths.length);
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AXA), paths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AXX), paths));
+ }
+
+ @Test
+ public void testGlobStatusFilterWithEmptyPathResults() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AXX), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ //This should return an empty set
+ FileStatus[] filteredPaths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop/?"),
+ DEFAULT_FILTER);
+ Assert.assertEquals(0,filteredPaths.length);
+ }
+
+ @Test
+ public void testGlobStatusFilterWithSomePathMatchesAndTrivialFilter()
+ throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AXX), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ //This should return all three (aaa, axa, axx)
+ FileStatus[] filteredPaths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop/*"),
+ DEFAULT_FILTER);
+ Assert.assertEquals(3, filteredPaths.length);
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AAA), filteredPaths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AXA), filteredPaths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AXX), filteredPaths));
+ }
+
+ @Test
+ public void testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter()
+ throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AXX), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ //This should return all three (aaa, axa, axx)
+ FileStatus[] filteredPaths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop/a??"),
+ DEFAULT_FILTER);
+ Assert.assertEquals(3, filteredPaths.length);
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA),
+ filteredPaths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA),
+ filteredPaths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX),
+ filteredPaths));
+ }
+
+ @Test
+ public void testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter()
+ throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AXX), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ //This should return two (axa, axx)
+ FileStatus[] filteredPaths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop/*"),
+ TEST_X_FILTER);
+ Assert.assertEquals(2, filteredPaths.length);
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AXA), filteredPaths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys,
+ TEST_DIR_AXX), filteredPaths));
+ }
+
+ @Test
+ public void testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter()
+ throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AXX), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ //This should return an empty set
+ FileStatus[] filteredPaths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop/?"),
+ TEST_X_FILTER);
+ Assert.assertEquals(0,filteredPaths.length);
+ }
+
+ @Test
+ public void testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter()
+ throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fSys, TEST_DIR_AAA),
+ getTestRootPath(fSys, TEST_DIR_AXA),
+ getTestRootPath(fSys, TEST_DIR_AXX),
+ getTestRootPath(fSys, TEST_DIR_AXX), };
+
+ if (exists(fSys, testDirs[0]) == false) {
+ for (Path path : testDirs) {
+ fSys.mkdirs(path);
+ }
+ }
+
+ //This should return two (axa, axx)
+ FileStatus[] filteredPaths = fSys.globStatus(
+ getTestRootPath(fSys, "test/hadoop/a??"),
+ TEST_X_FILTER);
+ Assert.assertEquals(2, filteredPaths.length);
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA),
+ filteredPaths));
+ Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX),
+ filteredPaths));
+ }
+
+ @Test
+ public void testWriteReadAndDeleteEmptyFile() throws Exception {
+ writeReadAndDelete(0);
+ }
+
+ @Test
+ public void testWriteReadAndDeleteHalfABlock() throws Exception {
+ writeReadAndDelete(getDefaultBlockSize() / 2);
+ }
+
+ @Test
+ public void testWriteReadAndDeleteOneBlock() throws Exception {
+ writeReadAndDelete(getDefaultBlockSize());
+ }
+
+ @Test
+ public void testWriteReadAndDeleteOneAndAHalfBlocks() throws Exception {
+ int blockSize = getDefaultBlockSize();
+ writeReadAndDelete(blockSize + (blockSize / 2));
+ }
+
+ @Test
+ public void testWriteReadAndDeleteTwoBlocks() throws Exception {
+ writeReadAndDelete(getDefaultBlockSize() * 2);
+ }
+
+ private void writeReadAndDelete(int len) throws IOException {
+ Path path = getTestRootPath(fSys, "test/hadoop/file");
+
+ fSys.mkdirs(path.getParent());
+
+
+ FSDataOutputStream out =
+ fSys.create(path, false, 4096, (short) 1, getDefaultBlockSize() );
+ out.write(data, 0, len);
+ out.close();
+
+ Assert.assertTrue("Exists", exists(fSys, path));
+ Assert.assertEquals("Length", len, fSys.getFileStatus(path).getLen());
+
+ FSDataInputStream in = fSys.open(path);
+ byte[] buf = new byte[len];
+ in.readFully(0, buf);
+ in.close();
+
+ Assert.assertEquals(len, buf.length);
+ for (int i = 0; i < buf.length; i++) {
+ Assert.assertEquals("Position " + i, data[i], buf[i]);
+ }
+
+ Assert.assertTrue("Deleted", fSys.delete(path, false));
+
+ Assert.assertFalse("No longer exists", exists(fSys, path));
+
+ }
+
+ @Test
+ public void testOverwrite() throws IOException {
+ Path path = getTestRootPath(fSys, "test/hadoop/file");
+
+ fSys.mkdirs(path.getParent());
+
+ createFile(path);
+
+ Assert.assertTrue("Exists", exists(fSys, path));
+ Assert.assertEquals("Length", data.length, fSys.getFileStatus(path).getLen());
+
+ try {
+ createFile(path);
+ Assert.fail("Should throw IOException.");
+ } catch (IOException e) {
+ // Expected
+ }
+
+ FSDataOutputStream out = fSys.create(path, true, 4096);
+ out.write(data, 0, data.length);
+ out.close();
+
+ Assert.assertTrue("Exists", exists(fSys, path));
+ Assert.assertEquals("Length", data.length, fSys.getFileStatus(path).getLen());
+
+ }
+
+ @Test
+ public void testWriteInNonExistentDirectory() throws IOException {
+ Path path = getTestRootPath(fSys, "test/hadoop/file");
+ Assert.assertFalse("Parent doesn't exist", exists(fSys, path.getParent()));
+ createFile(path);
+
+ Assert.assertTrue("Exists", exists(fSys, path));
+ Assert.assertEquals("Length", data.length, fSys.getFileStatus(path).getLen());
+ Assert.assertTrue("Parent exists", exists(fSys, path.getParent()));
+ }
+
+ @Test
+ public void testDeleteNonExistentFile() throws IOException {
+ Path path = getTestRootPath(fSys, "test/hadoop/file");
+ Assert.assertFalse("Doesn't exist", exists(fSys, path));
+ Assert.assertFalse("No deletion", fSys.delete(path, true));
+ }
+
+ @Test
+ public void testDeleteRecursively() throws IOException {
+ Path dir = getTestRootPath(fSys, "test/hadoop");
+ Path file = getTestRootPath(fSys, "test/hadoop/file");
+ Path subdir = getTestRootPath(fSys, "test/hadoop/subdir");
+
+ createFile(file);
+ fSys.mkdirs(subdir);
+
+ Assert.assertTrue("File exists", exists(fSys, file));
+ Assert.assertTrue("Dir exists", exists(fSys, dir));
+ Assert.assertTrue("Subdir exists", exists(fSys, subdir));
+
+ try {
+ fSys.delete(dir, false);
+ Assert.fail("Should throw IOException.");
+ } catch (IOException e) {
+ // expected
+ }
+ Assert.assertTrue("File still exists", exists(fSys, file));
+ Assert.assertTrue("Dir still exists", exists(fSys, dir));
+ Assert.assertTrue("Subdir still exists", exists(fSys, subdir));
+
+ Assert.assertTrue("Deleted", fSys.delete(dir, true));
+ Assert.assertFalse("File doesn't exist", exists(fSys, file));
+ Assert.assertFalse("Dir doesn't exist", exists(fSys, dir));
+ Assert.assertFalse("Subdir doesn't exist", exists(fSys, subdir));
+ }
+
+ @Test
+ public void testDeleteEmptyDirectory() throws IOException {
+ Path dir = getTestRootPath(fSys, "test/hadoop");
+ fSys.mkdirs(dir);
+ Assert.assertTrue("Dir exists", exists(fSys, dir));
+ Assert.assertTrue("Deleted", fSys.delete(dir, false));
+ Assert.assertFalse("Dir doesn't exist", exists(fSys, dir));
+ }
+
+ @Test
+ public void testInputStreamClosedTwice() throws IOException {
+ //HADOOP-4760 according to Closeable#close() closing already-closed
+ //streams should have no effect.
+ Path src = getTestRootPath(fSys, "test/hadoop/file");
+ createFile(src);
+ FSDataInputStream in = fSys.open(src);
+ in.close();
+ in.close();
+ }
+
+ @Test
+ public void testOutputStreamClosedTwice() throws IOException {
+ //HADOOP-4760 according to Closeable#close() closing already-closed
+ //streams should have no effect.
+ Path src = getTestRootPath(fSys, "test/hadoop/file");
+ FSDataOutputStream out = fSys.create(src);
+
+ out.writeChar('H'); //write some data
+ out.close();
+ out.close();
+ }
+
+
+ protected void createFile(Path path) throws IOException {
+ FileSystemTestHelper.createFile(fSys, path);
+ }
+
+ private boolean containsPath(Path path, FileStatus[] filteredPaths)
+ throws IOException {
+ for(int i = 0; i < filteredPaths.length; i ++) {
+ if (getTestRootPath(fSys, path.toString()).equals(
+ filteredPaths[i].getPath()))
+ return true;
+ }
+ return false;
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FileSystemTestHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FileSystemTestHelper.java?rev=1077649&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FileSystemTestHelper.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/FileSystemTestHelper.java Fri Mar 4 04:40:16 2011
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.FileNotFoundException;
+
+
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Assert;
+
+
+/**
+ * Helper class for unit tests.
+ */
+public final class FileSystemTestHelper {
+ // The test root is relative to the <wd>/build/test/data by default
+ public static final String TEST_ROOT_DIR =
+ System.getProperty("test.build.data", "build/test/data") + "/test";
+ private static final int DEFAULT_BLOCK_SIZE = 1024;
+ private static final int DEFAULT_NUM_BLOCKS = 2;
+ private static String absTestRootDir = null;
+
+ /** Hidden constructor */
+ private FileSystemTestHelper() {}
+
+ public static int getDefaultBlockSize() {
+ return DEFAULT_BLOCK_SIZE;
+ }
+
+ public static byte[] getFileData(int numOfBlocks, long blockSize) {
+ byte[] data = new byte[(int) (numOfBlocks * blockSize)];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = (byte) (i % 10);
+ }
+ return data;
+ }
+
+ public static Path getTestRootPath(FileSystem fSys) {
+ return fSys.makeQualified(new Path(TEST_ROOT_DIR));
+ }
+
+ public static Path getTestRootPath(FileSystem fSys, String pathString) {
+ return fSys.makeQualified(new Path(TEST_ROOT_DIR, pathString));
+ }
+
+
+ // the getAbsolutexxx method is needed because the root test dir
+ // can be messed up by changing the working dir.
+
+ public static String getAbsoluteTestRootDir(FileSystem fSys)
+ throws IOException {
+ if (absTestRootDir == null) {
+ if (TEST_ROOT_DIR.startsWith("/")) {
+ absTestRootDir = TEST_ROOT_DIR;
+ } else {
+ absTestRootDir = fSys.getWorkingDirectory().toString() + "/"
+ + TEST_ROOT_DIR;
+ }
+ }
+ return absTestRootDir;
+ }
+
+ public static Path getAbsoluteTestRootPath(FileSystem fSys) throws IOException {
+ return fSys.makeQualified(new Path(getAbsoluteTestRootDir(fSys)));
+ }
+
+ public static Path getDefaultWorkingDirectory(FileSystem fSys)
+ throws IOException {
+ return getTestRootPath(fSys, "/user/" + System.getProperty("user.name"))
+ .makeQualified(fSys.getUri(),
+ fSys.getWorkingDirectory());
+ }
+
+ /*
+ * Create files with numBlocks blocks each with block size blockSize.
+ */
+ public static void createFile(FileSystem fSys, Path path, int numBlocks,
+ int blockSize, boolean createParent) throws IOException {
+ FSDataOutputStream out =
+ fSys.create(path, false, 4096, fSys.getDefaultReplication(), blockSize );
+
+ byte[] data = getFileData(numBlocks, blockSize);
+ out.write(data, 0, data.length);
+ out.close();
+ }
+
+ public static void createFile(FileSystem fSys, Path path, int numBlocks,
+ int blockSize) throws IOException {
+ createFile(fSys, path, numBlocks, blockSize, true);
+ }
+
+ public static void createFile(FileSystem fSys, Path path) throws IOException {
+ createFile(fSys, path, DEFAULT_NUM_BLOCKS, DEFAULT_BLOCK_SIZE, true);
+ }
+
+ public static Path createFile(FileSystem fSys, String name) throws IOException {
+ Path path = getTestRootPath(fSys, name);
+ createFile(fSys, path);
+ return path;
+ }
+
+ public static boolean exists(FileSystem fSys, Path p) throws IOException {
+ return fSys.exists(p);
+ }
+
+ public static boolean isFile(FileSystem fSys, Path p) throws IOException {
+ try {
+ return !fSys.getFileStatus(p).isDir();
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+ public static boolean isDir(FileSystem fSys, Path p) throws IOException {
+ try {
+ return fSys.getFileStatus(p).isDir();
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+
+ public static void writeFile(FileSystem fSys, Path path,byte b[])
+ throws Exception {
+ FSDataOutputStream out =
+ fSys.create(path);
+ out.write(b);
+ out.close();
+ }
+
+ public static byte[] readFile(FileSystem fSys, Path path, int len )
+ throws Exception {
+ DataInputStream dis = fSys.open(path);
+ byte[] buffer = new byte[len];
+ IOUtils.readFully(dis, buffer, 0, len);
+ dis.close();
+ return buffer;
+ }
+ public static FileStatus containsPath(FileSystem fSys, Path path,
+ FileStatus[] dirList)
+ throws IOException {
+ for(int i = 0; i < dirList.length; i ++) {
+ if (getTestRootPath(fSys, path.toString()).equals(
+ dirList[i].getPath()))
+ return dirList[i];
+ }
+ return null;
+ }
+
+ public static FileStatus containsPath(Path path,
+ FileStatus[] dirList)
+ throws IOException {
+ for(int i = 0; i < dirList.length; i ++) {
+ if (path.equals(dirList[i].getPath()))
+ return dirList[i];
+ }
+ return null;
+ }
+
+
+ public static FileStatus containsPath(FileSystem fSys, String path, FileStatus[] dirList)
+ throws IOException {
+ return containsPath(fSys, new Path(path), dirList);
+ }
+
+ public static enum fileType {isDir, isFile, isSymlink};
+ public static void checkFileStatus(FileSystem aFs, String path,
+ fileType expectedType) throws IOException {
+ FileStatus s = aFs.getFileStatus(new Path(path));
+ Assert.assertNotNull(s);
+ if (expectedType == fileType.isDir) {
+ Assert.assertTrue(s.isDir());
+ } else if (expectedType == fileType.isFile) {
+ Assert.assertTrue(!s.isDir());
+ }
+ Assert.assertEquals(aFs.makeQualified(new Path(path)), s.getPath());
+ }
+}
Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java?rev=1077649&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java Fri Mar 4 04:40:16 2011
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Before;
+
+public class TestFSMainOperationsLocalFileSystem extends
+ FSMainOperationsBaseTest {
+
+ @Before
+ public void setUp() throws Exception {
+ fSys = FileSystem.getLocal(new Configuration());
+ super.setUp();
+ }
+
+ static Path wd = null;
+ protected Path getDefaultWorkingDirectory() throws IOException {
+ if (wd == null)
+ wd = FileSystem.getLocal(new Configuration()).getWorkingDirectory();
+ return wd;
+ }
+}