You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2013/10/17 20:54:54 UTC
svn commit: r1533208 - in
/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common: ./
src/main/java/ src/main/java/org/apache/hadoop/fs/shell/ src/site/apt/
src/test/java/org/apache/hadoop/fs/
src/test/java/org/apache/hadoop/fs/viewfs/
Author: arp
Date: Thu Oct 17 18:54:53 2013
New Revision: 1533208
URL: http://svn.apache.org/r1533208
Log:
Merging r1532911 through r1533207 from trunk to branch HDFS-2832
Modified:
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1533208&r1=1533207&r2=1533208&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt Thu Oct 17 18:54:53 2013
@@ -360,6 +360,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-9897. Add method to get path start position without drive specifier in
o.a.h.fs.Path. (Binglin Chang via cnauroth)
+ HADOOP-9078. enhance unit-test coverage of class
+ org.apache.hadoop.fs.FileContext (Ivan A. Veselovsky via jeagles)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
@@ -431,6 +434,9 @@ Release 2.2.1 - UNRELEASED
HADOOP-10040. hadoop.cmd in UNIX format and would not run by default on
Windows. (cnauroth)
+ HADOOP-10055. FileSystemShell.apt.vm doc has typo "numRepicas".
+ (Akira Ajisaka via cnauroth)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1532911-1533207
Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1532911-1533207
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java?rev=1533208&r1=1533207&r2=1533208&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java Thu Oct 17 18:54:53 2013
@@ -68,7 +68,7 @@ class SnapshotCommands extends FsCommand
throw new IllegalArgumentException("<snapshotDir> is missing.");
}
if (args.size() > 2) {
- throw new IllegalArgumentException("Too many arguements.");
+ throw new IllegalArgumentException("Too many arguments.");
}
if (args.size() == 2) {
snapshotName = args.removeLast();
@@ -110,7 +110,7 @@ class SnapshotCommands extends FsCommand
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
if (args.size() != 2) {
- throw new IOException("args number not 2: " + args.size());
+ throw new IllegalArgumentException("Incorrect number of arguments.");
}
snapshotName = args.removeLast();
}
@@ -150,7 +150,7 @@ class SnapshotCommands extends FsCommand
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
if (args.size() != 3) {
- throw new IOException("args number not 3: " + args.size());
+ throw new IllegalArgumentException("Incorrect number of arguments.");
}
newName = args.removeLast();
oldName = args.removeLast();
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm?rev=1533208&r1=1533207&r2=1533208&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm Thu Oct 17 18:54:53 2013
@@ -381,7 +381,7 @@ rmr
setrep
- Usage: <<<hdfs dfs -setrep [-R] [-w] <numRepicas> <path> >>>
+ Usage: <<<hdfs dfs -setrep [-R] [-w] <numReplicas> <path> >>>
Changes the replication factor of a file. If <path> is a directory then
the command recursively changes the replication factor of all files under
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java?rev=1533208&r1=1533207&r2=1533208&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java Thu Oct 17 18:54:53 2013
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.EnumSet;
+import java.util.NoSuchElementException;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -30,6 +31,7 @@ import org.apache.hadoop.fs.Options.Rena
import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.After;
import org.junit.Assert;
+import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.Test;
@@ -92,7 +94,7 @@ public abstract class FileContextMainOpe
}
};
- private static byte[] data = getFileData(numBlocks,
+ private static final byte[] data = getFileData(numBlocks,
getDefaultBlockSize());
@Before
@@ -107,7 +109,8 @@ public abstract class FileContextMainOpe
@After
public void tearDown() throws Exception {
- fc.delete(new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")), true);
+ boolean del = fc.delete(new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")), true);
+ assertTrue(del);
fc.delete(localFsRootPath, true);
}
@@ -194,6 +197,14 @@ public abstract class FileContextMainOpe
fc.setWorkingDirectory(absoluteDir);
Assert.assertEquals(absoluteDir, fc.getWorkingDirectory());
+ Path aRegularFile = new Path("aRegularFile");
+ createFile(aRegularFile);
+ try {
+ fc.setWorkingDirectory(aRegularFile);
+ fail("An IOException expected.");
+ } catch (IOException ioe) {
+ // okay
+ }
}
@Test
@@ -1195,6 +1206,136 @@ public abstract class FileContextMainOpe
return true;
}
return false;
+ }
+
+ @Test
+ public void testOpen2() throws IOException {
+ final Path rootPath = getTestRootPath(fc, "test");
+ //final Path rootPath = getAbsoluteTestRootPath(fc);
+ final Path path = new Path(rootPath, "zoo");
+ createFile(path);
+ final long length = fc.getFileStatus(path).getLen();
+ FSDataInputStream fsdis = fc.open(path, 2048);
+ try {
+ byte[] bb = new byte[(int)length];
+ fsdis.readFully(bb);
+ assertArrayEquals(data, bb);
+ } finally {
+ fsdis.close();
+ }
+ }
+
+ @Test
+ public void testSetVerifyChecksum() throws IOException {
+ final Path rootPath = getTestRootPath(fc, "test");
+ final Path path = new Path(rootPath, "zoo");
+
+ FSDataOutputStream out = fc.create(path, EnumSet.of(CREATE),
+ Options.CreateOpts.createParent());
+ try {
+ // instruct FS to verify checksum through the FileContext:
+ fc.setVerifyChecksum(true, path);
+ out.write(data, 0, data.length);
+ } finally {
+ out.close();
+ }
+
+ // NB: underlying FS may be different (this is an abstract test),
+ // so we cannot assert .zoo.crc existence.
+ // Instead, we check that the file is read correctly:
+ FileStatus fileStatus = fc.getFileStatus(path);
+ final long len = fileStatus.getLen();
+ assertTrue(len == data.length);
+ byte[] bb = new byte[(int)len];
+ FSDataInputStream fsdis = fc.open(path);
+ try {
+ fsdis.read(bb);
+ } finally {
+ fsdis.close();
+ }
+ assertArrayEquals(data, bb);
+ }
+
+ @Test
+ public void testListCorruptFileBlocks() throws IOException {
+ final Path rootPath = getTestRootPath(fc, "test");
+ final Path path = new Path(rootPath, "zoo");
+ createFile(path);
+ try {
+ final RemoteIterator<Path> remoteIterator = fc
+ .listCorruptFileBlocks(path);
+ if (listCorruptedBlocksSupported()) {
+ assertTrue(remoteIterator != null);
+ Path p;
+ while (remoteIterator.hasNext()) {
+ p = remoteIterator.next();
+ System.out.println("corrupted block: " + p);
+ }
+ try {
+ remoteIterator.next();
+ fail();
+ } catch (NoSuchElementException nsee) {
+ // okay
+ }
+ } else {
+ fail();
+ }
+ } catch (UnsupportedOperationException uoe) {
+ if (listCorruptedBlocksSupported()) {
+ fail(uoe.toString());
+ } else {
+ // okay
+ }
+ }
+ }
+
+ protected abstract boolean listCorruptedBlocksSupported();
+
+ @Test
+ public void testDeleteOnExitUnexisting() throws IOException {
+ final Path rootPath = getTestRootPath(fc, "test");
+ final Path path = new Path(rootPath, "zoo");
+ boolean registered = fc.deleteOnExit(path);
+ // because "zoo" does not exist:
+ assertTrue(!registered);
+ }
+
+ @Test
+ public void testFileContextStatistics() throws IOException {
+ FileContext.clearStatistics();
+
+ final Path rootPath = getTestRootPath(fc, "test");
+ final Path path = new Path(rootPath, "zoo");
+ createFile(path);
+ byte[] bb = new byte[data.length];
+ FSDataInputStream fsdis = fc.open(path);
+ try {
+ fsdis.read(bb);
+ } finally {
+ fsdis.close();
+ }
+ assertArrayEquals(data, bb);
+
+ FileContext.printStatistics();
+ }
+
+ @Test
+ /*
+ * Test method
+ * org.apache.hadoop.fs.FileContext.getFileContext(AbstractFileSystem)
+ */
+ public void testGetFileContext1() throws IOException {
+ final Path rootPath = getTestRootPath(fc, "test");
+ AbstractFileSystem asf = fc.getDefaultFileSystem();
+ // create FileContext using the protected #getFileContext(1) method:
+ FileContext fc2 = FileContext.getFileContext(asf);
+ // Now just check that this context can do something reasonable:
+ final Path path = new Path(rootPath, "zoo");
+ FSDataOutputStream out = fc2.create(path, EnumSet.of(CREATE),
+ Options.CreateOpts.createParent());
+ out.close();
+ Path pathResolved = fc2.resolvePath(path);
+ assertEquals(pathResolved.toUri().getPath(), path.toUri().getPath());
}
private Path getTestRootPath(FileContext fc, String pathString) {
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java?rev=1533208&r1=1533207&r2=1533208&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java Thu Oct 17 18:54:53 2013
@@ -35,6 +35,7 @@ import org.junit.Test;
import static org.apache.hadoop.fs.FileContextTestHelper.*;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
/**
* <p>
@@ -174,6 +175,13 @@ public abstract class FileContextPermiss
System.out.println("Not testing changing the group since user " +
"belongs to only one group.");
}
+
+ try {
+ fc.setOwner(f, null, null);
+ fail("Exception expected.");
+ } catch (IllegalArgumentException iae) {
+ // okay
+ }
}
finally {cleanupFile(fc, f);}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java?rev=1533208&r1=1533207&r2=1533208&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java Thu Oct 17 18:54:53 2013
@@ -49,6 +49,11 @@ public class TestLocalFSFileContextMainO
FileContext fc1 = FileContext.getLocalFSFileContext();
Assert.assertTrue(fc1 != fc);
}
+
+ @Override
+ protected boolean listCorruptedBlocksSupported() {
+ return false;
+ }
@Test
public void testDefaultFilePermission() throws IOException {
Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java?rev=1533208&r1=1533207&r2=1533208&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java Thu Oct 17 18:54:53 2013
@@ -18,13 +18,9 @@
package org.apache.hadoop.fs.viewfs;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
-import org.apache.hadoop.fs.FileContextTestHelper;
-import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.viewfs.ConfigUtil;
import org.junit.After;
import org.junit.Before;
@@ -49,4 +45,9 @@ public class TestFcMainOperationsLocalFs
super.tearDown();
ViewFsTestSetup.tearDownForViewFsLocalFs(fileContextTestHelper);
}
+
+ @Override
+ protected boolean listCorruptedBlocksSupported() {
+ return false;
+ }
}