You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:50:25 UTC
svn commit: r1619012 [5/35] - in
/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/ hadoop-...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java Tue Aug 19 23:49:39 2014
@@ -17,15 +17,602 @@
*/
package org.apache.hadoop.hdfs.nfs.nfs3;
-import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import org.jboss.netty.channel.Channel;
+import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.nfs.nfs3.FileHandle;
+import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
+import org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow;
+import org.apache.hadoop.nfs.nfs3.Nfs3Status;
+import org.apache.hadoop.nfs.nfs3.request.LOOKUP3Request;
+import org.apache.hadoop.nfs.nfs3.request.READ3Request;
+import org.apache.hadoop.nfs.nfs3.request.WRITE3Request;
+import org.apache.hadoop.nfs.nfs3.response.ACCESS3Response;
+import org.apache.hadoop.nfs.nfs3.response.COMMIT3Response;
+import org.apache.hadoop.nfs.nfs3.response.CREATE3Response;
+import org.apache.hadoop.nfs.nfs3.response.FSSTAT3Response;
+import org.apache.hadoop.nfs.nfs3.response.FSINFO3Response;
+import org.apache.hadoop.nfs.nfs3.response.GETATTR3Response;
+import org.apache.hadoop.nfs.nfs3.response.LOOKUP3Response;
+import org.apache.hadoop.nfs.nfs3.response.PATHCONF3Response;
+import org.apache.hadoop.nfs.nfs3.response.READ3Response;
+import org.apache.hadoop.nfs.nfs3.response.REMOVE3Response;
+import org.apache.hadoop.nfs.nfs3.response.RMDIR3Response;
+import org.apache.hadoop.nfs.nfs3.response.RENAME3Response;
+import org.apache.hadoop.nfs.nfs3.response.READDIR3Response;
+import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response;
+import org.apache.hadoop.nfs.nfs3.response.READLINK3Response;
+import org.apache.hadoop.nfs.nfs3.response.SETATTR3Response;
+import org.apache.hadoop.nfs.nfs3.response.SYMLINK3Response;
+import org.apache.hadoop.nfs.nfs3.response.WRITE3Response;
+import org.apache.hadoop.nfs.nfs3.request.SetAttr3;
+import org.apache.hadoop.oncrpc.XDR;
+import org.apache.hadoop.oncrpc.security.SecurityHandler;
+import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
+import org.apache.hadoop.security.authorize.ProxyUsers;
/**
* Tests for {@link RpcProgramNfs3}
*/
public class TestRpcProgramNfs3 {
+ static DistributedFileSystem hdfs;
+ static MiniDFSCluster cluster = null;
+ static NfsConfiguration config = new NfsConfiguration();
+ static NameNode nn;
+ static Nfs3 nfs;
+ static RpcProgramNfs3 nfsd;
+ static SecurityHandler securityHandler;
+ static SecurityHandler securityHandlerUnpriviledged;
+ static String testdir = "/tmp";
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ String currentUser = System.getProperty("user.name");
+
+ config.set("fs.permissions.umask-mode", "u=rwx,g=,o=");
+ config.set(DefaultImpersonationProvider.getTestProvider()
+ .getProxySuperuserGroupConfKey(currentUser), "*");
+ config.set(DefaultImpersonationProvider.getTestProvider()
+ .getProxySuperuserIpConfKey(currentUser), "*");
+ ProxyUsers.refreshSuperUserGroupsConfiguration(config);
+
+ cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
+ cluster.waitActive();
+ hdfs = cluster.getFileSystem();
+ nn = cluster.getNameNode();
+
+ // Use ephemeral ports in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
+
+ // Start NFS with allowed.hosts set to "* rw"
+ config.set("dfs.nfs.exports.allowed.hosts", "* rw");
+ nfs = new Nfs3(config);
+ nfs.startServiceInternal(false);
+ nfsd = (RpcProgramNfs3) nfs.getRpcProgram();
+
+
+ // Mock SecurityHandler which returns system user.name
+ securityHandler = Mockito.mock(SecurityHandler.class);
+ Mockito.when(securityHandler.getUser()).thenReturn(currentUser);
+
+ // Mock SecurityHandler which returns a dummy username "harry"
+ securityHandlerUnpriviledged = Mockito.mock(SecurityHandler.class);
+ Mockito.when(securityHandlerUnpriviledged.getUser()).thenReturn("harry");
+ }
+
+ @AfterClass
+ public static void shutdown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Before
+ public void createFiles() throws IllegalArgumentException, IOException {
+ hdfs.delete(new Path(testdir), true);
+ hdfs.mkdirs(new Path(testdir));
+ hdfs.mkdirs(new Path(testdir + "/foo"));
+ DFSTestUtil.createFile(hdfs, new Path(testdir + "/bar"), 0, (short) 1, 0);
+ }
+
+ @Test(timeout = 60000)
+ public void testGetattr() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ XDR xdr_req = new XDR();
+ handle.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ GETATTR3Response response1 = nfsd.getattr(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ GETATTR3Response response2 = nfsd.getattr(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testSetattr() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ XDR xdr_req = new XDR();
+ FileHandle handle = new FileHandle(dirId);
+ handle.serialize(xdr_req);
+ xdr_req.writeString("bar");
+ SetAttr3 symAttr = new SetAttr3();
+ symAttr.serialize(xdr_req);
+ xdr_req.writeBoolean(false);
+
+ // Attempt by an unpriviledged user should fail.
+ SETATTR3Response response1 = nfsd.setattr(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ SETATTR3Response response2 = nfsd.setattr(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testLookup() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ LOOKUP3Request lookupReq = new LOOKUP3Request(handle, "bar");
+ XDR xdr_req = new XDR();
+ lookupReq.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ LOOKUP3Response response1 = nfsd.lookup(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ LOOKUP3Response response2 = nfsd.lookup(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testAccess() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ XDR xdr_req = new XDR();
+ handle.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ ACCESS3Response response1 = nfsd.access(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ ACCESS3Response response2 = nfsd.access(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testReadlink() throws Exception {
+ // Create a symlink first.
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ XDR xdr_req = new XDR();
+ FileHandle handle = new FileHandle(dirId);
+ handle.serialize(xdr_req);
+ xdr_req.writeString("fubar");
+ SetAttr3 symAttr = new SetAttr3();
+ symAttr.serialize(xdr_req);
+ xdr_req.writeString("bar");
+
+ SYMLINK3Response response = nfsd.symlink(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response.getStatus());
+
+ // Now perform readlink operations.
+ FileHandle handle2 = response.getObjFileHandle();
+ XDR xdr_req2 = new XDR();
+ handle2.serialize(xdr_req2);
+
+ // Attempt by an unpriviledged user should fail.
+ READLINK3Response response1 = nfsd.readlink(xdr_req2.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ READLINK3Response response2 = nfsd.readlink(xdr_req2.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testRead() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+
+ READ3Request readReq = new READ3Request(handle, 0, 5);
+ XDR xdr_req = new XDR();
+ readReq.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ READ3Response response1 = nfsd.read(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ READ3Response response2 = nfsd.read(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testWrite() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+
+ byte[] buffer = new byte[10];
+ for (int i = 0; i < 10; i++) {
+ buffer[i] = (byte) i;
+ }
+
+ WRITE3Request writeReq = new WRITE3Request(handle, 0, 10,
+ WriteStableHow.DATA_SYNC, ByteBuffer.wrap(buffer));
+ XDR xdr_req = new XDR();
+ writeReq.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ WRITE3Response response1 = nfsd.write(xdr_req.asReadOnlyWrap(),
+ null, 1, securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ WRITE3Response response2 = nfsd.write(xdr_req.asReadOnlyWrap(),
+ null, 1, securityHandler,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect response:", null, response2);
+ }
+
+ @Test(timeout = 60000)
+ public void testCreate() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ XDR xdr_req = new XDR();
+ FileHandle handle = new FileHandle(dirId);
+ handle.serialize(xdr_req);
+ xdr_req.writeString("fubar");
+ xdr_req.writeInt(Nfs3Constant.CREATE_UNCHECKED);
+ SetAttr3 symAttr = new SetAttr3();
+ symAttr.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ CREATE3Response response1 = nfsd.create(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ CREATE3Response response2 = nfsd.create(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testMkdir() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ XDR xdr_req = new XDR();
+ FileHandle handle = new FileHandle(dirId);
+ handle.serialize(xdr_req);
+ xdr_req.writeString("fubar");
+ SetAttr3 symAttr = new SetAttr3();
+ symAttr.serialize(xdr_req);
+ xdr_req.writeString("bar");
+
+ // Attempt to remove by an unpriviledged user should fail.
+ SYMLINK3Response response1 = nfsd.symlink(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt to remove by a priviledged user should pass.
+ SYMLINK3Response response2 = nfsd.symlink(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testSymlink() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ XDR xdr_req = new XDR();
+ FileHandle handle = new FileHandle(dirId);
+ handle.serialize(xdr_req);
+ xdr_req.writeString("fubar");
+ SetAttr3 symAttr = new SetAttr3();
+ symAttr.serialize(xdr_req);
+ xdr_req.writeString("bar");
+
+ // Attempt by an unpriviledged user should fail.
+ SYMLINK3Response response1 = nfsd.symlink(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ SYMLINK3Response response2 = nfsd.symlink(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testRemove() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ XDR xdr_req = new XDR();
+ FileHandle handle = new FileHandle(dirId);
+ handle.serialize(xdr_req);
+ xdr_req.writeString("bar");
+
+ // Attempt by an unpriviledged user should fail.
+ REMOVE3Response response1 = nfsd.remove(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ REMOVE3Response response2 = nfsd.remove(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testRmdir() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ XDR xdr_req = new XDR();
+ FileHandle handle = new FileHandle(dirId);
+ handle.serialize(xdr_req);
+ xdr_req.writeString("foo");
+
+ // Attempt by an unpriviledged user should fail.
+ RMDIR3Response response1 = nfsd.rmdir(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ RMDIR3Response response2 = nfsd.rmdir(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testRename() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ XDR xdr_req = new XDR();
+ FileHandle handle = new FileHandle(dirId);
+ handle.serialize(xdr_req);
+ xdr_req.writeString("bar");
+ handle.serialize(xdr_req);
+ xdr_req.writeString("fubar");
+
+ // Attempt by an unpriviledged user should fail.
+ RENAME3Response response1 = nfsd.rename(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ RENAME3Response response2 = nfsd.rename(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testReaddir() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ XDR xdr_req = new XDR();
+ handle.serialize(xdr_req);
+ xdr_req.writeLongAsHyper(0);
+ xdr_req.writeLongAsHyper(0);
+ xdr_req.writeInt(100);
+
+ // Attempt by an unpriviledged user should fail.
+ READDIR3Response response1 = nfsd.readdir(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ READDIR3Response response2 = nfsd.readdir(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testReaddirplus() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ XDR xdr_req = new XDR();
+ handle.serialize(xdr_req);
+ xdr_req.writeLongAsHyper(0);
+ xdr_req.writeLongAsHyper(0);
+ xdr_req.writeInt(3);
+ xdr_req.writeInt(2);
+
+ // Attempt by an unpriviledged user should fail.
+ READDIRPLUS3Response response1 = nfsd.readdirplus(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ READDIRPLUS3Response response2 = nfsd.readdirplus(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testFsstat() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ XDR xdr_req = new XDR();
+ handle.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ FSSTAT3Response response1 = nfsd.fsstat(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ FSSTAT3Response response2 = nfsd.fsstat(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testFsinfo() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ XDR xdr_req = new XDR();
+ handle.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ FSINFO3Response response1 = nfsd.fsinfo(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ FSINFO3Response response2 = nfsd.fsinfo(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testPathconf() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ XDR xdr_req = new XDR();
+ handle.serialize(xdr_req);
+
+ // Attempt by an unpriviledged user should fail.
+ PATHCONF3Response response1 = nfsd.pathconf(xdr_req.asReadOnlyWrap(),
+ securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ PATHCONF3Response response2 = nfsd.pathconf(xdr_req.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3_OK,
+ response2.getStatus());
+ }
+
+ @Test(timeout = 60000)
+ public void testCommit() throws Exception {
+ HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
+ long dirId = status.getFileId();
+ FileHandle handle = new FileHandle(dirId);
+ XDR xdr_req = new XDR();
+ handle.serialize(xdr_req);
+ xdr_req.writeLongAsHyper(0);
+ xdr_req.writeInt(5);
+
+ Channel ch = Mockito.mock(Channel.class);
+
+ // Attempt by an unpriviledged user should fail.
+ COMMIT3Response response1 = nfsd.commit(xdr_req.asReadOnlyWrap(),
+ ch, 1, securityHandlerUnpriviledged,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect return code:", Nfs3Status.NFS3ERR_ACCES,
+ response1.getStatus());
+
+ // Attempt by a priviledged user should pass.
+ COMMIT3Response response2 = nfsd.commit(xdr_req.asReadOnlyWrap(),
+ ch, 1, securityHandler,
+ new InetSocketAddress("localhost", 1234));
+ assertEquals("Incorrect COMMIT3Response:", null, response2);
+ }
+
@Test(timeout=1000)
public void testIdempotent() {
Object[][] procedures = {
@@ -63,4 +650,41 @@ public class TestRpcProgramNfs3 {
}
}
}
+
+ @Test
+ public void testDeprecatedKeys() {
+ NfsConfiguration conf = new NfsConfiguration();
+ conf.setInt("nfs3.server.port", 998);
+ assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, 0) == 998);
+
+ conf.setInt("nfs3.mountd.port", 999);
+ assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY, 0) == 999);
+
+ conf.set("dfs.nfs.exports.allowed.hosts", "host1");
+ assertTrue(conf.get(CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
+ .equals("host1"));
+
+ conf.setInt("dfs.nfs.exports.cache.expirytime.millis", 1000);
+ assertTrue(conf.getInt(
+ Nfs3Constant.NFS_EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY, 0) == 1000);
+
+ conf.setInt("hadoop.nfs.userupdate.milly", 10);
+ assertTrue(conf.getInt(Nfs3Constant.NFS_USERGROUP_UPDATE_MILLIS_KEY, 0) == 10);
+
+ conf.set("dfs.nfs3.dump.dir", "/nfs/tmp");
+ assertTrue(conf.get(NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_KEY).equals(
+ "/nfs/tmp"));
+
+ conf.setBoolean("dfs.nfs3.enableDump", false);
+ assertTrue(conf.getBoolean(NfsConfigKeys.DFS_NFS_FILE_DUMP_KEY, true) == false);
+
+ conf.setInt("dfs.nfs3.max.open.files", 500);
+ assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_MAX_OPEN_FILES_KEY, 0) == 500);
+
+ conf.setInt("dfs.nfs3.stream.timeout", 6000);
+ assertTrue(conf.getInt(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_KEY, 0) == 6000);
+
+ conf.set("dfs.nfs3.export.point", "/dir1");
+ assertTrue(conf.get(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY).equals("/dir1"));
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java Tue Aug 19 23:49:39 2014
@@ -22,16 +22,16 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
-import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.concurrent.ConcurrentNavigableMap;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.COMMIT_STATUS;
import org.apache.hadoop.hdfs.nfs.nfs3.OpenFileCtx.CommitCtx;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -50,6 +50,7 @@ import org.apache.hadoop.nfs.nfs3.respon
import org.apache.hadoop.nfs.nfs3.response.READ3Response;
import org.apache.hadoop.oncrpc.XDR;
import org.apache.hadoop.oncrpc.security.SecurityHandler;
+import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.jboss.netty.channel.Channel;
import org.junit.Assert;
@@ -138,7 +139,7 @@ public class TestWrites {
Mockito.when(fos.getPos()).thenReturn((long) 0);
OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
- new IdUserGroup());
+ new IdUserGroup(new NfsConfiguration()));
COMMIT_STATUS ret;
@@ -190,6 +191,29 @@ public class TestWrites {
ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
}
+
+ @Test
+ public void testCheckCommitAixCompatMode() throws IOException {
+ DFSClient dfsClient = Mockito.mock(DFSClient.class);
+ Nfs3FileAttributes attr = new Nfs3FileAttributes();
+ HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
+
+ // Last argument "true" here to enable AIX compatibility mode.
+ OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
+ new IdUserGroup(new NfsConfiguration()), true);
+
+ // Test fall-through to pendingWrites check in the event that commitOffset
+ // is greater than the number of bytes we've so far flushed.
+ Mockito.when(fos.getPos()).thenReturn((long) 2);
+ COMMIT_STATUS status = ctx.checkCommitInternal(5, null, 1, attr, false);
+ Assert.assertTrue(status == COMMIT_STATUS.COMMIT_FINISHED);
+
+ // Test the case when we actually have received more bytes than we're trying
+ // to commit.
+ Mockito.when(fos.getPos()).thenReturn((long) 10);
+ status = ctx.checkCommitInternal(5, null, 1, attr, false);
+ Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
+ }
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which
@@ -200,13 +224,14 @@ public class TestWrites {
Nfs3FileAttributes attr = new Nfs3FileAttributes();
HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
Mockito.when(fos.getPos()).thenReturn((long) 0);
+ NfsConfiguration config = new NfsConfiguration();
OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
- new IdUserGroup());
+ new IdUserGroup(config));
FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath"
COMMIT_STATUS ret;
- WriteManager wm = new WriteManager(new IdUserGroup(), new Configuration());
+ WriteManager wm = new WriteManager(new IdUserGroup(config), config, false);
assertTrue(wm.addOpenFileStream(h, ctx));
// Test inactive open file context
@@ -279,7 +304,7 @@ public class TestWrites {
@Test
public void testWriteStableHow() throws IOException, InterruptedException {
- HdfsConfiguration config = new HdfsConfiguration();
+ NfsConfiguration config = new NfsConfiguration();
DFSClient client = null;
MiniDFSCluster cluster = null;
RpcProgramNfs3 nfsd;
@@ -288,10 +313,12 @@ public class TestWrites {
System.getProperty("user.name"));
String currentUser = System.getProperty("user.name");
config.set(
- ProxyUsers.getProxySuperuserGroupConfKey(currentUser),
+ DefaultImpersonationProvider.getTestProvider().
+ getProxySuperuserGroupConfKey(currentUser),
"*");
config.set(
- ProxyUsers.getProxySuperuserIpConfKey(currentUser),
+ DefaultImpersonationProvider.getTestProvider().
+ getProxySuperuserIpConfKey(currentUser),
"*");
ProxyUsers.refreshSuperUserGroupsConfiguration(config);
@@ -317,7 +344,7 @@ public class TestWrites {
XDR createXdr = new XDR();
createReq.serialize(createXdr);
CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
FileHandle handle = createRsp.getObjHandle();
// Test DATA_SYNC
@@ -330,7 +357,7 @@ public class TestWrites {
XDR writeXdr = new XDR();
writeReq.serialize(writeXdr);
nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler,
- InetAddress.getLocalHost());
+ new InetSocketAddress("localhost", 1234));
waitWrite(nfsd, handle, 60000);
@@ -339,7 +366,7 @@ public class TestWrites {
XDR readXdr = new XDR();
readReq.serialize(readXdr);
READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
assertTrue(Arrays.equals(buffer, readRsp.getData().array()));
@@ -351,7 +378,7 @@ public class TestWrites {
XDR createXdr2 = new XDR();
createReq2.serialize(createXdr2);
CREATE3Response createRsp2 = nfsd.create(createXdr2.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
FileHandle handle2 = createRsp2.getObjHandle();
WRITE3Request writeReq2 = new WRITE3Request(handle2, 0, 10,
@@ -359,7 +386,7 @@ public class TestWrites {
XDR writeXdr2 = new XDR();
writeReq2.serialize(writeXdr2);
nfsd.write(writeXdr2.asReadOnlyWrap(), null, 1, securityHandler,
- InetAddress.getLocalHost());
+ new InetSocketAddress("localhost", 1234));
waitWrite(nfsd, handle2, 60000);
@@ -368,7 +395,7 @@ public class TestWrites {
XDR readXdr2 = new XDR();
readReq2.serialize(readXdr2);
READ3Response readRsp2 = nfsd.read(readXdr2.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
assertTrue(Arrays.equals(buffer, readRsp2.getData().array()));
// FILE_SYNC should sync the file size
@@ -381,4 +408,80 @@ public class TestWrites {
}
}
}
+
+ @Test
+ public void testOOOWrites() throws IOException, InterruptedException {
+ NfsConfiguration config = new NfsConfiguration();
+ MiniDFSCluster cluster = null;
+ RpcProgramNfs3 nfsd;
+ final int bufSize = 32;
+ final int numOOO = 3;
+ SecurityHandler securityHandler = Mockito.mock(SecurityHandler.class);
+ Mockito.when(securityHandler.getUser()).thenReturn(
+ System.getProperty("user.name"));
+ String currentUser = System.getProperty("user.name");
+ config.set(
+ DefaultImpersonationProvider.getTestProvider().
+ getProxySuperuserGroupConfKey(currentUser),
+ "*");
+ config.set(
+ DefaultImpersonationProvider.getTestProvider().
+ getProxySuperuserIpConfKey(currentUser),
+ "*");
+ ProxyUsers.refreshSuperUserGroupsConfiguration(config);
+ // Use emphral port in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
+
+ try {
+ cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
+ cluster.waitActive();
+
+ Nfs3 nfs3 = new Nfs3(config);
+ nfs3.startServiceInternal(false);
+ nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
+
+ DFSClient dfsClient = new DFSClient(NameNode.getAddress(config), config);
+ HdfsFileStatus status = dfsClient.getFileInfo("/");
+ FileHandle rootHandle = new FileHandle(status.getFileId());
+
+ CREATE3Request createReq = new CREATE3Request(rootHandle,
+ "out-of-order-write" + System.currentTimeMillis(),
+ Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0);
+ XDR createXdr = new XDR();
+ createReq.serialize(createXdr);
+ CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ FileHandle handle = createRsp.getObjHandle();
+
+ byte[][] oooBuf = new byte[numOOO][bufSize];
+ for (int i = 0; i < numOOO; i++) {
+ Arrays.fill(oooBuf[i], (byte) i);
+ }
+
+ for (int i = 0; i < numOOO; i++) {
+ final long offset = (numOOO - 1 - i) * bufSize;
+ WRITE3Request writeReq = new WRITE3Request(handle, offset, bufSize,
+ WriteStableHow.UNSTABLE, ByteBuffer.wrap(oooBuf[i]));
+ XDR writeXdr = new XDR();
+ writeReq.serialize(writeXdr);
+ nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler,
+ new InetSocketAddress("localhost", 1234));
+ }
+
+ waitWrite(nfsd, handle, 60000);
+ READ3Request readReq = new READ3Request(handle, bufSize, bufSize);
+ XDR readXdr = new XDR();
+ readReq.serialize(readXdr);
+ READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", config.getInt(
+ NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY,
+ NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT)));
+ assertTrue(Arrays.equals(oooBuf[1], readRsp.getData().array()));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml Tue Aug 19 23:49:39 2014
@@ -18,12 +18,12 @@
<configuration>
<property>
- <name>nfs3.server.port</name>
+ <name>nfs.server.port</name>
<value>2079</value>
</property>
<property>
- <name>nfs3.mountd.port</name>
+ <name>nfs.mountd.port</name>
<value>4272</value>
</property>
</configuration>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Aug 19 23:49:39 2014
@@ -12,6 +12,8 @@ Trunk (Unreleased)
HDFS-5570. Deprecate hftp / hsftp and replace them with webhdfs / swebhdfs.
(wheat9)
+ HDFS-2538. option to disable fsck dots (Mohammad Kamrul Islam via aw)
+
NEW FEATURES
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
@@ -53,8 +55,6 @@ Trunk (Unreleased)
HDFS-3030. Remove getProtocolVersion and getProtocolSignature from translators.
(jitendra)
- HDFS-2976. Remove unnecessary method (tokenRefetchNeeded) in DFSClient.
-
HDFS-3111. Missing license headers in trunk. (umamahesh)
HDFS-3091. Update the usage limitations of ReplaceDatanodeOnFailure policy in
@@ -95,8 +95,6 @@ Trunk (Unreleased)
HDFS-3768. Exception in TestJettyHelper is incorrect.
(Eli Reisman via jghoman)
- HDFS-3851. DFSOutputStream class code cleanup. (Jing Zhao via suresh)
-
HDFS-2580. NameNode#main(...) can make use of GenericOptionsParser. (harsh)
HDFS-2127. Add a test that ensure AccessControlExceptions contain
@@ -125,10 +123,20 @@ Trunk (Unreleased)
HDFS-6228. comments typo fix for FsDatasetImpl.java (zhaoyunjiong via umamahesh)
+ HDFS-6246. Remove 'dfs.support.append' flag from trunk code. (umamahesh)
+
+ HDFS-6252. Phase out the old web UI in HDFS. (wheat9)
+
+ HDFS-6609. Use DirectorySnapshottableFeature to represent a snapshottable
+ directory. (Jing Zhao via wheat9)
+
+ HDFS-6482. Use block ID-based block layout on datanodes (James Thomas via
+ Colin Patrick McCabe)
+
OPTIMIZATIONS
BUG FIXES
-
+
HADOOP-9635 Fix potential Stack Overflow in DomainSocket.c (V. Karthik Kumar
via cmccabe)
@@ -179,9 +187,6 @@ Trunk (Unreleased)
HDFS-3549. Fix dist tar build fails in hadoop-hdfs-raid project. (Jason Lowe via daryn)
- HDFS-3482. hdfs balancer throws ArrayIndexOutOfBoundsException
- if option is specified without values. ( Madhukara Phatak via umamahesh)
-
HDFS-3614. Revert unused MiniDFSCluster constructor from HDFS-3049.
(acmurthy via eli)
@@ -193,9 +198,6 @@ Trunk (Unreleased)
HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from
Command. (Jing Zhao via suresh)
- HADOOP-8158. Interrupting hadoop fs -put from the command line
- causes a LeaseExpiredException. (daryn via harsh)
-
HDFS-2434. TestNameNodeMetrics.testCorruptBlock fails intermittently.
(Jing Zhao via suresh)
@@ -250,6 +252,274 @@ Trunk (Unreleased)
HDFS-5794. Fix the inconsistency of layout version number of
ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9)
+ HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI.
+ (Vinayakumar B via wheat 9)
+
+Release 2.6.0 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ HDFS-6613. Improve logging in caching classes. (wang)
+
+ HDFS-6511. BlockManager#computeInvalidateWork() could do nothing. (Juan Yu via wang)
+
+ HDFS-6638. Shorten test run time with a smaller retry timeout setting.
+ (Liang Xie via cnauroth)
+
+ HDFS-6627. Rename DataNode#checkWriteAccess to checkReadAccess.
+ (Liang Xie via cnauroth)
+
+ HDFS-6645. Add test for successive Snapshots between XAttr modifications.
+ (Stephen Chu via jing9)
+
+ HDFS-6643. Refactor INodeWithAdditionalFields.PermissionStatusFormat and
+ INodeFile.HeaderFormat. (szetszwo)
+
+ HDFS-6640. Syntax for MKDIRS, CREATESYMLINK, and SETXATTR are given wrongly
+ in WebHdfs document (missed webhdfs/v1). (Stephen Chu via jing9)
+
+ HDFS-5202. Support Centralized Cache Management on Windows. (cnauroth)
+
+ HDFS-2976. Remove unnecessary method (tokenRefetchNeeded) in DFSClient.
+ (Uma Maheswara Rao G)
+
+ HDFS-3851. DFSOutputStream class code cleanup. (Jing Zhao via suresh)
+
+ HDFS-2856. Fix block protocol so that Datanodes don't require root or jsvc.
+ (cnauroth)
+
+ HDFS-5624. Add HDFS tests for ACLs in combination with viewfs.
+ (Stephen Chu via cnauroth)
+
+ HDFS-6655. Add 'header banner' to 'explorer.html' also in Namenode UI
+ (vinayakumarb)
+
+ HDFS-4120. Add a new "-skipSharedEditsCheck" option for BootstrapStandby
+ (Liang Xie and Rakesh R via vinayakumarb)
+
+ HDFS-6597. Add a new option to NN upgrade to terminate the process after
+ upgrade on NN is completed. (Danilo Vunjak via cnauroth)
+
+ HDFS-6700. BlockPlacementPolicy shoud choose storage but not datanode for
+ deletion. (szetszwo)
+
+ HDFS-6616. Add exclude-datanodes feature to WebHDFS redirection so that it
+ will not redirect retries to the same datanode. (zhaoyunjiong via szetszwo)
+
+ HDFS-6702. Change DFSClient to pass the StorageType from the namenode to
+ datanodes and change datanode to write block replicas using the specified
+ storage type. (szetszwo)
+
+ HDFS-6701. Make seed optional in NetworkTopology#sortByDistance.
+ (Ashwin Shankar via wang)
+
+ HDFS-6755. There is an unnecessary sleep in the code path where
+ DFSOutputStream#close gives up its attempt to contact the namenode
+ (mitdesai21 via cmccabe)
+
+ HDFS-6750. The DataNode should use its shared memory segment to mark
+ short-circuit replicas that have been unlinked as stale (cmccabe)
+
+ HDFS-6739. Add getDatanodeStorageReport to ClientProtocol. (szetszwo)
+
+ HDFS-6665. Add tests for XAttrs in combination with viewfs.
+ (Stephen Chu via wang)
+
+ HDFS-6778. The extended attributes javadoc should simply refer to the
+ user docs. (clamb via wang)
+
+ HDFS-6570. add api that enables checking if a user has certain permissions on
+ a file. (Jitendra Pandey via cnauroth)
+
+ HDFS-6441. Add ability to exclude/include specific datanodes while
+ balancing. (Benoy Antony and Yu Li via Arpit Agarwal)
+
+ HDFS-6685. Balancer should preserve storage type of replicas. (szetszwo)
+
+ HDFS-6798. Add test case for incorrect data node condition during
+ balancing. (Benoy Antony via Arpit Agarwal)
+
+ HDFS-6796. Improve the argument check during balancer command line parsing.
+ (Benoy Antony via szetszwo)
+
+ HDFS-6794. Update BlockManager methods to use DatanodeStorageInfo
+ where possible (Arpit Agarwal)
+
+ HDFS-6802. Some tests in TestDFSClientFailover are missing @Test
+ annotation. (Akira Ajisaka via wang)
+
+ HDFS-6788. Improve synchronization in BPOfferService with read write lock.
+ (Yongjun Zhang via wang)
+
+ HDFS-6787. Remove duplicate code in FSDirectory#unprotectedConcat. (Yi Liu via umamahesh)
+
+ HDFS-6809. Move Balancer's inner classes MovedBlocks and Matcher as to
+ standalone classes and separates KeyManager from NameNodeConnector.
+ (szetszwo)
+
+ HDFS-6812. Remove addBlock and replaceBlock from DatanodeDescriptor.
+ (szetszwo)
+
+ HDFS-6781. Separate HDFS commands from CommandsManual.apt.vm. (Akira
+ Ajisaka via Arpit Agarwal)
+
+ HDFS-6728. Dynamically add new volumes to DataStorage, formatted if
+ necessary. (Lei Xu via atm)
+
+ HDFS-6740. Make FSDataset support adding data volumes dynamically. (Lei
+ Xu via atm)
+
+ HDFS-6722. Display readable last contact time for dead nodes on NN webUI.
+ (Ming Ma via wheat9)
+
+ HDFS-6772. Get DN storages out of blockContentsStale state faster after
+ NN restarts. (Ming Ma via Arpit Agarwal)
+
+ HDFS-573. Porting libhdfs to Windows. (cnauroth)
+
+ HDFS-6828. Separate block replica dispatching from Balancer. (szetszwo via
+ jing9)
+
+ HDFS-6837. Code cleanup for Balancer and Dispatcher. (szetszwo via
+ jing9)
+
+ HDFS-6838. Code cleanup for unnecessary INode replacement.
+ (Jing Zhao via wheat9)
+
+ HDFS-6836. HDFS INFO logging is verbose & uses file appenders. (Xiaoyu
+ Yao via Arpit Agarwal)
+
+ HDFS-6567. Normalize the order of public final in HdfsFileStatus.
+ (Tassapol Athiapinya via wheat9)
+
+ HDFS-6849. Replace HttpFS custom proxyuser handling with common
+ implementation. (tucu)
+
+ HDFS-6850. Move NFS out of order write unit tests into TestWrites class.
+ (Zhe Zhang via atm)
+
+ HDFS-6188. An ip whitelist based implementation of TrustedChannelResolver.
+ (Benoy Antony via Arpit Agarwal)
+
+ OPTIMIZATIONS
+
+ HDFS-6690. Deduplicate xattr names in memory. (wang)
+
+ BUG FIXES
+
+ HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for
+ insecure HDFS (Allen Wittenauer via raviprak)
+
+ HDFS-6517. Remove hadoop-metrics2.properties from hdfs project (Akira
+ AJISAKA via aw)
+
+ HDFS-6617. Flake TestDFSZKFailoverController.testManualFailoverWithDFSHAAdmin
+ due to a long edit log sync op. (Liang Xie via cnauroth)
+
+ HDFS-6646. [ HDFS Rolling Upgrade - Shell ] shutdownDatanode and getDatanodeInfo
+ usage is missed ( Brahma Reddy Battula via vinayakumarb)
+
+ HDFS-6630. Unable to fetch the block information by Browsing the file system on
+ Namenode UI through IE9 ( Haohui Mai via vinayakumarb)
+
+ HADOOP-8158. Interrupting hadoop fs -put from the command line
+ causes a LeaseExpiredException. (daryn via harsh)
+
+ HDFS-6678. MiniDFSCluster may still be partially running after initialization
+ fails. (cnauroth)
+
+ HDFS-5809. BlockPoolSliceScanner and high speed hdfs appending make
+ datanode to drop into infinite loop (cmccabe)
+
+ HDFS-6456. NFS should throw error for invalid entry in
+ dfs.nfs.exports.allowed.hosts (Abhiraj Butala via brandonli)
+
+ HDFS-6689. NFS doesn't return correct lookup access for direcories (brandonli)
+
+ HDFS-6478. RemoteException can't be retried properly for non-HA scenario.
+ (Ming Ma via jing9)
+
+ HDFS-6693. TestDFSAdminWithHA fails on windows ( vinayakumarb )
+
+ HDFS-6667. In HDFS HA mode, Distcp/SLive with webhdfs on secure cluster fails
+ with Client cannot authenticate via:[TOKEN, KERBEROS] error. (jing9)
+
+ HDFS-6704. Fix the command to launch JournalNode in HDFS-HA document.
+ (Akira AJISAKA via jing9)
+
+ HDFS-6731. Run "hdfs zkfc-formatZK" on a server in a non-namenode will cause
+ a null pointer exception. (Masatake Iwasaki via brandonli)
+
+ HDFS-6114. Block Scan log rolling will never happen if blocks written
+ continuously leading to huge size of dncp_block_verification.log.curr
+ (vinayakumarb via cmccabe)
+
+ HDFS-6455. NFS: Exception should be added in NFS log for invalid separator in
+ nfs.exports.allowed.hosts. (Abhiraj Butala via brandonli)
+
+ HDFS-6715. Webhdfs wont fail over when it gets java.io.IOException: Namenode
+ is in startup mode. (jing9)
+
+ HDFS-5919. FileJournalManager doesn't purge empty and corrupt inprogress edits
+ files (vinayakumarb)
+
+ HDFS-6752. Avoid Address bind errors in TestDatanodeConfig#testMemlockLimit
+ (vinayakumarb)
+
+ HDFS-6749. FSNamesystem methods should call resolvePath.
+ (Charles Lamb via cnauroth)
+
+ HDFS-4629. Using com.sun.org.apache.xml.internal.serialize.* in
+ XmlEditsVisitor.java is JVM vendor specific. Breaks IBM JAVA.
+ (Amir Sanjar via stevel)
+
+ HDFS-3482. hdfs balancer throws ArrayIndexOutOfBoundsException
+ if option is specified without values. ( Madhukara Phatak via umamahesh)
+
+ HDFS-6797. DataNode logs wrong layoutversion during upgrade. (Benoy Antony
+ via Arpit Agarwal)
+
+ HDFS-6810. StorageReport array is initialized with wrong size in
+ DatanodeDescriptor#getStorageReports. (szetszwo via Arpit Agarwal)
+
+ HDFS-5723. Append failed FINALIZED replica should not be accepted as valid
+ when that block is underconstruction (vinayakumarb)
+
+ HDFS-5185. DN fails to startup if one of the data dir is full. (vinayakumarb)
+
+ HDFS-6451. NFS should not return NFS3ERR_IO for AccessControlException
+ (Abhiraj Butala via brandonli)
+
+ HDFS-6717. JIRA HDFS-5804 breaks default nfs-gateway behavior for unsecured config
+ (brandonli)
+
+ HDFS-6790. DFSUtil Should Use configuration.getPassword for SSL passwords
+ (Larry McCay via brandonli)
+
+ HDFS-6791. A block could remain under replicated if all of its replicas are on
+ decommissioned nodes. (Ming Ma via jing9)
+
+ HDFS-6582. Missing null check in RpcProgramNfs3#read(XDR, SecurityHandler)
+ (Abhiraj Butala via brandonli)
+
+ HDFS-6830. BlockInfo.addStorage fails when DN changes the storage for a
+ block replica (Arpit Agarwal)
+
+ HDFS-6247. Avoid timeouts for replaceBlock() call by sending intermediate
+ responses to Balancer (vinayakumarb)
+
+ HDFS-6783. Fix HDFS CacheReplicationMonitor rescan logic. (Yi Liu and Colin Patrick McCabe via umamahesh)
+
+ HDFS-6825. Edit log corruption due to delayed block removal.
+ (Yongjun Zhang via wang)
+
+ HDFS-6569. OOB message can't be sent to the client when DataNode shuts down for upgrade
+ (brandonli)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -260,6 +530,27 @@ Release 2.5.0 - UNRELEASED
NEW FEATURES
+ HDFS-6281. Provide option to use the NFS Gateway without having to use the
+ Hadoop portmapper. (atm)
+
+ HDFS-5168. Add cross node dependency support to BlockPlacementPolicy.
+ (Nikola Vujic via szetszwo)
+
+ HDFS-6334. Client failover proxy provider for IP failover based NN HA.
+ (kihwal)
+
+ HDFS-6406. Add capability for NFS gateway to reject connections from
+ unprivileged ports. (atm)
+
+ HDFS-2006. Ability to support storing extended attributes per file.
+
+ HDFS-5978. Create a tool to take fsimage and expose read-only WebHDFS API.
+ (Akira Ajisaka via wheat9)
+
+ HDFS-6278. Create HTML5-based UI for SNN. (wheat9)
+
+ HDFS-6279. Create new index page for JN / DN. (wheat9)
+
IMPROVEMENTS
HDFS-6007. Update documentation about short-circuit local reads (iwasakims
@@ -277,9 +568,6 @@ Release 2.5.0 - UNRELEASED
HDFS-6158. Clean up dead code for OfflineImageViewer. (wheat9)
- HDFS-5978. Create a tool to take fsimage and expose read-only WebHDFS API.
- (Akira Ajisaka via wheat9)
-
HDFS-6164. Remove lsr in OfflineImageViewer. (wheat9)
HDFS-6167. Relocate the non-public API classes in the hdfs.client package.
@@ -302,10 +590,199 @@ Release 2.5.0 - UNRELEASED
HDFS-6219. Proxy superuser configuration should use true client IP for
address checks. (daryn via kihwal)
+ HDFS-6256. Clean up ImageVisitor and SpotCheckImageVisitor.
+ (Akira Ajisaka via wheat9)
+
+ HDFS-6265. Prepare HDFS codebase for JUnit 4.11. (cnauroth)
+
+ HDFS-5693. Few NN metrics data points were collected via JMX when NN
+ is under heavy load. (Ming Ma via jing9)
+
+ HDFS-6273. Config options to allow wildcard endpoints for namenode HTTP
+ and HTTPS servers. (Arpit Agarwal)
+
+ HDFS-6282. Re-add testIncludeByRegistrationName. (cmccabe)
+
+ HDFS-6266. Identify full path for a given INode. (jing9)
+
+ HDFS-6210. Support GETACLSTATUS operation in WebImageViewer.
+ (Akira Ajisaka via wheat9)
+
+ HDFS-6269. NameNode Audit Log should differentiate between webHDFS open and
+ HDFS open. (Eric Payne via jeagles)
+
+ HDFS-6304. Consolidate the logic of path resolution in FSDirectory.
+ (wheat9)
+
+ HDFS-6295. Add "decommissioning" state and node state filtering to
+ dfsadmin. (wang)
+
+ HDFS-6294. Use INode IDs to avoid conflicts when a file open for write is
+ renamed. (cmccabe)
+
+ HDFS-6328. Clean up dead code in FSDirectory. (wheat9)
+
+ HDFS-6230. Expose upgrade status through NameNode web UI.
+ (Mit Desai via wheat9)
+
+ HDFS-6186. Pause deletion of blocks when the namenode starts up. (jing9)
+
+ HDFS-6293. Issues with OIV processing PB-based fsimages. (kihwal)
+
+ HDFS-2949. Add check to active state transition to prevent operator-induced
+ split brain. (Rushabh S Shah via kihwal)
+
+ HDFS-6287. Add vecsum test of libhdfs read access times (cmccabe)
+
+ HDFS-5683. Better audit log messages for caching operations.
+ (Abhiraj Butala via wang)
+
+ HDFS-6345. DFS.listCacheDirectives() should allow filtering based on
+ cache directive ID. (wang)
+
+ HDFS-6432. Add snapshot related APIs to webhdfs. (jing9)
+
+ HDFS-6396. Remove support for ACL feature from INodeSymlink.
+ (Charles Lamb via wang)
+
+ HDFS-6435. Add support for specifying a static uid/gid mapping for the NFS
+ gateway. (atm via wang)
+
+ HDFS-6416. Use Time#monotonicNow in OpenFileCtx and OpenFileCtxCatch to
+ avoid system clock bugs (Abhiraj Butala via brandonli)
+
+ HDFS-6356. Fix typo in DatanodeLayoutVersion. (Tulasi G via wang)
+
+ HDFS-6447. balancer should timestamp the completion message.
+ (Juan Yu via wang)
+
+ HDFS-6463. Clarify behavior of AclStorage#createFsPermissionForExtendedAcl
+ in comments. (cnauroth)
+
+ HDFS-6472. fix typo in webapps/hdfs/explorer.js. (Juan Yu via wang)
+
+ HDFS-6056. Clean up NFS config settings (brandonli)
+
+ HDFS-6110 adding more slow action log in critical write path
+ (Liang Xie via stack)
+
+ HDFS-6109 let sync_file_range() system call run in background
+ (Liang Xie via stack)
+
+ HDFS-6268. Better sorting in NetworkTopology#pseudoSortByDistance when
+ no local node is found. (wang)
+
+ HDFS-6369. Document that BlockReader#available() can return more bytes than
+ are remaining in the block (Ted Yu via Colin Patrick McCabe)
+
+ HDFS-6487. TestStandbyCheckpoint#testSBNCheckpoints is racy.
+ (Mit Desai via wang)
+
+ HDFS-6297. Add CLI testcases to reflect new features of dfs and dfsadmin
+ (Dasha Boudnik via cos)
+
+ HDFS-6399. Add note about setfacl in HDFS permissions guide.
+ (cnauroth via wang)
+
+ HDFS-6315. Decouple recording edit logs from FSDirectory. (wheat9)
+
+ HDFS-6379. HTTPFS - Implement ACLs support. (yoderme via tucu)
+
+ HDFS-6471. Make moveFromLocal CLI testcases to be non-disruptive
+ (Dasha Boudnik via cos)
+
+ HDFS-6395. Skip checking xattr limits for non-user-visible namespaces.
+ (Yi Liu via wang).
+
+ HDFS-3493. Invalidate excess corrupted blocks as long as minimum
+ replication is satisfied. (Juan Yu and Vinayakumar B via wang)
+
+ HDFS-6330. Move mkdirs() to FSNamesystem. (wheat9)
+
+ HDFS-6470. TestBPOfferService.testBPInitErrorHandling is flaky.
+ (Ming Ma via wang)
+
+ HDFS-6529. Trace logging for RemoteBlockReader2 to identify remote datanode
+ and file being read. (Anubhav Dhoot via atm)
+
+ HDFS-6499. Use NativeIO#renameTo instead of File#renameTo in
+ FileJournalManager. (Yongjun Zhang via atm)
+
+ HDFS-6518. TestCacheDirectives#testExceedsCapacity should
+ take FSN read lock when accessing pendingCached list.
+ (wang)
+
+ HDFS-6528. Add XAttrs to TestOfflineImageViewer. (Stephen Chu via wang)
+
+ HDFS-6545. Finalizing rolling upgrade can make NN unavailable for a long
+ duration. (kihwal)
+
+ HDFS-6530. Fix Balancer documentation. (szetszwo)
+
+ HDFS-6480. Move waitForReady() from FSDirectory to FSNamesystem. (wheat9)
+
+ HDFS-6403. Add metrics for log warnings reported by JVM pauses. (Yongjun
+ Zhang via atm)
+
+ HDFS-6557. Move the reference of fsimage to FSNamesystem. (wheat9)
+
+ HDFS-4667. Capture renamed files/directories in snapshot diff report. (jing9
+ and Binglin Chang via jing9)
+
+ HDFS-6507. Improve DFSAdmin to support HA cluster better.
+ (Zesheng Wu via vinayakumarb)
+
+ HDFS-6578. add toString method to DatanodeStorage for easier debugging.
+ (Yongjun Zhang via Arpit Agarwal)
+
+ HDFS-6562. Refactor rename() in FSDirectory. (wheat9)
+
+ HDFS-6486. Add user doc for XAttrs via WebHDFS. (Yi Liu via umamahesh)
+
+ HDFS-6430. HTTPFS - Implement XAttr support. (Yi Liu via tucu)
+
+ HDFS-6593. Move SnapshotDiffInfo out of INodeDirectorySnapshottable.
+ (Jing Zhao via wheat9)
+
+ HDFS-6595. Allow the maximum threads for balancing on datanodes to be
+ configurable. (Benoy Antony via szetszwo)
+
+ HDFS-6572. Add an option to the NameNode that prints the software and
+ on-disk image versions. (Charles Lamb via cnauroth)
+
+ HDFS-6603. Add XAttr with ACL test. (Stephen Chu via cnauroth)
+
+ HDFS-6612. MiniDFSNNTopology#simpleFederatedTopology(int)
+ always hardcode nameservice ID. (Juan Yu via wang)
+
+ HDFS-6614. shorten TestPread run time with a smaller retry timeout setting.
+ (Liang Xie via cnauroth)
+
+ HDFS-6610. TestShortCircuitLocalRead tests sometimes timeout on slow
+ machines. (Charles Lamb via wang)
+
+ HDFS-6620. Snapshot docs should specify about preserve options with cp command
+ (Stephen Chu via umamahesh)
+
+ HDFS-6493. Change dfs.namenode.startup.delay.block.deletion to second
+ instead of millisecond. (Juan Yu via wang)
+
+ HDFS-6680. BlockPlacementPolicyDefault does not choose favored nodes
+ correctly. (szetszwo)
+
+ HDFS-6712. Document HDFS Multihoming Settings. (Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
+ HDFS-6460. Ignore stale and decommissioned nodes in
+ NetworkTopology#sortByDistance. (Yongjun Zhang via wang)
+
+ HDFS-6583. Remove clientNode in FileUnderConstructionFeature. (wheat9)
+
+ HDFS-6599. 2.4 addBlock is 10 to 20 times slower compared to 0.23 (daryn)
+
BUG FIXES
HDFS-6112. NFS Gateway docs are incorrect for allowed hosts configuration.
@@ -362,7 +839,312 @@ Release 2.5.0 - UNRELEASED
HDFS-6153. Document "fileId" and "childrenNum" fields in the FileStatus Json schema
(Akira Ajisaka via vinayakumarb)
-Release 2.4.1 - UNRELEASED
+ HDFS-6178. Decommission on standby NN couldn't finish. (Ming Ma via jing9)
+
+ HDFS-6213. TestDataNodeConfig failing on Jenkins runs due to DN web port
+ in use. (wang)
+
+ HDFS-6274. Cleanup javadoc warnings in HDFS code. (suresh)
+
+ HDFS-6275. Fix warnings - type arguments can be inferred and redudant
+ local variable. (suresh)
+
+ HDFS-6217. Webhdfs PUT operations may not work via a http proxy.
+ (Daryn Sharp via kihwal)
+
+ HDFS-6276. Remove unnecessary conditions and null check. (suresh)
+
+ HDFS-5865. Update OfflineImageViewer document. (Akira Ajisaka via wheat9)
+
+ HDFS-6270. Secondary namenode status page shows transaction count in bytes.
+ (Benoy Antony via wheat9)
+
+ HDFS-6218. Audit log should use true client IP for proxied webhdfs
+ operations. (daryn via kihwal)
+
+ HDFS-6288. DFSInputStream Pread doesn't update ReadStatistics.
+ (Juan Yu via wang)
+
+ HDFS-6289. HA failover can fail if there are pending DN messages for DNs
+ which no longer exist. (atm)
+
+ HDFS-6337. Setfacl testcase is failing due to dash character in username
+ in TestAclCLI (umamahesh)
+
+ HDFS-5381. ExtendedBlock#hashCode should use both blockId and block pool ID
+ (Benoy Antony via Colin Patrick McCabe)
+
+ HDFS-6240. WebImageViewer returns 404 if LISTSTATUS to an empty directory.
+ (Akira Ajisaka via wheat9)
+
+ HDFS-6351. Command hdfs dfs -rm -r can't remove empty directory.
+ (Yongjun Zhang via wang)
+
+ HDFS-5522. Datanode disk error check may be incorrectly skipped.
+ (Rushabh S Shah via kihwal)
+
+ HDFS-6367. EnumSetParam$Domain#parse fails for parameter containing more than one enum.
+ (Yi Liu via umamahesh)
+
+ HDFS-6305. WebHdfs response decoding may throw RuntimeExceptions (Daryn
+ Sharp via jeagles)
+
+ HDFS-6355. Fix divide-by-zero, improper use of wall-clock time in
+ BlockPoolSliceScanner (cmccabe)
+
+ HDFS-6370. Web UI fails to display in intranet under IE.
+ (Haohui Mai via cnauroth)
+
+ HDFS-6381. Fix a typo in INodeReference.java. (Binglin Chang via jing9)
+
+ HDFS-6400. Cannot execute hdfs oiv_legacy. (Akira AJISAKA via kihwal)
+
+ HDFS-6250. Fix test failed in TestBalancerWithNodeGroup.testBalancerWithRackLocality
+ (Binglin Chang and Chen He via junping_du)
+
+ HDFS-4913. Deleting file through fuse-dfs when using trash fails requiring
+ root permissions (cmccabe)
+
+ HDFS-6421. Fix vecsum.c compile on BSD and some other systems. (Mit Desai
+ via Colin Patrick McCabe)
+
+ HDFS-6419. TestBookKeeperHACheckpoints#TestSBNCheckpoints fails on trunk.
+ (Akira AJISAKA via kihwal)
+
+ HDFS-6409. Fix typo in log message about NameNode layout version upgrade.
+ (Chen He via cnauroth)
+
+ HDFS-6433. Replace BytesMoved class with AtomicLong.
+ (Benoy Antony via cnauroth)
+
+ HDFS-6438. DeleteSnapshot should be a DELETE request in WebHdfs. (jing9)
+
+ HDFS-6423. Diskspace quota usage should be updated when appending data to
+ partial block. (jing9)
+
+ HDFS-6443. Fix MiniQJMHACluster related test failures. (Zesheng Wu via
+ Arpit Agarwal)
+
+ HDFS-6227. ShortCircuitCache#unref should purge ShortCircuitReplicas whose
+ streams have been closed by java interrupts. (Colin Patrick McCabe via jing9)
+
+ HDFS-6442. Fix TestEditLogAutoroll and TestStandbyCheckpoints failure
+ caused by port conficts. (Zesheng Wu via Arpit Agarwal)
+
+ HDFS-6448. BlockReaderLocalLegacy should set socket timeout based on
+ conf.socketTimeout (liangxie via cmccabe)
+
+ HDFS-6453. Use Time#monotonicNow to avoid system clock reset.
+ (Liang Xie via wang)
+
+ HDFS-6461. Use Time#monotonicNow to compute duration in DataNode#shutDown.
+ (James Thomas via wang)
+
+ HDFS-6462. NFS: fsstat request fails with the secure hdfs (brandonli)
+
+ HDFS-6404. HttpFS should use a 000 umask for mkdir and create
+ operations. (yoderme via tucu)
+
+ HDFS-6424. blockReport doesn't need to invalidate blocks on SBN. (Ming Ma
+ via jing9)
+
+ HDFS-6497. Make TestAvailableSpaceVolumeChoosingPolicy deterministic
+ (cmccabe)
+
+ HDFS-6500. Snapshot shouldn't be removed silently after renaming to an
+ existing snapshot. (Nicholas SZE via junping_du)
+
+ HDFS-6257. TestCacheDirectives#testExceedsCapacity fails occasionally
+ (cmccabe)
+
+ HDFS-6364. Incorrect check for unknown datanode in Balancer. (Benoy
+ Antony via Arpit Agarwal)
+
+ HDFS-6503. Fix typo of DFSAdmin restoreFailedStorage.
+ (Zesheng Wu via wheat9)
+
+ HDFS-6464. Support multiple xattr.name parameters for WebHDFS getXAttrs.
+ (Yi Liu via umamahesh)
+
+ HDFS-6539. test_native_mini_dfs is skipped in hadoop-hdfs/pom.xml
+ (decstery via cmccabe)
+
+ HDFS-6527. Edit log corruption due to defered INode removal. (kihwal and
+ jing9 via jing9)
+
+ HDFS-6552. add DN storage to a BlockInfo will not replace the different
+ storage from same DN. (Amir Langer via Arpit Agarwal)
+
+ HDFS-6551. Rename with OVERWRITE option may throw NPE when the target
+ file/directory is a reference INode. (jing9)
+
+ HDFS-6439. NFS should not reject NFS requests to the NULL procedure whether
+ port monitoring is enabled or not. (brandonli)
+
+ HDFS-6559. Fix wrong option "dfsadmin -rollingUpgrade start" in the
+ document. (Akira Ajisaka via Arpit Agarwal)
+
+ HDFS-6553. Add missing DeprecationDeltas for NFS Kerberos configurations
+ (Stephen Chu via brandonli)
+
+ HDFS-6563. NameNode cannot save fsimage in certain circumstances when
+ snapshots are in use. (atm)
+
+ HDFS-3848. A Bug in recoverLeaseInternal method of FSNameSystem class
+ (Hooman Peiro Sajjad and Chen He via kihwal)
+
+ HDFS-6549. Add support for accessing the NFS gateway from the AIX NFS
+ client. (atm)
+
+ HDFS-6535. HDFS quota update is wrong when file is appended. (George Wong
+ via jing9)
+
+ HDFS-6222. Remove background token renewer from webhdfs.
+ (Rushabh Shah and Daryn Sharp via cnauroth)
+
+ HDFS-6580. FSNamesystem.mkdirsInt should call the getAuditFileInfo()
+ wrapper. (Zhilei Xu via wheat9)
+
+ HDFS-6587. Bug in TestBPOfferService can cause test failure. (Zhilei Xu
+ via Arpit Agarwal)
+
+ HDFS-6598. Fix a typo in message issued from explorer.js. (Yongjun Zhang
+ via wheat9)
+
+ HDFS-6475. WebHdfs clients fail without retry because incorrect handling
+ of StandbyException. (Yongjun Zhang via atm)
+
+ HADOOP-10701. NFS should not validate the access premission only based on
+ the user's primary group (Harsh J via atm)
+
+ HDFS-6556. Refine XAttr permissions (umamahesh)
+
+ HDFS-6601. Issues in finalizing rolling upgrade when there is a layout
+ version change (kihwal)
+
+ HDFS-6418. Regression: DFS_NAMENODE_USER_NAME_KEY missing
+ (szetszwo via stevel)
+
+ HDFS-6558. Missing newline in the description of dfsadmin -rollingUpgrade.
+ (Chen He via kihwal)
+
+ HDFS-6591. while loop is executed tens of thousands of times in Hedged Read
+ (Liang Xie via cnauroth)
+
+ HDFS-6604. The short-circuit cache doesn't correctly time out replicas that
+ haven't been used in a while (cmccabe)
+
+ HDFS-4286. Changes from BOOKKEEPER-203 broken capability of including
+ bookkeeper-server jar in hidden package of BKJM (Rakesh R via umamahesh)
+
+ HDFS-4221. Remove the format limitation point from BKJM documentation as HDFS-3810
+ closed. (Rakesh R via umamahesh)
+
+ HDFS-5411. Update Bookkeeper dependency to 4.2.3. (Rakesh R via umamahesh)
+
+ HDFS-6631. TestPread#testHedgedReadLoopTooManyTimes fails intermittently.
+ (Liang Xie via cnauroth)
+
+ HDFS-6647. Edit log corruption when pipeline recovery occurs for deleted
+ file present in snapshot (kihwal)
+
+ HDFS-6378. NFS registration should timeout instead of hanging when
+ portmap/rpcbind is not available (Abhiraj Butala via brandonli)
+
+ HDFS-6703. NFS: Files can be deleted from a read-only mount
+ (Srikanth Upputuri via brandonli)
+
+ HDFS-6422. getfattr in CLI doesn't throw exception or return non-0 return code
+ when xattr doesn't exist. (Charles Lamb via umamahesh)
+
+ HDFS-6696. Name node cannot start if the path of a file under
+ construction contains ".snapshot". (wang)
+
+ HDFS-6312. WebHdfs HA failover is broken on secure clusters.
+ (daryn via tucu)
+
+ HDFS-6618. FSNamesystem#delete drops the FSN lock between removing INodes
+ from the tree and deleting them from the inode map (kihwal via cmccabe)
+
+ HDFS-6622. Rename and AddBlock may race and produce invalid edits (kihwal
+ via cmccabe)
+
+ HDFS-6723. New NN webUI no longer displays decommissioned state for dead node.
+ (Ming Ma via wheat9)
+
+ HDFS-6768. Fix a few unit tests that use hard-coded port numbers. (Arpit
+ Agarwal)
+
+ BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
+
+ HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh)
+
+ HDFS-6302. Implement XAttr as a INode feature. (Yi Liu via umamahesh)
+
+ HDFS-6309. Javadocs for Xattrs apis in DFSClient and other minor fixups. (Charles Lamb via umamahesh)
+
+ HDFS-6258. Namenode server-side storage for XAttrs. (Yi Liu via umamahesh)
+
+ HDFS-6303. HDFS implementation of FileContext API for XAttrs. (Yi Liu and Charles Lamb via umamahesh)
+
+ HDFS-6324. Shift XAttr helper code out for reuse. (Yi Liu via umamahesh)
+
+ HDFS-6301. NameNode: persist XAttrs in fsimage and record XAttrs modifications to edit log.
+ (Yi Liu via umamahesh)
+
+ HDFS-6298. XML based End-to-End test for getfattr and setfattr commands. (Yi Liu via umamahesh)
+
+ HDFS-6314. Test cases for XAttrs. (Yi Liu via umamahesh)
+
+ HDFS-6344. Maximum limit on the size of an xattr. (Yi Liu via umamahesh)
+
+ HDFS-6377. Unify xattr name and value limits into a single limit. (wang)
+
+ HDFS-6373. Remove support for extended attributes on symlinks. (Charles Lamb via wang)
+
+ HDFS-6283. Write end user documentation for xattrs. (wang)
+
+ HDFS-6412. Interface audience and stability annotations missing from
+ several new classes related to xattrs. (wang)
+
+ HDFS-6259. Support extended attributes via WebHDFS. (yliu)
+
+ HDFS-6346. Optimize OP_SET_XATTRS by persisting single Xattr entry per setXattr/removeXattr api call
+ (Yi Liu via umamahesh)
+
+ HDFS-6331. ClientProtocol#setXattr should not be annotated idempotent.
+ (umamahesh via wang)
+
+ HDFS-6335. TestOfflineEditsViewer for XAttr. (Yi Liu via umamahesh)
+
+ HDFS-6343. fix TestNamenodeRetryCache and TestRetryCacheWithHA failures. (umamahesh)
+
+ HDFS-6366. FsImage loading failed with RemoveXattr op (umamahesh)
+
+ HDFS-6357. SetXattr should persist rpcIDs for handling retrycache with Namenode restart and HA
+ (umamahesh)
+
+ HDFS-6372. Handle setXattr rpcIDs for OfflineEditsViewer. (umamahesh)
+
+ HDFS-6410. DFSClient unwraps AclException in xattr methods, but those
+ methods cannot throw AclException. (wang)
+
+ HDFS-6413. xattr names erroneously handled as case-insensitive.
+ (Charles Lamb via cnauroth)
+
+ HDFS-6414. xattr modification operations are based on state of latest
+ snapshot instead of current version of inode. (Andrew Wang via cnauroth)
+
+ HDFS-6374. setXAttr should require the user to be the owner of the file
+ or directory (Charles Lamb via wang)
+
+ HDFS-6375. Listing extended attributes with the search permission.
+ (Charles Lamb via wang)
+
+ HDFS-6492. Support create-time xattrs and atomically setting multiple
+ xattrs. (wang)
+
+Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
@@ -414,6 +1196,40 @@ Release 2.4.1 - UNRELEASED
HDFS-6236. ImageServlet should use Time#monotonicNow to measure latency.
(cnauroth)
+ HDFS-6245. datanode fails to start with a bad disk even when failed
+ volumes is set. (Arpit Agarwal)
+
+ HDFS-2882. DN continues to start up, even if block pool fails to initialize
+ (vinayakumarb)
+
+ HDFS-6340. DN can't finalize upgrade. (Rahul Singhal via Arpit Agarwal)
+
+ HDFS-6329. WebHdfs does not work if HA is enabled on NN but logical URI is
+ not configured. (kihwal)
+
+ HDFS-6313. WebHdfs may use the wrong NN when configured for multiple HA NNs
+ (kihwal)
+
+ HDFS-6326. WebHdfs ACL compatibility is broken. (cnauroth)
+
+ HDFS-6361. TestIdUserGroup.testUserUpdateSetting failed due to out of range
+ nfsnobody Id. (Yongjun Zhang via brandonli)
+
+ HDFS-6362. InvalidateBlocks is inconsistent in usage of DatanodeUuid and
+ StorageID. (Arpit Agarwal)
+
+ HDFS-6402. Suppress findbugs warning for failure to override equals and
+ hashCode in FsAclPermission. (cnauroth)
+
+ HDFS-6325. Append should fail if the last block has insufficient number of
+ replicas (Keith Pak via cos)
+
+ HDFS-6397. NN shows inconsistent value in deadnode count.
+ (Mohammad Kamrul Islam via kihwal)
+
+ HDFS-6411. nfs-hdfs-gateway mount raises I/O error and hangs when a
+ unauthorized user attempts to access it (brandonli)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES