You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2014/07/24 20:30:23 UTC
svn commit: r1613239 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/
src/test/java/org/apache/hadoop/hdfs/web/
Author: jing9
Date: Thu Jul 24 18:30:22 2014
New Revision: 1613239
URL: http://svn.apache.org/r1613239
Log:
HDFS-6715. Merge r1613237 from trunk.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1613239&r1=1613238&r2=1613239&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Jul 24 18:30:22 2014
@@ -112,6 +112,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6455. NFS: Exception should be added in NFS log for invalid separator in
nfs.exports.allowed.hosts. (Abhiraj Butala via brandonli)
+ HDFS-6715. Webhdfs wont fail over when it gets java.io.IOException: Namenode
+ is in startup mode. (jing9)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1613239&r1=1613238&r2=1613239&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Thu Jul 24 18:30:22 2014
@@ -113,6 +113,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.XAttrSetFlagParam;
import org.apache.hadoop.hdfs.web.resources.XAttrValueParam;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetworkTopology.InvalidTopologyException;
import org.apache.hadoop.net.Node;
@@ -190,7 +191,7 @@ public class NamenodeWebHdfsMethods {
throws IOException {
final NamenodeProtocols np = namenode.getRpcServer();
if (np == null) {
- throw new IOException("Namenode is in startup mode");
+ throw new RetriableException("Namenode is in startup mode");
}
return np;
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java?rev=1613239&r1=1613238&r2=1613239&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java Thu Jul 24 18:30:22 2014
@@ -39,14 +39,18 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.TestDFSClientRetries;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
-import org.apache.hadoop.hdfs.TestDFSClientRetries;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.internal.util.reflection.Whitebox;
/** Test WebHDFS */
public class TestWebHDFS {
@@ -445,4 +449,37 @@ public class TestWebHDFS {
}
}
}
+
+ /**
+ * Make sure a RetriableException is thrown when rpcServer is null in
+ * NamenodeWebHdfsMethods.
+ */
+ @Test
+ public void testRaceWhileNNStartup() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+ final NameNode namenode = cluster.getNameNode();
+ final NamenodeProtocols rpcServer = namenode.getRpcServer();
+ Whitebox.setInternalState(namenode, "rpcServer", null);
+
+ final Path foo = new Path("/foo");
+ final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsFileSystem.SCHEME);
+ try {
+ webHdfs.mkdirs(foo);
+ fail("Expected RetriableException");
+ } catch (RetriableException e) {
+ GenericTestUtils.assertExceptionContains("Namenode is in startup mode",
+ e);
+ }
+ Whitebox.setInternalState(namenode, "rpcServer", rpcServer);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java?rev=1613239&r1=1613238&r2=1613239&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java Thu Jul 24 18:30:22 2014
@@ -18,6 +18,15 @@
package org.apache.hadoop.hdfs.web;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -29,18 +38,14 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URI;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
+import org.mockito.internal.util.reflection.Whitebox;
public class TestWebHDFSForHA {
private static final String LOGICAL_NAME = "minidfs";
@@ -182,4 +187,61 @@ public class TestWebHDFSForHA {
}
}
}
+
+ /**
+ * Make sure the WebHdfsFileSystem will retry based on RetriableException when
+ * rpcServer is null in NamenodeWebHdfsMethods while NameNode starts up.
+ */
+ @Test (timeout=120000)
+ public void testRetryWhileNNStartup() throws Exception {
+ final Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
+ MiniDFSCluster cluster = null;
+ final Map<String, Boolean> resultMap = new HashMap<String, Boolean>();
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo)
+ .numDataNodes(0).build();
+ HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME);
+ cluster.waitActive();
+ cluster.transitionToActive(0);
+
+ final NameNode namenode = cluster.getNameNode(0);
+ final NamenodeProtocols rpcServer = namenode.getRpcServer();
+ Whitebox.setInternalState(namenode, "rpcServer", null);
+
+ new Thread() {
+ @Override
+ public void run() {
+ boolean result = false;
+ FileSystem fs = null;
+ try {
+ fs = FileSystem.get(WEBHDFS_URI, conf);
+ final Path dir = new Path("/test");
+ result = fs.mkdirs(dir);
+ } catch (IOException e) {
+ result = false;
+ } finally {
+ IOUtils.cleanup(null, fs);
+ }
+ synchronized (TestWebHDFSForHA.this) {
+ resultMap.put("mkdirs", result);
+ TestWebHDFSForHA.this.notifyAll();
+ }
+ }
+ }.start();
+
+ Thread.sleep(1000);
+ Whitebox.setInternalState(namenode, "rpcServer", rpcServer);
+ synchronized (this) {
+ while (!resultMap.containsKey("mkdirs")) {
+ this.wait();
+ }
+ Assert.assertTrue(resultMap.get("mkdirs"));
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}