You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by he...@apache.org on 2021/04/29 13:14:48 UTC
[hadoop] branch trunk updated: HDFS-15561. RBF: Remove NPE when
local namenode is not configured (#2954). Contributed by Fengnan Li.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 552e9dc HDFS-15561. RBF: Remove NPE when local namenode is not configured (#2954). Contributed by Fengnan Li.
552e9dc is described below
commit 552e9dcc6c778df02e24d514750bf68151d0a77f
Author: lfengnan <lf...@uber.com>
AuthorDate: Thu Apr 29 06:14:18 2021 -0700
HDFS-15561. RBF: Remove NPE when local namenode is not configured (#2954). Contributed by Fengnan Li.
Reviewed-by: He Xiaoqiao <he...@apache.org>
---
.../hdfs/server/federation/router/Router.java | 17 +++++++++-
.../router/TestRouterNamenodeHeartbeat.java | 38 ++++++++++++++++++++++
.../router/TestRouterWebHdfsMethods.java | 1 +
3 files changed, 55 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index d6e5a1c..ae9b62b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -550,14 +550,20 @@ public class Router extends CompositeService implements
*
* @return Updater of the status for the local Namenode.
*/
- protected NamenodeHeartbeatService createLocalNamenodeHeartbeatService() {
+ @VisibleForTesting
+ public NamenodeHeartbeatService createLocalNamenodeHeartbeatService() {
// Detect NN running in this machine
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
+ if (nsId == null) {
+ LOG.error("Cannot find local nameservice id");
+ return null;
+ }
String nnId = null;
if (HAUtil.isHAEnabled(conf, nsId)) {
nnId = HAUtil.getNameNodeId(conf, nsId);
if (nnId == null) {
LOG.error("Cannot find namenode id for local {}", nsId);
+ return null;
}
}
@@ -787,4 +793,13 @@ public class Router extends CompositeService implements
return adminServer;
}
+ /**
+ * Set router configuration.
+ * @param conf
+ */
+ @VisibleForTesting
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
index d2bc5d6..38419ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
@@ -17,15 +17,21 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.federation.MockResolver;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
@@ -106,6 +112,38 @@ public class TestRouterNamenodeHeartbeat {
}
@Test
+ public void testLocalNamenodeHeartbeatService() throws IOException {
+ Router router = new Router();
+ Configuration conf = new Configuration();
+ assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf));
+
+ // case 1: no local nn is configured
+ router.setConf(conf);
+ assertNull(router.createLocalNamenodeHeartbeatService());
+
+ // case 2: local nn is configured
+ conf.set(DFS_NAMESERVICES, "ns1");
+ assertEquals("ns1", DFSUtil.getNamenodeNameServiceId(conf));
+ conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),
+ "nn1,nn2");
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"),
+ "localhost:8020");
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"),
+ "ns1-nn2.example.com:8020");
+ router.setConf(conf);
+ NamenodeHeartbeatService heartbeatService =
+ router.createLocalNamenodeHeartbeatService();
+ assertNotNull(heartbeatService);
+ // we have to start the service to get the serviceAddress assigned
+ heartbeatService.init(conf);
+ assertEquals("ns1-nn1:localhost:8020",
+ heartbeatService.getNamenodeDesc());
+ heartbeatService.stop();
+ }
+
+ @Test
public void testHearbeat() throws InterruptedException, IOException {
// Set NAMENODE1 to active for all nameservices
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
index 7028928..8e82d44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
@@ -61,6 +61,7 @@ public class TestRouterWebHdfsMethods {
.admin()
.build();
cluster.addRouterOverrides(conf);
+ cluster.setIndependentDNs();
cluster.startCluster();
cluster.startRouters();
cluster.waitClusterUp();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org