You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2026/04/03 06:26:40 UTC
(hadoop) branch branch-3.5 updated: HDFS-17888. Fix MiniCluster startup & HttpServer service's conf issues. (#8342)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 8cc7a77d866 HDFS-17888. Fix MiniCluster startup & HttpServer service's conf issues. (#8342)
8cc7a77d866 is described below
commit 8cc7a77d866e859c0cd9cb13d9610ffbfdca43f8
Author: Zhaobo Huang <hu...@126.com>
AuthorDate: Fri Apr 3 14:26:34 2026 +0800
HDFS-17888. Fix MiniCluster startup & HttpServer service's conf issues. (#8342)
---
.../org/apache/hadoop/test/GenericTestUtils.java | 5 +++--
.../hadoop/hdfs/server/datanode/DataNode.java | 5 ++++-
.../server/datanode/web/DatanodeHttpServer.java | 8 ++++++--
.../hdfs/server/namenode/NameNodeHttpServer.java | 19 ++++++++++++++----
.../src/main/webapps/datanode/datanode.html | 2 +-
.../org/apache/hadoop/hdfs/TestDatanodeConfig.java | 23 ++++++++++++++++++++++
6 files changed, 52 insertions(+), 10 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index ada8a9cdb29..65c6c8e2cd5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -221,8 +221,9 @@ public static File getTestDir() {
prop = DEFAULT_TEST_DATA_DIR;
}
File dir = new File(prop).getAbsoluteFile();
- dir.mkdirs();
- assertExists(dir);
+ if (dir.mkdirs() && !dir.exists()) {
+ throw new IllegalStateException("Directory " + dir + " not created");
+ }
return dir;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 3a1b1e07f36..7ee9a4b1729 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1562,7 +1562,10 @@ private void initIpcServer() throws IOException {
DFSUtil.addInternalPBProtocol(getConf(), InterDatanodeProtocolPB.class, service,
ipcServer);
- LOG.info("Opened IPC server at {}", ipcServer.getListenerAddress());
+ InetSocketAddress listenerAddress = ipcServer.getListenerAddress();
+ LOG.info("Opened IPC server at {}", listenerAddress);
+ dnConf.getConf().set(DFS_DATANODE_IPC_ADDRESS_KEY,
+ listenerAddress.getHostName() + ":" + listenerAddress.getPort());
// set service-level authorization security policy
if (getConf().getBoolean(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index bd8c7378e8a..f2800f2ba56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -303,7 +303,9 @@ public void start() throws IOException {
if (httpServer != null) {
InetSocketAddress infoAddr = DataNode.getInfoAddr(conf);
httpAddress = getChannelLocalAddress(httpServer, infoAddr);
- LOG.info("Listening HTTP traffic on " + httpAddress);
+ conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,
+ NetUtils.getHostPortString(httpAddress));
+ LOG.info("Listening for HTTP traffic on {}", httpAddress);
}
if (httpsServer != null) {
@@ -312,7 +314,9 @@ public void start() throws IOException {
DFS_DATANODE_HTTPS_ADDRESS_KEY,
DFS_DATANODE_HTTPS_ADDRESS_DEFAULT));
httpsAddress = getChannelLocalAddress(httpsServer, secInfoSocAddr);
- LOG.info("Listening HTTPS traffic on " + httpsAddress);
+ conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY,
+ NetUtils.getHostPortString(httpsAddress));
+ LOG.info("Listening for HTTPS traffic on {}", httpsAddress);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index af527df28bf..8a7158d536e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -54,12 +54,17 @@
import org.glassfish.jersey.internal.inject.AbstractBinder;
import org.glassfish.jersey.server.ResourceConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Encapsulates the HTTP server started by the NameNode.
*/
@InterfaceAudience.Private
public class NameNodeHttpServer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NameNodeHttpServer.class);
+
private HttpServer2 httpServer;
private final Configuration conf;
private final NameNode nn;
@@ -185,14 +190,20 @@ void start() throws IOException {
int connIdx = 0;
if (policy.isHttpEnabled()) {
httpAddress = httpServer.getConnectorAddress(connIdx++);
- conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
- NetUtils.getHostPortString(httpAddress));
+ if (httpAddress != null) {
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
+ NetUtils.getHostPortString(httpAddress));
+ LOG.info("Listening for HTTP traffic on {}", httpAddress);
+ }
}
if (policy.isHttpsEnabled()) {
httpsAddress = httpServer.getConnectorAddress(connIdx);
- conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
- NetUtils.getHostPortString(httpsAddress));
+ if (httpsAddress != null) {
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
+ NetUtils.getHostPortString(httpsAddress));
+ LOG.info("Listening for HTTPS traffic on {}", httpsAddress);
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
index 28cba0153c1..b97fe053e7d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
@@ -68,7 +68,7 @@
<script type="text/x-dust-template" id="tmpl-dn">
{#dn}
-<div class="page-header"><h1>DataNode on <small>{HostName}:{DataPort}</small></h1></div>
+<div class="page-header"><h1>DataNode on <small>{HostName}:{RpcPort}</small></h1></div>
<table class="table table-bordered table-striped">
<tr><th>Cluster ID:</th><td>{ClusterId}</td></tr>
<tr><th>Started:</th><td>{DNStartedTimeInMillis|date_tostring}</td></tr>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java
index 1473576177f..ff09cd5c5aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -25,6 +26,7 @@
import java.io.File;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
@@ -167,4 +169,25 @@ public void testMemlockLimit() throws Exception {
prevLimit);
}
}
+
+ @Test
+ public void testDataNodeIpcAndHttpSeverConf() throws Exception {
+ Configuration conf = cluster.getConfiguration(0);
+ DataNode dn = null;
+ try {
+ dn = DataNode.createDataNode(new String[] {}, conf);
+ Configuration dnConf = dn.getConf();
+ InetSocketAddress listenerAddress = dn.ipcServer.getListenerAddress();
+ assertThat(listenerAddress.getHostName() + ":" + listenerAddress.getPort())
+ .describedAs("IPC Address is inconsistent")
+ .isEqualTo(dnConf.get(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY));
+ assertThat(listenerAddress.getHostName() + ":" + dn.getHttpPort())
+ .describedAs("HTTP Address is inconsistent")
+ .isEqualTo(dnConf.get(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY));
+ } finally {
+ if (dn != null) {
+ dn.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org