You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2026/04/03 06:26:40 UTC

(hadoop) branch branch-3.5 updated: HDFS-17888. Fix MiniCluster startup & HttpServer service's conf issues. (#8342)

This is an automated email from the ASF dual-hosted git repository.

ayushsaxena pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.5 by this push:
     new 8cc7a77d866 HDFS-17888. Fix MiniCluster startup & HttpServer service's conf issues. (#8342)
8cc7a77d866 is described below

commit 8cc7a77d866e859c0cd9cb13d9610ffbfdca43f8
Author: Zhaobo Huang <hu...@126.com>
AuthorDate: Fri Apr 3 14:26:34 2026 +0800

    HDFS-17888. Fix MiniCluster startup & HttpServer service's conf issues. (#8342)
---
 .../org/apache/hadoop/test/GenericTestUtils.java   |  5 +++--
 .../hadoop/hdfs/server/datanode/DataNode.java      |  5 ++++-
 .../server/datanode/web/DatanodeHttpServer.java    |  8 ++++++--
 .../hdfs/server/namenode/NameNodeHttpServer.java   | 19 ++++++++++++++----
 .../src/main/webapps/datanode/datanode.html        |  2 +-
 .../org/apache/hadoop/hdfs/TestDatanodeConfig.java | 23 ++++++++++++++++++++++
 6 files changed, 52 insertions(+), 10 deletions(-)

diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index ada8a9cdb29..65c6c8e2cd5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -221,8 +221,9 @@ public static File getTestDir() {
       prop = DEFAULT_TEST_DATA_DIR;
     }
     File dir = new File(prop).getAbsoluteFile();
-    dir.mkdirs();
-    assertExists(dir);
+    if (dir.mkdirs() && !dir.exists()) {
+      throw new IllegalStateException("Directory " + dir + " not created");
+    }
     return dir;
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 3a1b1e07f36..7ee9a4b1729 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1562,7 +1562,10 @@ private void initIpcServer() throws IOException {
     DFSUtil.addInternalPBProtocol(getConf(), InterDatanodeProtocolPB.class, service,
         ipcServer);
 
-    LOG.info("Opened IPC server at {}", ipcServer.getListenerAddress());
+    InetSocketAddress listenerAddress = ipcServer.getListenerAddress();
+    LOG.info("Opened IPC server at {}", listenerAddress);
+    dnConf.getConf().set(DFS_DATANODE_IPC_ADDRESS_KEY,
+        listenerAddress.getHostName() + ":" + listenerAddress.getPort());
 
     // set service-level authorization security policy
     if (getConf().getBoolean(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index bd8c7378e8a..f2800f2ba56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -303,7 +303,9 @@ public void start() throws IOException {
     if (httpServer != null) {
       InetSocketAddress infoAddr = DataNode.getInfoAddr(conf);
       httpAddress = getChannelLocalAddress(httpServer, infoAddr);
-      LOG.info("Listening HTTP traffic on " + httpAddress);
+      conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,
+          NetUtils.getHostPortString(httpAddress));
+      LOG.info("Listening for HTTP traffic on {}", httpAddress);
     }
 
     if (httpsServer != null) {
@@ -312,7 +314,9 @@ public void start() throws IOException {
               DFS_DATANODE_HTTPS_ADDRESS_KEY,
               DFS_DATANODE_HTTPS_ADDRESS_DEFAULT));
       httpsAddress = getChannelLocalAddress(httpsServer, secInfoSocAddr);
-      LOG.info("Listening HTTPS traffic on " + httpsAddress);
+      conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY,
+          NetUtils.getHostPortString(httpsAddress));
+      LOG.info("Listening for HTTPS traffic on {}", httpsAddress);
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index af527df28bf..8a7158d536e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -54,12 +54,17 @@
 
 import org.glassfish.jersey.internal.inject.AbstractBinder;
 import org.glassfish.jersey.server.ResourceConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Encapsulates the HTTP server started by the NameNode. 
  */
 @InterfaceAudience.Private
 public class NameNodeHttpServer {
+
+  private static final Logger LOG = LoggerFactory.getLogger(NameNodeHttpServer.class);
+
   private HttpServer2 httpServer;
   private final Configuration conf;
   private final NameNode nn;
@@ -185,14 +190,20 @@ void start() throws IOException {
     int connIdx = 0;
     if (policy.isHttpEnabled()) {
       httpAddress = httpServer.getConnectorAddress(connIdx++);
-      conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
-          NetUtils.getHostPortString(httpAddress));
+      if (httpAddress != null) {
+        conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
+            NetUtils.getHostPortString(httpAddress));
+        LOG.info("Listening for HTTP traffic on {}", httpAddress);
+      }
     }
 
     if (policy.isHttpsEnabled()) {
       httpsAddress = httpServer.getConnectorAddress(connIdx);
-      conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
-          NetUtils.getHostPortString(httpsAddress));
+      if (httpsAddress != null) {
+        conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
+            NetUtils.getHostPortString(httpsAddress));
+        LOG.info("Listening for HTTPS traffic on {}", httpsAddress);
+      }
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
index 28cba0153c1..b97fe053e7d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
@@ -68,7 +68,7 @@
 
 <script type="text/x-dust-template" id="tmpl-dn">
 {#dn}
-<div class="page-header"><h1>DataNode on <small>{HostName}:{DataPort}</small></h1></div>
+<div class="page-header"><h1>DataNode on <small>{HostName}:{RpcPort}</small></h1></div>
 <table class="table table-bordered table-striped">
   <tr><th>Cluster ID:</th><td>{ClusterId}</td></tr>
   <tr><th>Started:</th><td>{DNStartedTimeInMillis|date_tostring}</td></tr>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java
index 1473576177f..ff09cd5c5aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs;
 
 import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
@@ -25,6 +26,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 
@@ -167,4 +169,25 @@ public void testMemlockLimit() throws Exception {
           prevLimit);
     }
   }
+
+  @Test
+  public void testDataNodeIpcAndHttpSeverConf() throws Exception {
+    Configuration conf = cluster.getConfiguration(0);
+    DataNode dn = null;
+    try {
+      dn = DataNode.createDataNode(new String[] {}, conf);
+      Configuration dnConf = dn.getConf();
+      InetSocketAddress listenerAddress = dn.ipcServer.getListenerAddress();
+      assertThat(listenerAddress.getHostName() + ":" + listenerAddress.getPort())
+          .describedAs("IPC Address is inconsistent")
+          .isEqualTo(dnConf.get(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY));
+      assertThat(listenerAddress.getHostName() + ":" + dn.getHttpPort())
+          .describedAs("HTTP Address is inconsistent")
+          .isEqualTo(dnConf.get(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY));
+    } finally {
+      if (dn != null) {
+        dn.shutdown();
+      }
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org