You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ta...@apache.org on 2021/04/19 11:43:00 UTC
[hadoop] branch trunk updated: HDFS-15970. Print network topology
on the web (#2896)
This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 85a3532 HDFS-15970. Print network topology on the web (#2896)
85a3532 is described below
commit 85a3532849d81ca929bc52cb7ca905c4f53652ec
Author: litao <to...@gmail.com>
AuthorDate: Mon Apr 19 19:42:30 2021 +0800
HDFS-15970. Print network topology on the web (#2896)
---
.../hdfs/server/namenode/NameNodeHttpServer.java | 6 +-
.../server/namenode/NetworkTopologyServlet.java | 187 +++++++++++++++++++
.../src/main/webapps/hdfs/dfshealth.html | 1 +
.../src/main/webapps/hdfs/explorer.html | 1 +
.../namenode/TestNetworkTopologyServlet.java | 202 +++++++++++++++++++++
5 files changed, 395 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index 3391322..7ca5241 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -166,7 +166,7 @@ public class NameNodeHttpServer {
httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
- setupServlets(httpServer, conf);
+ setupServlets(httpServer);
httpServer.start();
int connIdx = 0;
@@ -243,7 +243,7 @@ public class NameNodeHttpServer {
httpServer.setAttribute(ALIASMAP_ATTRIBUTE_KEY, aliasMap);
}
- private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
+ private static void setupServlets(HttpServer2 httpServer) {
httpServer.addInternalServlet("startupProgress",
StartupProgressServlet.PATH_SPEC, StartupProgressServlet.class);
httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class,
@@ -253,6 +253,8 @@ public class NameNodeHttpServer {
httpServer.addInternalServlet(IsNameNodeActiveServlet.SERVLET_NAME,
IsNameNodeActiveServlet.PATH_SPEC,
IsNameNodeActiveServlet.class);
+ httpServer.addInternalServlet("topology",
+ NetworkTopologyServlet.PATH_SPEC, NetworkTopologyServlet.class);
}
static FSImage getFsImageFromContext(ServletContext context) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java
new file mode 100644
index 0000000..5d08971
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.util.StringUtils;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.HttpHeaders;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+/**
+ * A servlet to print out the network topology.
+ */
+@InterfaceAudience.Private
+public class NetworkTopologyServlet extends DfsServlet {
+
+ public static final String PATH_SPEC = "/topology";
+
+ protected static final String FORMAT_JSON = "json";
+ protected static final String FORMAT_TEXT = "text";
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws IOException {
+ final ServletContext context = getServletContext();
+
+ String format = parseAcceptHeader(request);
+ if (FORMAT_TEXT.equals(format)) {
+ response.setContentType("text/plain; charset=UTF-8");
+ } else if (FORMAT_JSON.equals(format)) {
+ response.setContentType("application/json; charset=UTF-8");
+ }
+
+ NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context);
+ BlockManager bm = nn.getNamesystem().getBlockManager();
+ List<Node> leaves = bm.getDatanodeManager().getNetworkTopology()
+ .getLeaves(NodeBase.ROOT);
+
+ try (PrintStream out = new PrintStream(
+ response.getOutputStream(), false, "UTF-8")) {
+ printTopology(out, leaves, format);
+ } catch (Throwable t) {
+ String errMsg = "Print network topology failed. "
+ + StringUtils.stringifyException(t);
+ response.sendError(HttpServletResponse.SC_GONE, errMsg);
+ throw new IOException(errMsg);
+ } finally {
+ response.getOutputStream().close();
+ }
+ }
+
+ /**
+ * Display each rack and the nodes assigned to that rack, as determined
+ * by the NameNode, in a hierarchical manner. The nodes and racks are
+ * sorted alphabetically.
+ *
+ * @param stream print stream
+ * @param leaves leaves nodes under base scope
+ * @param format the response format
+ */
+ public void printTopology(PrintStream stream, List<Node> leaves,
+ String format) throws BadFormatException, IOException {
+ if (leaves.isEmpty()) {
+ stream.print("No DataNodes");
+ return;
+ }
+
+ // Build a map of rack -> nodes
+ Map<String, TreeSet<String>> tree = new HashMap<>();
+ for(Node dni : leaves) {
+ String location = dni.getNetworkLocation();
+ String name = dni.getName();
+
+ tree.putIfAbsent(location, new TreeSet<>());
+ tree.get(location).add(name);
+ }
+
+ // Sort the racks (and nodes) alphabetically, display in order
+ ArrayList<String> racks = new ArrayList<>(tree.keySet());
+ Collections.sort(racks);
+
+ if (FORMAT_JSON.equals(format)) {
+ printJsonFormat(stream, tree, racks);
+ } else if (FORMAT_TEXT.equals(format)) {
+ printTextFormat(stream, tree, racks);
+ } else {
+ throw new BadFormatException("Bad format: " + format);
+ }
+ }
+
+ private void printJsonFormat(PrintStream stream, Map<String,
+ TreeSet<String>> tree, ArrayList<String> racks) throws IOException {
+ JsonFactory dumpFactory = new JsonFactory();
+ JsonGenerator dumpGenerator = dumpFactory.createGenerator(stream);
+ dumpGenerator.writeStartArray();
+
+ for(String r : racks) {
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeFieldName(r);
+ TreeSet<String> nodes = tree.get(r);
+ dumpGenerator.writeStartArray();
+
+ for(String n : nodes) {
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeStringField("ip", n);
+ String hostname = NetUtils.getHostNameOfIP(n);
+ if(hostname != null) {
+ dumpGenerator.writeStringField("hostname", hostname);
+ }
+ dumpGenerator.writeEndObject();
+ }
+ dumpGenerator.writeEndArray();
+ dumpGenerator.writeEndObject();
+ }
+ dumpGenerator.writeEndArray();
+ dumpGenerator.flush();
+
+ if (!dumpGenerator.isClosed()) {
+ dumpGenerator.close();
+ }
+ }
+
+ private void printTextFormat(PrintStream stream, Map<String,
+ TreeSet<String>> tree, ArrayList<String> racks) {
+ for(String r : racks) {
+ stream.println("Rack: " + r);
+ TreeSet<String> nodes = tree.get(r);
+
+ for(String n : nodes) {
+ stream.print(" " + n);
+ String hostname = NetUtils.getHostNameOfIP(n);
+ if(hostname != null) {
+ stream.print(" (" + hostname + ")");
+ }
+ stream.println();
+ }
+ stream.println();
+ }
+ }
+
+ @VisibleForTesting
+ static String parseAcceptHeader(HttpServletRequest request) {
+ String format = request.getHeader(HttpHeaders.ACCEPT);
+ return format != null && format.contains(FORMAT_JSON) ?
+ FORMAT_JSON : FORMAT_TEXT;
+ }
+
+ public static class BadFormatException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public BadFormatException(String msg) {
+ super(msg);
+ }
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 6e4eade..8622e4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -52,6 +52,7 @@
<li><a href="jmx">Metrics</a></li>
<li><a href="conf">Configuration</a></li>
<li><a href="stacks">Process Thread Dump</a></li>
+ <li><a href="topology">Network Topology</a></li>
</ul>
</li>
</ul>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
index 73bfbd4..3f0509a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
@@ -48,6 +48,7 @@
<li><a href="jmx">Metrics</a></li>
<li><a href="conf">Configuration</a></li>
<li><a href="stacks">Process Thread Dump</a></li>
+ <li><a href="topology">Network Topology</a></li>
</ul>
</li>
</ul>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java
new file mode 100644
index 0000000..7796ed4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.StaticMapping;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestNetworkTopologyServlet {
+
+ @Test
+ public void testPrintTopologyTextFormat() throws IOException {
+ StaticMapping.resetMap();
+ Configuration conf = new HdfsConfiguration();
+ int dataNodesNum = 0;
+ final ArrayList<String> rackList = new ArrayList<String>();
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j < 2; j++) {
+ rackList.add("/rack" + i);
+ dataNodesNum++;
+ }
+ }
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(dataNodesNum)
+ .racks(rackList.toArray(new String[rackList.size()]))
+ .build();
+ cluster.waitActive();
+
+ // get http uri
+ String httpUri = cluster.getHttpUri(0);
+
+ // send http request
+ URL url = new URL(httpUri + "/topology");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setReadTimeout(20000);
+ conn.setConnectTimeout(20000);
+ conn.connect();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+ StringBuilder sb =
+ new StringBuilder("-- Network Topology -- \n");
+ sb.append(out);
+ sb.append("\n-- Network Topology -- ");
+ String topology = sb.toString();
+
+ // assert rack info
+ assertTrue(topology.contains("/rack0"));
+ assertTrue(topology.contains("/rack1"));
+ assertTrue(topology.contains("/rack2"));
+ assertTrue(topology.contains("/rack3"));
+ assertTrue(topology.contains("/rack4"));
+
+ // assert node number
+ assertEquals(topology.split("127.0.0.1").length - 1,
+ dataNodesNum);
+ }
+
+ @Test
+ public void testPrintTopologyJsonFormat() throws IOException {
+ StaticMapping.resetMap();
+ Configuration conf = new HdfsConfiguration();
+ int dataNodesNum = 0;
+ final ArrayList<String> rackList = new ArrayList<String>();
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j < 2; j++) {
+ rackList.add("/rack" + i);
+ dataNodesNum++;
+ }
+ }
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(dataNodesNum)
+ .racks(rackList.toArray(new String[rackList.size()]))
+ .build();
+ cluster.waitActive();
+
+ // get http uri
+ String httpUri = cluster.getHttpUri(0);
+
+ // send http request
+ URL url = new URL(httpUri + "/topology");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setReadTimeout(20000);
+ conn.setConnectTimeout(20000);
+ conn.setRequestProperty("Accept", "application/json");
+ conn.connect();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+ String topology = out.toString();
+
+ // parse json
+ JsonNode racks = new ObjectMapper().readTree(topology);
+
+ // assert rack number
+ assertEquals(racks.size(), 5);
+
+ // assert node number
+ Iterator<JsonNode> elements = racks.elements();
+ int dataNodesCount = 0;
+ while(elements.hasNext()){
+ JsonNode rack = elements.next();
+ Iterator<Map.Entry<String, JsonNode>> fields = rack.fields();
+ while (fields.hasNext()) {
+ dataNodesCount += fields.next().getValue().size();
+ }
+ }
+ assertEquals(dataNodesCount, dataNodesNum);
+ }
+
+ @Test
+ public void testPrintTopologyNoDatanodesTextFormat() throws IOException {
+ StaticMapping.resetMap();
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0)
+ .build();
+ cluster.waitActive();
+
+ // get http uri
+ String httpUri = cluster.getHttpUri(0);
+
+ // send http request
+ URL url = new URL(httpUri + "/topology");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setReadTimeout(20000);
+ conn.setConnectTimeout(20000);
+ conn.connect();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+ StringBuilder sb =
+ new StringBuilder("-- Network Topology -- \n");
+ sb.append(out);
+ sb.append("\n-- Network Topology -- ");
+ String topology = sb.toString();
+
+ // assert node number
+ assertTrue(topology.contains("No DataNodes"));
+ }
+
+ @Test
+ public void testPrintTopologyNoDatanodesJsonFormat() throws IOException {
+ StaticMapping.resetMap();
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0)
+ .build();
+ cluster.waitActive();
+
+ // get http uri
+ String httpUri = cluster.getHttpUri(0);
+
+ // send http request
+ URL url = new URL(httpUri + "/topology");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setReadTimeout(20000);
+ conn.setConnectTimeout(20000);
+ conn.setRequestProperty("Accept", "application/json");
+ conn.connect();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+ StringBuilder sb =
+ new StringBuilder("-- Network Topology -- \n");
+ sb.append(out);
+ sb.append("\n-- Network Topology -- ");
+ String topology = sb.toString();
+
+ // assert node number
+ assertTrue(topology.contains("No DataNodes"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org