You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aj...@apache.org on 2019/03/05 20:05:03 UTC
[hadoop] branch trunk updated: HDDS-919. Enable prometheus
endpoints for Ozone datanodes (#502)
This is an automated email from the ASF dual-hosted git repository.
ajay pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7f636b4 HDDS-919. Enable prometheus endpoints for Ozone datanodes (#502)
7f636b4 is described below
commit 7f636b47b5e430e340c8fd731fd9e35e22f9ddbf
Author: Elek, Márton <el...@users.noreply.github.com>
AuthorDate: Tue Mar 5 21:04:57 2019 +0100
HDDS-919. Enable prometheus endpoints for Ozone datanodes (#502)
Pull request from Elek, Márton. (#502)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 20 +++++
.../common/src/main/resources/ozone-default.xml | 66 ++++++++++++++++-
.../hadoop/ozone/HddsDatanodeHttpServer.java | 86 ++++++++++++++++++++++
.../apache/hadoop/ozone/HddsDatanodeService.java | 16 ++++
.../main/resources/webapps/hddsDatanode/.gitkeep | 17 +++++
.../src/main/compose/ozone/docker-compose.yaml | 1 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 1 +
7 files changed, 206 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index c265771..184e3d8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -227,4 +227,24 @@ public final class HddsConfigKeys {
public static final String HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL =
"hdds.security.client.scm.certificate.protocol.acl";
+ public static final String HDDS_DATANODE_HTTP_ENABLED_KEY =
+ "hdds.datanode.http.enabled";
+ public static final String HDDS_DATANODE_HTTP_BIND_HOST_KEY =
+ "hdds.datanode.http-bind-host";
+ public static final String HDDS_DATANODE_HTTPS_BIND_HOST_KEY =
+ "hdds.datanode.https-bind-host";
+ public static final String HDDS_DATANODE_HTTP_ADDRESS_KEY =
+ "hdds.datanode.http-address";
+ public static final String HDDS_DATANODE_HTTPS_ADDRESS_KEY =
+ "hdds.datanode.https-address";
+
+ public static final String HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
+ public static final int HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT = 9882;
+ public static final int HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT = 9883;
+ public static final String
+ HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY =
+ "hdds.datanode.http.kerberos.principal";
+ public static final String
+ HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY =
+ "hdds.datanode.http.kerberos.keytab";
}
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index f7fecb7..a4f49e7 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2030,6 +2030,70 @@
</property>
<property>
+ <name>hdds.datanode.http.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ <tag>HDDS, SECURITY, MANAGEMENT</tag>
+ <description>
+ The kerberos principal for the datanode http server.
+ </description>
+ </property>
+ <property>
+ <name>hdds.datanode.http.kerberos.keytab</name>
+ <value>/etc/security/keytabs/HTTP.keytab</value>
+ <tag>HDDS, SECURITY, MANAGEMENT</tag>
+ <description>
+ The kerberos keytab file for datanode http server
+ </description>
+ </property>
+ <property>
+ <name>hdds.datanode.http-address</name>
+ <value>0.0.0.0:9882</value>
+ <tag>HDDS, MANAGEMENT</tag>
+ <description>
+ The address and the base port where the Datanode web ui will listen on.
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+ <property>
+ <name>hdds.datanode.http-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>HDDS, MANAGEMENT</tag>
+ <description>
+ The actual address the Datanode web server will bind to. If this
+ optional address is set, it overrides only the hostname portion of
+ hdds.datanode.http-address.
+ </description>
+ </property>
+ <property>
+ <name>hdds.datanode.http.enabled</name>
+ <value>true</value>
+ <tag>HDDS, MANAGEMENT</tag>
+ <description>
+ Property to enable or disable Datanode web ui.
+ </description>
+ </property>
+ <property>
+ <name>hdds.datanode.https-address</name>
+ <value>0.0.0.0:9883</value>
+ <tag>HDDS, MANAGEMENT, SECURITY</tag>
+ <description>
+ The address and the base port where the Datanode web UI will listen
+ on using HTTPS.
+
+ If the port is 0 then the server will start on a free port.
+ </description>
+ </property>
+ <property>
+ <name>hdds.datanode.https-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>HDDS, MANAGEMENT, SECURITY</tag>
+ <description>
+ The actual address the Datanode web server will bind to using HTTPS.
+ If this optional address is set, it overrides only the hostname portion of
+ hdds.datanode.http-address.
+ </description>
+ </property>
+ <property>
<name>ozone.client.retry.max.attempts</name>
<value>10</value>
<description>
@@ -2070,4 +2134,4 @@
milliseconds.
</description>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java
new file mode 100644
index 0000000..3dcfcfe
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.server.BaseHttpServer;
+
+/**
+ * Simple http server to provide basic monitoring for hdds datanode.
+ * <p>
+ * This server is used to access default /conf /prom /prof endpoints.
+ */
+public class HddsDatanodeHttpServer extends BaseHttpServer {
+
+ public HddsDatanodeHttpServer(Configuration conf) throws IOException {
+ super(conf, "hddsDatanode");
+ }
+
+ @Override
+ protected String getHttpAddressKey() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY;
+ }
+
+ @Override
+ protected String getHttpBindHostKey() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_HOST_KEY;
+ }
+
+ @Override
+ protected String getHttpsAddressKey() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTPS_ADDRESS_KEY;
+ }
+
+ @Override
+ protected String getHttpsBindHostKey() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTPS_BIND_HOST_KEY;
+ }
+
+ @Override
+ protected String getBindHostDefault() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT;
+ }
+
+ @Override
+ protected int getHttpBindPortDefault() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT;
+ }
+
+ @Override
+ protected int getHttpsBindPortDefault() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT;
+ }
+
+ @Override
+ protected String getKeytabFile() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY;
+ }
+
+ @Override
+ protected String getSpnegoPrincipal() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY;
+ }
+
+ @Override
+ protected String getEnabledKey() {
+ return HddsConfigKeys.HDDS_DATANODE_HTTP_ENABLED_KEY;
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 3a92a4a..3c205e6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
@@ -67,6 +68,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
private DatanodeDetails datanodeDetails;
private DatanodeStateMachine datanodeStateMachine;
private List<ServicePlugin> plugins;
+ private HddsDatanodeHttpServer httpServer;
/**
* Default constructor.
@@ -180,6 +182,12 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
LOG.info("Hdds Datanode login successful.");
}
datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf);
+ try {
+ httpServer = new HddsDatanodeHttpServer(conf);
+ httpServer.start();
+ } catch (Exception ex) {
+ LOG.error("HttpServer failed to start.", ex);
+ }
startPlugins();
// Starting HDDS Daemons
datanodeStateMachine.startDaemon();
@@ -294,6 +302,14 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
if (datanodeStateMachine != null) {
datanodeStateMachine.stopDaemon();
}
+ if (httpServer != null) {
+ try {
+ httpServer.stop();
+ } catch (Exception e) {
+ LOG.error("Stopping HttpServer is failed.", e);
+ }
+ }
+
}
@Override
diff --git a/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/.gitkeep b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/.gitkeep
new file mode 100644
index 0000000..ff1232e
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/.gitkeep
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
index 487c4eb..9110592 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
@@ -23,6 +23,7 @@ services:
- ../..:/opt/hadoop
ports:
- 9864
+ - 9882
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- ./docker-config
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 0ed8e8b..dbeb0b5 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -577,6 +577,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
private void configureHddsDatanodes() {
conf.set(ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+ conf.set(HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
conf.set(HDDS_DATANODE_PLUGINS_KEY,
"org.apache.hadoop.ozone.web.OzoneHddsDatanodeService");
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org