You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2022/08/24 03:31:41 UTC
[hbase] branch branch-2 updated: HBASE-20904 Prometheus /metrics http endpoint for monitoring (#4691)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new c04edf78359 HBASE-20904 Prometheus /metrics http endpoint for monitoring (#4691)
c04edf78359 is described below
commit c04edf78359f7181aa53fce525d5fcdbcf8209bd
Author: Luca Kovács <ko...@gmail.com>
AuthorDate: Wed Aug 24 04:54:27 2022 +0200
HBASE-20904 Prometheus /metrics http endpoint for monitoring (#4691)
Co-authored-by: Luca Kovacs <ko...@gmail.com>
Co-authored-by: Madhusoodan P <ak...@gmail.com>
Co-authored-by: Luca Kovacs <lk...@cloudera.com>
Signed-off-by: Duo Zhang <zh...@apache.org>
(cherry picked from commit f9ea7ee0d6614e30bccd51b9a920aa80830dddf4)
Conflicts:
src/main/asciidoc/_chapters/ops_mgt.adoc
---
hbase-common/src/main/resources/hbase-default.xml | 8 +++
.../hadoop/metrics2/impl/MetricsExportHelper.java | 42 +++++++++++
.../hbase/metrics/TestMetricsExportHelper.java | 75 +++++++++++++++++++
hbase-http/pom.xml | 13 ++++
.../org/apache/hadoop/hbase/http/HttpServer.java | 41 ++++++++---
.../apache/hadoop/hbase/http/ServletConfig.java | 47 ++++++++++++
.../http/prometheus/PrometheusHadoopServlet.java | 84 ++++++++++++++++++++++
.../http/prometheus/TestPrometheusServlet.java | 84 ++++++++++++++++++++++
8 files changed, 383 insertions(+), 11 deletions(-)
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index f3a243a6e80..c0212607d56 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -1768,6 +1768,14 @@ possible configurations would overwhelm and obscure the important.
ThreadPool.
</description>
</property>
+ <property>
+ <name>hbase.http.metrics.servlets</name>
+ <value>jmx,metrics,prometheus</value>
+ <description>
+ Comma separated list of servlet names to enable for metrics collection. Supported
+ servlets are jmx, metrics, prometheus
+ </description>
+ </property>
<property>
<name>hbase.replication.rpc.codec</name>
<value>org.apache.hadoop.hbase.codec.KeyValueCodecWithTags</value>
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/impl/MetricsExportHelper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/impl/MetricsExportHelper.java
new file mode 100644
index 00000000000..9232bcc1765
--- /dev/null
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/impl/MetricsExportHelper.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics2.impl;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public final class MetricsExportHelper {
+ private MetricsExportHelper() {
+ }
+
+ public static Collection<MetricsRecord> export() {
+ MetricsSystemImpl instance = (MetricsSystemImpl) DefaultMetricsSystem.instance();
+ MetricsBuffer metricsBuffer = instance.sampleMetrics();
+ List<MetricsRecord> metrics = new ArrayList<>();
+ for (MetricsBuffer.Entry entry : metricsBuffer) {
+ entry.records().forEach(metrics::add);
+ }
+ return metrics;
+ }
+
+}
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsExportHelper.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsExportHelper.java
new file mode 100644
index 00000000000..3a86dd04966
--- /dev/null
+++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsExportHelper.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import java.util.Collection;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.MetricsTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.impl.MetricsExportHelper;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MetricsTests.class, SmallTests.class })
+public class TestMetricsExportHelper {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMetricsExportHelper.class);
+
+ @Test
+ public void testExportHelper() {
+ DefaultMetricsSystem.initialize("exportHelperTestSystem");
+ DefaultMetricsSystem.instance().start();
+
+ String metricsName = "exportMetricsTestGrp";
+ String gaugeName = "exportMetricsTestGauge";
+ String counterName = "exportMetricsTestCounter";
+
+ BaseSourceImpl baseSource = new BaseSourceImpl(metricsName, "", metricsName, metricsName);
+
+ baseSource.setGauge(gaugeName, 0);
+ baseSource.incCounters(counterName, 1);
+
+ Collection<MetricsRecord> metrics = MetricsExportHelper.export();
+ DefaultMetricsSystem.instance().stop();
+
+ Assert.assertTrue(metrics.stream().anyMatch(mr -> mr.name().equals(metricsName)));
+ Assert.assertTrue(gaugeName + " is missing in the export",
+ contains(metrics, metricsName, gaugeName));
+ Assert.assertTrue(counterName + " is missing in the export",
+ contains(metrics, metricsName, counterName));
+ }
+
+ private boolean contains(Collection<MetricsRecord> metrics, String metricsName,
+ String metricName) {
+ return metrics.stream().filter(mr -> mr.name().equals(metricsName)).anyMatch(mr -> {
+ for (AbstractMetric metric : mr.metrics()) {
+ if (metric.name().equals(metricName)) {
+ return true;
+ }
+ }
+ return false;
+ });
+ }
+}
diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml
index ee71b1e0a48..2a682431620 100644
--- a/hbase-http/pom.xml
+++ b/hbase-http/pom.xml
@@ -57,6 +57,19 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-metrics-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-metrics</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ </dependency>
<!-- resource bundle only needed at build time -->
<dependency>
<groupId>org.apache.hbase</groupId>
diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
index 8c45d4f8aca..50cefc4c39a 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.http.conf.ConfServlet;
-import org.apache.hadoop.hbase.http.jmx.JMXJsonServlet;
import org.apache.hadoop.hbase.http.log.LogLevel;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.hbase.util.Threads;
@@ -70,6 +69,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion;
import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler;
@@ -154,6 +154,18 @@ public class HttpServer implements FilterContainer {
public static final String NO_CACHE_FILTER = "NoCacheFilter";
public static final String APP_DIR = "webapps";
+ public static final String METRIC_SERVLETS_CONF_KEY = "hbase.http.metrics.servlets";
+ public static final String[] METRICS_SERVLETS_DEFAULT = { "jmx", "metrics", "prometheus" };
+ private static final ImmutableMap<String,
+ ServletConfig> METRIC_SERVLETS = new ImmutableMap.Builder<String, ServletConfig>()
+ .put("jmx",
+ new ServletConfig("jmx", "/jmx", "org.apache.hadoop.hbase.http.jmx.JMXJsonServlet"))
+ .put("metrics",
+ new ServletConfig("metrics", "/metrics", "org.apache.hadoop.metrics.MetricsServlet"))
+ .put("prometheus", new ServletConfig("prometheus", "/prometheus",
+ "org.apache.hadoop.hbase.http.prometheus.PrometheusHadoopServlet"))
+ .build();
+
private final AccessControlList adminsAcl;
protected final Server webServer;
@@ -751,16 +763,7 @@ public class HttpServer implements FilterContainer {
// set up default servlets
addPrivilegedServlet("stacks", "/stacks", StackServlet.class);
addPrivilegedServlet("logLevel", "/logLevel", LogLevel.Servlet.class);
- // Hadoop3 has moved completely to metrics2, and dropped support for Metrics v1's
- // MetricsServlet (see HADOOP-12504). We'll using reflection to load if against hadoop2.
- // Remove when we drop support for hbase on hadoop2.x.
- try {
- Class<?> clz = Class.forName("org.apache.hadoop.metrics.MetricsServlet");
- addPrivilegedServlet("metrics", "/metrics", clz.asSubclass(HttpServlet.class));
- } catch (Exception e) {
- // do nothing
- }
- addPrivilegedServlet("jmx", "/jmx", JMXJsonServlet.class);
+
// While we don't expect users to have sensitive information in their configuration, they
// might. Give them an option to not expose the service configuration to all users.
if (conf.getBoolean(HTTP_PRIVILEGED_CONF_KEY, HTTP_PRIVILEGED_CONF_DEFAULT)) {
@@ -784,6 +787,22 @@ public class HttpServer implements FilterContainer {
LOG.info("ASYNC_PROFILER_HOME environment variable and async.profiler.home system property "
+ "not specified. Disabling /prof endpoint.");
}
+
+ /* register metrics servlets */
+ String[] enabledServlets = conf.getStrings(METRIC_SERVLETS_CONF_KEY, METRICS_SERVLETS_DEFAULT);
+ for (String enabledServlet : enabledServlets) {
+ try {
+ ServletConfig servletConfig = METRIC_SERVLETS.get(enabledServlet);
+ if (servletConfig != null) {
+ Class<?> clz = Class.forName(servletConfig.getClazz());
+ addPrivilegedServlet(servletConfig.getName(), servletConfig.getPathSpec(),
+ clz.asSubclass(HttpServlet.class));
+ }
+ } catch (Exception e) {
+ /* shouldn't be fatal, so warn the user about it */
+ LOG.warn("Couldn't register the servlet " + enabledServlet, e);
+ }
+ }
}
/**
diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ServletConfig.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ServletConfig.java
new file mode 100644
index 00000000000..befe6095760
--- /dev/null
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ServletConfig.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.http;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+/* pojo to hold the servlet info */
+
+@InterfaceAudience.Private
+class ServletConfig {
+ private String name;
+ private String pathSpec;
+ private String clazz;
+
+ public ServletConfig(String name, String pathSpec, String clazz) {
+ this.name = name;
+ this.pathSpec = pathSpec;
+ this.clazz = clazz;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getPathSpec() {
+ return pathSpec;
+ }
+
+ public String getClazz() {
+ return clazz;
+ }
+}
diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/prometheus/PrometheusHadoopServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/prometheus/PrometheusHadoopServlet.java
new file mode 100644
index 00000000000..c6e13b37c66
--- /dev/null
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/prometheus/PrometheusHadoopServlet.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.http.prometheus;
+
+import com.google.errorprone.annotations.RestrictedApi;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Collection;
+import java.util.regex.Pattern;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricType;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.apache.hadoop.metrics2.impl.MetricsExportHelper;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class PrometheusHadoopServlet extends HttpServlet {
+
+ private static final Pattern SPLIT_PATTERN =
+ Pattern.compile("(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=([A-Z][a-z]))|\\W|(_)+");
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
+ writeMetrics(resp.getWriter());
+ }
+
+ static String toPrometheusName(String metricRecordName, String metricName) {
+ String baseName = metricRecordName + StringUtils.capitalize(metricName);
+ String[] parts = SPLIT_PATTERN.split(baseName);
+ return String.join("_", parts).toLowerCase();
+ }
+
+ /*
+ * SimpleClient for Prometheus is not used, because the format is very easy to implement and this
+ * solution doesn't add any dependencies to the project. You can check the Prometheus format here:
+ * https://prometheus.io/docs/instrumenting/exposition_formats/
+ */
+ @RestrictedApi(explanation = "Should only be called in tests or self", link = "",
+ allowedOnPath = ".*/src/test/.*|.*/PrometheusHadoopServlet\\.java")
+ void writeMetrics(Writer writer) throws IOException {
+ Collection<MetricsRecord> metricRecords = MetricsExportHelper.export();
+ for (MetricsRecord metricsRecord : metricRecords) {
+ for (AbstractMetric metrics : metricsRecord.metrics()) {
+ if (metrics.type() == MetricType.COUNTER || metrics.type() == MetricType.GAUGE) {
+
+ String key = toPrometheusName(metricsRecord.name(), metrics.name());
+ writer.append("# TYPE ").append(key).append(" ")
+ .append(metrics.type().toString().toLowerCase()).append("\n").append(key).append("{");
+
+ /* add tags */
+ String sep = "";
+ for (MetricsTag tag : metricsRecord.tags()) {
+ String tagName = tag.name().toLowerCase();
+ writer.append(sep).append(tagName).append("=\"").append(tag.value()).append("\"");
+ sep = ",";
+ }
+ writer.append("} ");
+ writer.append(metrics.value().toString()).append('\n');
+ }
+ }
+ }
+ writer.flush();
+ }
+}
diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/prometheus/TestPrometheusServlet.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/prometheus/TestPrometheusServlet.java
new file mode 100644
index 00000000000..fcfde82ad41
--- /dev/null
+++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/prometheus/TestPrometheusServlet.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.http.prometheus;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test prometheus Sink.
+ */
+@Category({ SmallTests.class, MiscTests.class })
+public class TestPrometheusServlet {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_TEST_RULE =
+ HBaseClassTestRule.forClass(TestPrometheusServlet.class);
+
+ @Test
+ public void testPublish() throws IOException {
+ // GIVEN
+ MetricsSystem metrics = DefaultMetricsSystem.instance();
+ metrics.init("test");
+ TestMetrics testMetrics = metrics.register("TestMetrics", "Testing metrics", new TestMetrics());
+ metrics.start();
+
+ testMetrics.numBucketCreateFails.incr();
+ metrics.publishMetricsNow();
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ OutputStreamWriter writer = new OutputStreamWriter(stream, UTF_8);
+
+ // WHEN
+ PrometheusHadoopServlet prom2Servlet = new PrometheusHadoopServlet();
+ prom2Servlet.writeMetrics(writer);
+
+ // THEN
+ String writtenMetrics = stream.toString(UTF_8.name());
+ System.out.println(writtenMetrics);
+ Assert.assertTrue("The expected metric line is missing from prometheus metrics output",
+ writtenMetrics.contains("test_metrics_num_bucket_create_fails{context=\"dfs\""));
+
+ metrics.stop();
+ metrics.shutdown();
+ }
+
+ /**
+ * Example metric pojo.
+ */
+ @Metrics(about = "Test Metrics", context = "dfs")
+ private static class TestMetrics {
+
+ @Metric
+ private MutableCounterLong numBucketCreateFails;
+ }
+}