You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2021/03/24 17:04:16 UTC
[ozone] branch master updated: HDDS-5010. Fixing Prometheus
duplicated TYPE definition (#2074)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 520ba00 HDDS-5010. Fixing Prometheus duplicated TYPE definition (#2074)
520ba00 is described below
commit 520ba00da07f82707b8e3c8e1e91ee324f71353a
Author: Gyorgy Ruck <40...@users.noreply.github.com>
AuthorDate: Wed Mar 24 18:03:53 2021 +0100
HDDS-5010. Fixing Prometheus duplicated TYPE definition (#2074)
---
.../hdds/server/http/PrometheusMetricsSink.java | 90 ++++++------
.../server/http/TestPrometheusMetricsSink.java | 163 ++++++++++++---------
2 files changed, 143 insertions(+), 110 deletions(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/PrometheusMetricsSink.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/PrometheusMetricsSink.java
index 1bd4734..3a6a079 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/PrometheusMetricsSink.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/PrometheusMetricsSink.java
@@ -21,10 +21,11 @@ import static org.apache.hadoop.hdds.utils.RocksDBStoreMBean.ROCKSDB_CONTEXT_PRE
import java.io.IOException;
import java.io.Writer;
+import java.util.Collections;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.TreeMap;
import java.util.regex.Pattern;
-
+import org.apache.commons.configuration2.SubsetConfiguration;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricType;
@@ -32,8 +33,6 @@ import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsSink;
import org.apache.hadoop.metrics2.MetricsTag;
-import org.apache.commons.configuration2.SubsetConfiguration;
-
/**
* Metrics sink for prometheus exporter.
* <p>
@@ -44,7 +43,8 @@ public class PrometheusMetricsSink implements MetricsSink {
/**
* Cached output lines for each metrics.
*/
- private final Map<String, String> metricLines = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, String>> metricLines =
+ Collections.synchronizedSortedMap(new TreeMap<>());
private static final Pattern SPLIT_PATTERN =
Pattern.compile("(?<!(^|[A-Z_]))(?=[A-Z])|(?<!^)(?=[A-Z][a-z])");
@@ -64,43 +64,45 @@ public class PrometheusMetricsSink implements MetricsSink {
String key = prometheusName(
metricsRecord.name(), metrics.name());
- StringBuilder builder = new StringBuilder();
- builder.append("# TYPE ")
- .append(key)
- .append(" ")
- .append(metrics.type().toString().toLowerCase())
- .append("\n");
-
- StringBuilder prometheusMetricKey = new StringBuilder();
- prometheusMetricKey.append(key)
- .append("{");
- String sep = "";
-
- //add tags
- for (MetricsTag tag : metricsRecord.tags()) {
- String tagName = tag.name().toLowerCase();
-
- //ignore specific tag which includes sub-hierarchy
- if (!tagName.equals("numopenconnectionsperuser")) {
- prometheusMetricKey.append(sep)
- .append(tagName)
- .append("=\"")
- .append(tag.value())
- .append("\"");
- sep = ",";
- }
- }
- prometheusMetricKey.append("}");
-
- String prometheusMetricKeyAsString = prometheusMetricKey.toString();
- builder.append(prometheusMetricKeyAsString);
- builder.append(" ");
- builder.append(metrics.value());
- builder.append("\n");
- metricLines.put(prometheusMetricKeyAsString, builder.toString());
+ String prometheusMetricKeyAsString =
+ getPrometheusMetricKeyAsString(metricsRecord, key);
+
+ String metricKey = "# TYPE "
+ + key
+ + " "
+ + metrics.type().toString().toLowerCase();
+
+ metricLines.computeIfAbsent(metricKey,
+ any -> Collections.synchronizedSortedMap(new TreeMap<>()))
+ .put(prometheusMetricKeyAsString, String.valueOf(metrics.value()));
+ }
+ }
+ }
+ private String getPrometheusMetricKeyAsString(MetricsRecord metricsRecord,
+ String key) {
+ StringBuilder prometheusMetricKey = new StringBuilder();
+ prometheusMetricKey.append(key)
+ .append("{");
+ String sep = "";
+
+ //add tags
+ for (MetricsTag tag : metricsRecord.tags()) {
+ String tagName = tag.name().toLowerCase();
+
+ //ignore specific tag which includes sub-hierarchy
+ if (!tagName.equals("numopenconnectionsperuser")) {
+ prometheusMetricKey.append(sep)
+ .append(tagName)
+ .append("=\"")
+ .append(tag.value())
+ .append("\"");
+ sep = ",";
}
}
+ prometheusMetricKey.append("}");
+
+ return prometheusMetricKey.toString();
}
/**
@@ -139,8 +141,14 @@ public class PrometheusMetricsSink implements MetricsSink {
}
public void writeMetrics(Writer writer) throws IOException {
- for (String line : metricLines.values()) {
- writer.write(line);
+ for (Map.Entry<String, Map<String, String>> metricsEntry
+ : metricLines.entrySet()) {
+ writer.write(metricsEntry.getKey() + "\n");
+
+ for (Map.Entry<String, String> metrics
+ : metricsEntry.getValue().entrySet()) {
+ writer.write(metrics.getKey() + " " + metrics.getValue() + "\n");
+ }
}
}
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestPrometheusMetricsSink.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestPrometheusMetricsSink.java
index 5f3ffce..98ab896 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestPrometheusMetricsSink.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestPrometheusMetricsSink.java
@@ -17,10 +17,12 @@
*/
package org.apache.hadoop.hdds.server.http;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
-
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.MetricsSystem;
@@ -29,9 +31,9 @@ import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
/**
@@ -39,67 +41,85 @@ import org.junit.Test;
*/
public class TestPrometheusMetricsSink {
- @Test
- public void testPublish() throws IOException {
- //GIVEN
- MetricsSystem metrics = DefaultMetricsSystem.instance();
+ private MetricsSystem metrics;
+ private PrometheusMetricsSink sink;
+
+ private static final MetricsInfo PORT_INFO = new MetricsInfo() {
+ @Override
+ public String name() {
+ return "PORT";
+ }
+
+ @Override
+ public String description() {
+ return "port";
+ }
+ };
+
+ private static final MetricsInfo COUNTER_INFO = new MetricsInfo() {
+ @Override
+ public String name() {
+ return "COUNTER";
+ }
+
+ @Override
+ public String description() {
+ return "counter";
+ }
+ };
+
+ private static final int COUNTER_1 = 123;
+ private static final int COUNTER_2 = 234;
+
+ @Before
+ public void init() {
+ metrics = DefaultMetricsSystem.instance();
metrics.init("test");
- PrometheusMetricsSink sink = new PrometheusMetricsSink();
+ sink = new PrometheusMetricsSink();
metrics.register("Prometheus", "Prometheus", sink);
+ }
+
+ @After
+ public void tearDown() {
+ metrics.stop();
+ metrics.shutdown();
+ }
+
+ @Test
+ public void testPublish() throws IOException {
+ //GIVEN
TestMetrics testMetrics = metrics
.register("TestMetrics", "Testing metrics", new TestMetrics());
- metrics.start();
testMetrics.numBucketCreateFails.incr();
- metrics.publishMetricsNow();
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- OutputStreamWriter writer = new OutputStreamWriter(stream, UTF_8);
//WHEN
- sink.writeMetrics(writer);
- writer.flush();
+ String writtenMetrics = publishMetricsAndGetOutput();
//THEN
- String writtenMetrics = stream.toString(UTF_8.name());
Assert.assertTrue(
"The expected metric line is missing from prometheus metrics output",
writtenMetrics.contains(
"test_metrics_num_bucket_create_fails{context=\"dfs\"")
);
-
- metrics.stop();
- metrics.shutdown();
}
@Test
public void testPublishWithSameName() throws IOException {
//GIVEN
- MetricsSystem metrics = DefaultMetricsSystem.instance();
-
- metrics.init("test");
- PrometheusMetricsSink sink = new PrometheusMetricsSink();
- metrics.register("Prometheus", "Prometheus", sink);
metrics.register("FooBar", "fooBar", (MetricsSource) (collector, all) -> {
collector.addRecord("RpcMetrics").add(new MetricsTag(PORT_INFO, "1234"))
- .addGauge(COUNTER_INFO, 123).endRecord();
+ .addGauge(COUNTER_INFO, COUNTER_1).endRecord();
collector.addRecord("RpcMetrics").add(new MetricsTag(
- PORT_INFO, "2345")).addGauge(COUNTER_INFO, 234).endRecord();
+ PORT_INFO, "2345")).addGauge(COUNTER_INFO, COUNTER_2).endRecord();
});
- metrics.start();
- metrics.publishMetricsNow();
+ // WHEN
+ String writtenMetrics = publishMetricsAndGetOutput();
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- OutputStreamWriter writer = new OutputStreamWriter(stream, UTF_8);
-
- //WHEN
- sink.writeMetrics(writer);
- writer.flush();
-
- //THEN
- String writtenMetrics = stream.toString(UTF_8.name());
+ // THEN
Assert.assertTrue(
"The expected metric line is missing from prometheus metrics output",
writtenMetrics.contains(
@@ -110,15 +130,30 @@ public class TestPrometheusMetricsSink {
+ "output",
writtenMetrics.contains(
"rpc_metrics_counter{port=\"1234\""));
+ }
- metrics.stop();
- metrics.shutdown();
+ @Test
+ public void testTypeWithSameNameButDifferentLabels() throws IOException {
+ //GIVEN
+ metrics.register("SameName", "sameName",
+ (MetricsSource) (collector, all) -> {
+ collector.addRecord("SameName").add(new MetricsTag(PORT_INFO, "1234"))
+ .addGauge(COUNTER_INFO, COUNTER_1).endRecord();
+ collector.addRecord("SameName").add(new MetricsTag(PORT_INFO, "2345"))
+ .addGauge(COUNTER_INFO, COUNTER_2).endRecord();
+ });
+
+ // WHEN
+ String writtenMetrics = publishMetricsAndGetOutput();
+
+ // THEN
+ Assert.assertEquals(1, StringUtils.countMatches(writtenMetrics,
+ "# TYPE same_name_counter"));
}
@Test
public void testNamingCamelCase() {
- PrometheusMetricsSink sink = new PrometheusMetricsSink();
-
+ //THEN
Assert.assertEquals("rpc_time_some_metrics",
sink.prometheusName("RpcTime", "SomeMetrics"));
@@ -132,18 +167,19 @@ public class TestPrometheusMetricsSink {
@Test
public void testNamingRocksDB() {
//RocksDB metrics are handled differently.
- PrometheusMetricsSink sink = new PrometheusMetricsSink();
+ // THEN
Assert.assertEquals("rocksdb_om_db_num_open_connections",
sink.prometheusName("Rocksdb_om.db", "num_open_connections"));
}
@Test
public void testNamingPipeline() {
- PrometheusMetricsSink sink = new PrometheusMetricsSink();
-
+ // GIVEN
String recordName = "SCMPipelineMetrics";
String metricName = "NumBlocksAllocated-"
+ "RATIS-THREE-47659e3d-40c9-43b3-9792-4982fc279aba";
+
+ // THEN
Assert.assertEquals(
"scm_pipeline_metrics_"
+ "num_blocks_allocated_"
@@ -153,15 +189,28 @@ public class TestPrometheusMetricsSink {
@Test
public void testNamingSpaces() {
- PrometheusMetricsSink sink = new PrometheusMetricsSink();
-
+ //GIVEN
String recordName = "JvmMetrics";
String metricName = "GcTimeMillisG1 Young Generation";
+
+ // THEN
Assert.assertEquals(
"jvm_metrics_gc_time_millis_g1_young_generation",
sink.prometheusName(recordName, metricName));
}
+ private String publishMetricsAndGetOutput() throws IOException {
+ metrics.publishMetricsNow();
+
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ OutputStreamWriter writer = new OutputStreamWriter(stream, UTF_8);
+
+ sink.writeMetrics(writer);
+ writer.flush();
+
+ return stream.toString(UTF_8.name());
+ }
+
/**
* Example metric pojo.
*/
@@ -172,28 +221,4 @@ public class TestPrometheusMetricsSink {
private MutableCounterLong numBucketCreateFails;
}
- public static final MetricsInfo PORT_INFO = new MetricsInfo() {
- @Override
- public String name() {
- return "PORT";
- }
-
- @Override
- public String description() {
- return "port";
- }
- };
-
- public static final MetricsInfo COUNTER_INFO = new MetricsInfo() {
- @Override
- public String name() {
- return "COUNTER";
- }
-
- @Override
- public String description() {
- return "counter";
- }
- };
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org