You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2023/06/12 04:53:59 UTC
[solr] branch branch_9x updated: SOLR-16273: Prometheus Exporter is very slow when collecting large amounts of sample data (#1627)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 2e81cad59ab SOLR-16273: Prometheus Exporter is very slow when collecting large amounts of sample data (#1627)
2e81cad59ab is described below
commit 2e81cad59ab1990e8c80085b7cc2fadb0aa01d6b
Author: Matthew Biscocho <54...@users.noreply.github.com>
AuthorDate: Sun Jun 11 18:42:44 2023 -0400
SOLR-16273: Prometheus Exporter is very slow when collecting large amounts of sample data (#1627)
Replaced a O(N^2) algorithm with a O(1) using a HashSet of seen samples.
---------
Co-authored-by: Christine Poerschke <cp...@apache.org>
---
.../solr/prometheus/collector/MetricSamples.java | 18 ++++++++++--
.../prometheus/collector/MetricSamplesTest.java | 32 +++++++++++++++++++++-
2 files changed, 46 insertions(+), 4 deletions(-)
diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/MetricSamples.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/MetricSamples.java
index 9fae6ce23ed..328d76d054c 100644
--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/MetricSamples.java
+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/MetricSamples.java
@@ -19,24 +19,35 @@ package org.apache.solr.prometheus.collector;
import io.prometheus.client.Collector;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
public class MetricSamples {
private final Map<String, Collector.MetricFamilySamples> samplesByMetricName;
+ private final Set<Collector.MetricFamilySamples.Sample> seenSamples;
+
public MetricSamples(Map<String, Collector.MetricFamilySamples> input) {
samplesByMetricName = input;
+ seenSamples = new HashSet<>();
+ for (Collector.MetricFamilySamples metricFamilySamples : input.values()) {
+ seenSamples.addAll(metricFamilySamples.samples);
+ }
}
public MetricSamples() {
this(new HashMap<>());
}
- public void addSamplesIfNotPresent(String metricName, Collector.MetricFamilySamples samples) {
- samplesByMetricName.putIfAbsent(metricName, samples);
+ public void addSamplesIfNotPresent(
+ String metricName, Collector.MetricFamilySamples metricFamilySamples) {
+ if (samplesByMetricName.putIfAbsent(metricName, metricFamilySamples) == null) {
+ seenSamples.addAll(metricFamilySamples.samples);
+ }
}
public void addSampleIfMetricExists(
@@ -47,7 +58,7 @@ public class MetricSamples {
return;
}
- if (!sampleFamily.samples.contains(sample)) {
+ if (seenSamples.add(sample)) {
sampleFamily.samples.add(sample);
}
}
@@ -62,6 +73,7 @@ public class MetricSamples {
}
} else {
this.samplesByMetricName.put(key, entry.getValue());
+ seenSamples.addAll(entry.getValue().samples);
}
}
}
diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/collector/MetricSamplesTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/collector/MetricSamplesTest.java
index de2d754c8f6..8707e4fc7b3 100644
--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/collector/MetricSamplesTest.java
+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/collector/MetricSamplesTest.java
@@ -90,8 +90,13 @@ public class MetricSamplesTest extends SolrTestCase {
Map.of(
"same",
samples(
- "test1", Collector.Type.GAUGE, sample("test1", 3.0), sample("test1", 4.0)),
+ "test1",
+ Collector.Type.GAUGE,
+ sample("test1", 3.0),
+ sample("test1", 4.0),
+ sample("same", 1.0)),
"diff2", samples("diff2", Collector.Type.GAUGE, sample("diff2", 1.0))));
+
lhs.addAll(rhs);
List<Collector.MetricFamilySamples> output = lhs.asList();
@@ -100,4 +105,29 @@ public class MetricSamplesTest extends SolrTestCase {
validateMetricSamples(output, "diff1", Collections.singletonList(1.0));
validateMetricSamples(output, "diff2", Collections.singletonList(1.0));
}
+
+ @Test
+ public void addSamplesIfNotPresent() {
+
+ MetricSamples testMetricSamples =
+ new MetricSamples(
+ new HashMap<>(
+ Map.of("same", samples("same", Collector.Type.GAUGE, sample("same", 1.0)))));
+
+ Collector.MetricFamilySamples sameSamples =
+ samples("same", Collector.Type.GAUGE, sample("same", 1.0));
+ Collector.MetricFamilySamples newSamples =
+ samples("new", Collector.Type.GAUGE, sample("new", 2.0), sample("new", 3.0));
+ Collector.MetricFamilySamples alreadyPresentSamples =
+ samples("new", Collector.Type.GAUGE, sample("new", 4.0));
+
+ testMetricSamples.addSamplesIfNotPresent("same", sameSamples);
+ testMetricSamples.addSamplesIfNotPresent("new", newSamples);
+ testMetricSamples.addSamplesIfNotPresent("new", alreadyPresentSamples);
+
+ List<Collector.MetricFamilySamples> output = testMetricSamples.asList();
+
+ validateMetricSamples(output, "same", Collections.singletonList(1.0));
+ validateMetricSamples(output, "new", Arrays.asList(2.0, 3.0));
+ }
}