You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2023/12/06 13:12:27 UTC
(streampipes) 01/01: feat: add monitoring metrics for adapters
This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch prometheus-docker
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit ada31173ffed1896436b3b84d919474898af8553
Author: bossenti <bo...@posteo.de>
AuthorDate: Wed Dec 6 14:12:10 2023 +0100
feat: add monitoring metrics for adapters
---
.../prometheus/StreamPipesCollectorRegistry.java | 8 ++
.../commons/prometheus/adapter/AdapterMetrics.java | 126 +++++++++++++++++++++
.../prometheus/adapter/AdapterMetricsManager.java} | 29 +++--
.../prometheus/adapter/AdapterMetricsTest.java | 63 +++++++++++
.../commons/resources}/TestResources.java | 4 +-
.../management/health/AdapterHealthCheck.java | 42 ++++++-
.../management/AdapterMasterManagement.java | 12 +-
7 files changed, 268 insertions(+), 16 deletions(-)
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
index 1795ea03d..242344b1f 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
@@ -43,6 +43,14 @@ public class StreamPipesCollectorRegistry {
.register(collectorRegistry);
}
+ public static Gauge registerGauge(String name, String help, String... labelNames) {
+ return Gauge.build()
+ .name(name)
+ .help(help)
+ .labelNames(labelNames)
+ .register(collectorRegistry);
+ }
+
public static Counter registerCounter(String name, String help) {
return Counter.build()
.name(name)
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java
new file mode 100644
index 000000000..174531fb6
--- /dev/null
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.commons.prometheus.adapter;
+
+import org.apache.streampipes.commons.prometheus.StreamPipesCollectorRegistry;
+
+import io.prometheus.client.Gauge;
+
+import java.util.HashMap;
+import java.util.NoSuchElementException;
+
+/**
+ * AdapterMetrics class manages monitoring metrics related to StreamPipes adapters.
+ * It allows register and remove adapter instances as well as updating the values of metrics.
+ * Currently, the following metrics are supported:
+ * <ul>
+ * <li><strong>adapter_events_published_total:</strong> The total number of events published per adapter</li>
+ * </ul>
+ * <p>Usage Example:
+ * <pre>{@code
+ * AdapterMetrics adapterMetrics = new AdapterMetrics();
+ * adapterMetrics.register("adapter-1", "Sample Adapter");
+ * adapterMetrics.updateTotalEventsPublished("adapter-1", "Sample Adapter", 100);
+ * adapterMetrics.remove("adapter-1", "Sample Adapter");
+ * }</pre>
+ */
+public class AdapterMetrics {
+
+ // Map to keep book about the registered adapters and to serve as mapping between adapter id and adapter name
+ // Both are used as dimensions for the metric
+ private final HashMap<String, String> registeredAdapters;
+
+ private static final String ELEMENT_NOT_FOUND_TEMPLATE = "No entry for adapter '%s' found. Please register it first.";
+
+ private final Gauge totalAdapterEventsPublishedMetric;
+
+ public AdapterMetrics() {
+ this.totalAdapterEventsPublishedMetric = StreamPipesCollectorRegistry.registerGauge(
+ "adapter_events_published_total",
+ "Total amount of events published per adapter",
+ "adapterId", "adapterName"
+ );
+ this.registeredAdapters = new HashMap<>();
+ }
+
+ /**
+ * Registers a new adapter with the given adapterId and adapterName.
+ *
+ * @param adapterId The unique identifier for the adapter
+ * @param adapterName The name of the adapter
+ */
+ public void register(String adapterId, String adapterName) {
+ if (!contains(adapterId)) {
+ this.registeredAdapters.put(adapterId, adapterName);
+ }
+ }
+
+ /**
+ * Checks if an adapter with the specified adapterId is registered.
+ *
+ * @param adapterId The unique identifier for the adapter
+ * @return True if the adapter is registered, false otherwise
+ */
+ public boolean contains(String adapterId) {
+ return this.registeredAdapters.containsKey(adapterId);
+ }
+
+ /**
+ * Updates the total number of events published for a specific adapter.
+ *
+ * @param adapterId The unique identifier for the adapter
+ * @param adapterName The name of the adapter
+ * @param totalEventsPublished The total number of events published by the adapter
+ * @throws NoSuchElementException If the adapter is not registered yet
+ */
+ public void updateTotalEventsPublished(String adapterId, String adapterName, long totalEventsPublished) {
+ if (contains(adapterId)) {
+ // Order of labels needs to be preserved and match the one in AdapterMetrics#register
+ totalAdapterEventsPublishedMetric.labels(adapterId, adapterName)
+ .set(totalEventsPublished);
+ } else {
+ throw new NoSuchElementException(ELEMENT_NOT_FOUND_TEMPLATE.formatted(adapterId));
+ }
+ }
+
+ /**
+ * Removes the specified adapter from the registry.
+ * This results in no more metrics being published for this adapter.
+ *
+ * @param adapterId The unique identifier for the adapter
+ * @param adapterName The name of the adapter
+ * @throws NoSuchElementException If the adapter is not found in the registry
+ */
+ public void remove(String adapterId, String adapterName) {
+ if (contains(adapterId)) {
+ this.totalAdapterEventsPublishedMetric.remove(adapterId, adapterName);
+ this.registeredAdapters.remove(adapterId);
+ } else {
+ throw new NoSuchElementException(ELEMENT_NOT_FOUND_TEMPLATE.formatted(adapterId));
+ }
+ }
+
+ /**
+ * Returns the number of registered adapters in the registry.
+ *
+ * @return The number of registered adapters
+ */
+ public int size() {
+ return registeredAdapters.size();
+ }
+}
diff --git a/streampipes-commons/src/test/java/TestResources.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
similarity index 56%
copy from streampipes-commons/src/test/java/TestResources.java
copy to streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
index ec7174e33..8e211ac7e 100644
--- a/streampipes-commons/src/test/java/TestResources.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
@@ -16,21 +16,28 @@
*
*/
-import org.apache.streampipes.commons.resources.Resources;
+package org.apache.streampipes.commons.prometheus.adapter;
-import org.junit.Test;
+/**
+ * The `AdapterMetricsManager` enum provides a single access point for retrieving the instance of
+ * the `AdapterMetrics` class and ensures that only one instance of the metrics manager exists throughout the
+ * application's lifecycle.
+ */
+public enum AdapterMetricsManager {
+
+ INSTANCE;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+ private final AdapterMetrics adapterMetrics;
-import static junit.framework.TestCase.assertEquals;
+ AdapterMetricsManager() {
+ this.adapterMetrics = new AdapterMetrics();
+ }
-public class TestResources {
+ public static AdapterMetricsManager getInstance() {
+ return INSTANCE;
+ }
- @Test
- public void testResourceLoader() throws IOException {
- String filename = "test.txt";
- String content = Resources.asString(filename, StandardCharsets.UTF_8);
- assertEquals("HelloWorld", content.replaceAll("\\s+",""));
+ public AdapterMetrics getAdapterMetrics() {
+ return adapterMetrics;
}
}
diff --git a/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java b/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java
new file mode 100644
index 000000000..7068c5c5c
--- /dev/null
+++ b/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.commons.prometheus.adapter;
+
+import org.junit.Test;
+
+import java.util.NoSuchElementException;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertThrows;
+
+public class AdapterMetricsTest {
+
+ @Test
+ public void register() {
+
+ var metrics = new AdapterMetrics();
+ var adapterId = "adapterId";
+ var adapterName = "adapterName";
+
+ assertEquals(0, metrics.size());
+
+ metrics.register(adapterId, adapterName);
+
+ assertEquals(1, metrics.size());
+ assertTrue(metrics.contains(adapterId));
+ }
+
+ @Test
+ public void removeNoSuchElement() {
+ var metrics = new AdapterMetrics();
+
+ assertThrows(NoSuchElementException.class, () -> metrics.remove("test-id", "test-name"));
+ }
+
+ @Test
+ public void updateTotalEventsPublishedNoSuchElement() {
+ var metrics = new AdapterMetrics();
+
+ assertThrows(
+ NoSuchElementException.class,
+ () -> metrics.updateTotalEventsPublished("test-id", "test-name", 0)
+ );
+
+ }
+}
diff --git a/streampipes-commons/src/test/java/TestResources.java b/streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
similarity index 90%
rename from streampipes-commons/src/test/java/TestResources.java
rename to streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
index ec7174e33..7db525a3a 100644
--- a/streampipes-commons/src/test/java/TestResources.java
+++ b/streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
@@ -16,7 +16,7 @@
*
*/
-import org.apache.streampipes.commons.resources.Resources;
+package org.apache.streampipes.commons.resources;
import org.junit.Test;
@@ -31,6 +31,6 @@ public class TestResources {
public void testResourceLoader() throws IOException {
String filename = "test.txt";
String content = Resources.asString(filename, StandardCharsets.UTF_8);
- assertEquals("HelloWorld", content.replaceAll("\\s+",""));
+ assertEquals("HelloWorld", content.replaceAll("\\s+", ""));
}
}
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
index fabf477af..8653ea080 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
@@ -19,9 +19,12 @@
package org.apache.streampipes.connect.management.health;
import org.apache.streampipes.commons.exceptions.connect.AdapterException;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetrics;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager;
import org.apache.streampipes.connect.management.management.AdapterMasterManagement;
import org.apache.streampipes.connect.management.management.WorkerRestClient;
import org.apache.streampipes.connect.management.util.WorkerPaths;
+import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsLogProvider;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
import org.apache.streampipes.storage.api.IAdapterStorage;
@@ -57,12 +60,18 @@ public class AdapterHealthCheck implements Runnable {
* In this method it is checked which adapters are currently running.
* Then it calls all workers to validate if the adapter instance is
* still running as expected. If the adapter is not running anymore a new worker instance is invoked.
+ * In addition, it publishes monitoring metrics for all running adapters (in line with
+ * {@link org.apache.streampipes.manager.health.PipelineHealthCheck}).
*/
public void checkAndRestoreAdapters() {
- // Get all adapters
+ // Get all running adapters
Map<String, AdapterDescription> allRunningInstancesAdapterDescriptions =
this.getAllRunningInstancesAdapterDescriptions();
+ if (!allRunningInstancesAdapterDescriptions.isEmpty()) {
+ updateMonitoringMetrics(allRunningInstancesAdapterDescriptions);
+ }
+
// Get all worker containers that run adapters
Map<String, List<AdapterDescription>> groupByWorker =
this.getAllWorkersWithAdapters(allRunningInstancesAdapterDescriptions);
@@ -75,6 +84,37 @@ public class AdapterHealthCheck implements Runnable {
this.recoverAdapters(allAdaptersToRecover);
}
+ /**
+ * Updates the monitoring metrics based on the descriptions of running adapters.
+ *
+ * @param runningAdapterDescriptions A map containing the descriptions of running adapters, where the key is the
+ * adapter's element ID and the value is the corresponding adapter description.
+ */
+ protected void updateMonitoringMetrics(Map<String, AdapterDescription> runningAdapterDescriptions) {
+
+ var adapterMetrics = AdapterMetricsManager.getInstance()
+ .getAdapterMetrics();
+ runningAdapterDescriptions.values()
+ .forEach(
+ adapterDescription -> updateTotalEventsPublished(
+ adapterMetrics,
+ adapterDescription.getElementId(),
+ adapterDescription.getName()
+ ));
+ LOG.info("Monitoring {} adapter instances", adapterMetrics.size());
+ }
+
+ private void updateTotalEventsPublished(AdapterMetrics adapterMetrics, String adapterId, String adapterName) {
+ adapterMetrics.updateTotalEventsPublished(
+ adapterId,
+ adapterName,
+ ExtensionsLogProvider.INSTANCE.getMetricInfosForResource(adapterId)
+ .getMessagesOut()
+ .getCounter()
+ );
+ }
+
+
public Map<String, AdapterDescription> getAllRunningInstancesAdapterDescriptions() {
Map<String, AdapterDescription> result = new HashMap<>();
List<AdapterDescription> allRunningInstancesAdapterDescription = this.adapterStorage.getAllAdapters();
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
index c8d16e4d3..6490a499f 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
@@ -21,6 +21,7 @@ package org.apache.streampipes.connect.management.management;
import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException;
import org.apache.streampipes.commons.exceptions.SepaParseException;
import org.apache.streampipes.commons.exceptions.connect.AdapterException;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager;
import org.apache.streampipes.connect.management.util.GroundingUtils;
import org.apache.streampipes.connect.management.util.WorkerPaths;
import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsLogProvider;
@@ -32,7 +33,7 @@ import org.apache.streampipes.resource.management.AdapterResourceManager;
import org.apache.streampipes.resource.management.DataStreamResourceManager;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.IAdapterStorage;
-import org.apache.streampipes.storage.couchdb.impl.AdapterInstanceStorageImpl;
+import org.apache.streampipes.storage.management.StorageDispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -149,6 +150,10 @@ public class AdapterMasterManagement {
WorkerRestClient.stopStreamAdapter(ad.getSelectedEndpointUrl(), ad);
ExtensionsLogProvider.INSTANCE.reset(elementId);
+
+ // remove the adapter from the metrics manager so that
+ // no metrics for this adapter are exposed anymore
+ AdapterMetricsManager.INSTANCE.getAdapterMetrics().remove(ad.getElementId(), ad.getName());
}
public void startStreamAdapter(String elementId) throws AdapterException {
@@ -166,6 +171,9 @@ public class AdapterMasterManagement {
// Invoke adapter instance
WorkerRestClient.invokeStreamAdapter(baseUrl, elementId);
+ // register the adapter at the metrics manager so that the AdapterHealthCheck can send metrics
+ AdapterMetricsManager.INSTANCE.getAdapterMetrics().register(ad.getElementId(), ad.getName());
+
LOG.info("Started adapter " + elementId + " on: " + baseUrl);
} catch (NoServiceEndpointsAvailableException | URISyntaxException e) {
throw new AdapterException("Could not start adapter due to unavailable service endpoint", e);
@@ -184,6 +192,6 @@ public class AdapterMasterManagement {
}
private IAdapterStorage getAdapterInstanceStorage() {
- return new AdapterInstanceStorageImpl();
+ return StorageDispatcher.INSTANCE.getNoSqlStore().getAdapterInstanceStorage();
}
}