You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2023/12/06 13:12:27 UTC

(streampipes) 01/01: feat: add monitoring metrics for adapters

This is an automated email from the ASF dual-hosted git repository.

bossenti pushed a commit to branch prometheus-docker
in repository https://gitbox.apache.org/repos/asf/streampipes.git

commit ada31173ffed1896436b3b84d919474898af8553
Author: bossenti <bo...@posteo.de>
AuthorDate: Wed Dec 6 14:12:10 2023 +0100

    feat: add monitoring metrics for adapters
---
 .../prometheus/StreamPipesCollectorRegistry.java   |   8 ++
 .../commons/prometheus/adapter/AdapterMetrics.java | 126 +++++++++++++++++++++
 .../prometheus/adapter/AdapterMetricsManager.java} |  29 +++--
 .../prometheus/adapter/AdapterMetricsTest.java     |  63 +++++++++++
 .../commons/resources}/TestResources.java          |   4 +-
 .../management/health/AdapterHealthCheck.java      |  42 ++++++-
 .../management/AdapterMasterManagement.java        |  12 +-
 7 files changed, 268 insertions(+), 16 deletions(-)

diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
index 1795ea03d..242344b1f 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
@@ -43,6 +43,14 @@ public class StreamPipesCollectorRegistry {
       .register(collectorRegistry);
   }
 
+  public static Gauge registerGauge(String name, String help, String... labelNames) {
+    return Gauge.build()
+                .name(name)
+                .help(help)
+                .labelNames(labelNames)
+                .register(collectorRegistry);
+  }
+
   public static Counter registerCounter(String name, String help) {
     return Counter.build()
       .name(name)
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java
new file mode 100644
index 000000000..174531fb6
--- /dev/null
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.commons.prometheus.adapter;
+
+import org.apache.streampipes.commons.prometheus.StreamPipesCollectorRegistry;
+
+import io.prometheus.client.Gauge;
+
+import java.util.HashMap;
+import java.util.NoSuchElementException;
+
+/**
+ * AdapterMetrics class manages monitoring metrics related to StreamPipes adapters.
+ * It allows register and remove adapter instances as well as updating the values of metrics.
+ * Currently, the following metrics are supported:
+ * <ul>
+ *   <li><strong>adapter_events_published_total:</strong> The total number of events published per adapter</li>
+ * </ul>
+ * <p>Usage Example:
+ * <pre>{@code
+ * AdapterMetrics adapterMetrics = new AdapterMetrics();
+ * adapterMetrics.register("adapter-1", "Sample Adapter");
+ * adapterMetrics.updateTotalEventsPublished("adapter-1", "Sample Adapter", 100);
+ * adapterMetrics.remove("adapter-1", "Sample Adapter");
+ * }</pre>
+ */
+public class AdapterMetrics {
+
+  // Map to keep book about the registered adapters and to serve as mapping between adapter id and adapter name
+  // Both are used as dimensions for the metric
+  private final HashMap<String, String> registeredAdapters;
+
+  private static final String ELEMENT_NOT_FOUND_TEMPLATE = "No entry for adapter '%s' found. Please register it first.";
+
+  private final Gauge totalAdapterEventsPublishedMetric;
+
+  public AdapterMetrics() {
+    this.totalAdapterEventsPublishedMetric = StreamPipesCollectorRegistry.registerGauge(
+        "adapter_events_published_total",
+        "Total amount of events published per adapter",
+        "adapterId", "adapterName"
+    );
+    this.registeredAdapters = new HashMap<>();
+  }
+
+  /**
+   * Registers a new adapter with the given adapterId and adapterName.
+   *
+   * @param adapterId   The unique identifier for the adapter
+   * @param adapterName The name of the adapter
+   */
+  public void register(String adapterId, String adapterName) {
+    if (!contains(adapterId)) {
+      this.registeredAdapters.put(adapterId, adapterName);
+    }
+  }
+
+  /**
+   * Checks if an adapter with the specified adapterId is registered.
+   *
+   * @param adapterId The unique identifier for the adapter
+   * @return True if the adapter is registered, false otherwise
+   */
+  public boolean contains(String adapterId) {
+    return this.registeredAdapters.containsKey(adapterId);
+  }
+
+  /**
+   * Updates the total number of events published for a specific adapter.
+   *
+   * @param adapterId            The unique identifier for the adapter
+   * @param adapterName          The name of the adapter
+   * @param totalEventsPublished The total number of events published by the adapter
+   * @throws NoSuchElementException If the adapter is not registered yet
+   */
+  public void updateTotalEventsPublished(String adapterId, String adapterName, long totalEventsPublished) {
+    if (contains(adapterId)) {
+      // Order of labels needs to be preserved and match the one in AdapterMetrics#register
+      totalAdapterEventsPublishedMetric.labels(adapterId, adapterName)
+                                       .set(totalEventsPublished);
+    } else {
+      throw new NoSuchElementException(ELEMENT_NOT_FOUND_TEMPLATE.formatted(adapterId));
+    }
+  }
+
+  /**
+   * Removes the specified adapter from the registry.
+   * This results in no more metrics being published for this adapter.
+   *
+   * @param adapterId   The unique identifier for the adapter
+   * @param adapterName The name of the adapter
+   * @throws NoSuchElementException If the adapter is not found in the registry
+   */
+  public void remove(String adapterId, String adapterName) {
+    if (contains(adapterId)) {
+      this.totalAdapterEventsPublishedMetric.remove(adapterId, adapterName);
+      this.registeredAdapters.remove(adapterId);
+    } else {
+      throw new NoSuchElementException(ELEMENT_NOT_FOUND_TEMPLATE.formatted(adapterId));
+    }
+  }
+
+  /**
+   * Returns the number of registered adapters in the registry.
+   *
+   * @return The number of registered adapters
+   */
+  public int size() {
+    return registeredAdapters.size();
+  }
+}
diff --git a/streampipes-commons/src/test/java/TestResources.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
similarity index 56%
copy from streampipes-commons/src/test/java/TestResources.java
copy to streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
index ec7174e33..8e211ac7e 100644
--- a/streampipes-commons/src/test/java/TestResources.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
@@ -16,21 +16,28 @@
  *
  */
 
-import org.apache.streampipes.commons.resources.Resources;
+package org.apache.streampipes.commons.prometheus.adapter;
 
-import org.junit.Test;
+/**
+ * The `AdapterMetricsManager` enum provides a single access point for retrieving the instance of
+ * the `AdapterMetrics` class and ensures that only one instance of the metrics manager exists throughout the
+ * application's lifecycle.
+ */
+public enum AdapterMetricsManager {
+
+  INSTANCE;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+  private final AdapterMetrics adapterMetrics;
 
-import static junit.framework.TestCase.assertEquals;
+  AdapterMetricsManager() {
+    this.adapterMetrics = new AdapterMetrics();
+  }
 
-public class TestResources {
+  public static AdapterMetricsManager getInstance() {
+    return INSTANCE;
+  }
 
-  @Test
-  public void testResourceLoader() throws IOException {
-    String filename = "test.txt";
-    String content = Resources.asString(filename, StandardCharsets.UTF_8);
-    assertEquals("HelloWorld", content.replaceAll("\\s+",""));
+  public AdapterMetrics getAdapterMetrics() {
+    return adapterMetrics;
   }
 }
diff --git a/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java b/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java
new file mode 100644
index 000000000..7068c5c5c
--- /dev/null
+++ b/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.commons.prometheus.adapter;
+
+import org.junit.Test;
+
+import java.util.NoSuchElementException;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertThrows;
+
+public class AdapterMetricsTest {
+
+  @Test
+  public void register() {
+
+    var metrics = new AdapterMetrics();
+    var adapterId = "adapterId";
+    var adapterName = "adapterName";
+
+    assertEquals(0, metrics.size());
+
+    metrics.register(adapterId, adapterName);
+
+    assertEquals(1, metrics.size());
+    assertTrue(metrics.contains(adapterId));
+  }
+
+  @Test
+  public void removeNoSuchElement() {
+    var metrics = new AdapterMetrics();
+
+    assertThrows(NoSuchElementException.class, () -> metrics.remove("test-id", "test-name"));
+  }
+
+  @Test
+  public void updateTotalEventsPublishedNoSuchElement() {
+    var metrics = new AdapterMetrics();
+
+    assertThrows(
+        NoSuchElementException.class,
+        () -> metrics.updateTotalEventsPublished("test-id", "test-name", 0)
+    );
+
+  }
+}
diff --git a/streampipes-commons/src/test/java/TestResources.java b/streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
similarity index 90%
rename from streampipes-commons/src/test/java/TestResources.java
rename to streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
index ec7174e33..7db525a3a 100644
--- a/streampipes-commons/src/test/java/TestResources.java
+++ b/streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
@@ -16,7 +16,7 @@
  *
  */
 
-import org.apache.streampipes.commons.resources.Resources;
+package org.apache.streampipes.commons.resources;
 
 import org.junit.Test;
 
@@ -31,6 +31,6 @@ public class TestResources {
   public void testResourceLoader() throws IOException {
     String filename = "test.txt";
     String content = Resources.asString(filename, StandardCharsets.UTF_8);
-    assertEquals("HelloWorld", content.replaceAll("\\s+",""));
+    assertEquals("HelloWorld", content.replaceAll("\\s+", ""));
   }
 }
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
index fabf477af..8653ea080 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
@@ -19,9 +19,12 @@
 package org.apache.streampipes.connect.management.health;
 
 import org.apache.streampipes.commons.exceptions.connect.AdapterException;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetrics;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager;
 import org.apache.streampipes.connect.management.management.AdapterMasterManagement;
 import org.apache.streampipes.connect.management.management.WorkerRestClient;
 import org.apache.streampipes.connect.management.util.WorkerPaths;
+import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsLogProvider;
 import org.apache.streampipes.model.connect.adapter.AdapterDescription;
 import org.apache.streampipes.storage.api.IAdapterStorage;
 
@@ -57,12 +60,18 @@ public class AdapterHealthCheck implements Runnable {
    * In this method it is checked which adapters are currently running.
    * Then it calls all workers to validate if the adapter instance is
    * still running as expected. If the adapter is not running anymore a new worker instance is invoked.
+   * In addition, it publishes monitoring metrics for all running adapters (in line with
+   * {@link org.apache.streampipes.manager.health.PipelineHealthCheck}).
    */
   public void checkAndRestoreAdapters() {
-    // Get all adapters
+    // Get all running adapters
     Map<String, AdapterDescription> allRunningInstancesAdapterDescriptions =
         this.getAllRunningInstancesAdapterDescriptions();
 
+    if (!allRunningInstancesAdapterDescriptions.isEmpty()) {
+      updateMonitoringMetrics(allRunningInstancesAdapterDescriptions);
+    }
+
     // Get all worker containers that run adapters
     Map<String, List<AdapterDescription>> groupByWorker =
         this.getAllWorkersWithAdapters(allRunningInstancesAdapterDescriptions);
@@ -75,6 +84,37 @@ public class AdapterHealthCheck implements Runnable {
     this.recoverAdapters(allAdaptersToRecover);
   }
 
+  /**
+   * Updates the monitoring metrics based on the descriptions of running adapters.
+   *
+   * @param runningAdapterDescriptions A map containing the descriptions of running adapters, where the key is the
+   *                                   adapter's element ID and the value is the corresponding adapter description.
+   */
+  protected void updateMonitoringMetrics(Map<String, AdapterDescription> runningAdapterDescriptions) {
+
+    var adapterMetrics = AdapterMetricsManager.getInstance()
+                                              .getAdapterMetrics();
+    runningAdapterDescriptions.values()
+                              .forEach(
+                                  adapterDescription -> updateTotalEventsPublished(
+                                      adapterMetrics,
+                                      adapterDescription.getElementId(),
+                                      adapterDescription.getName()
+                                  ));
+    LOG.info("Monitoring {} adapter instances", adapterMetrics.size());
+  }
+
+  private void updateTotalEventsPublished(AdapterMetrics adapterMetrics, String adapterId, String adapterName) {
+    adapterMetrics.updateTotalEventsPublished(
+        adapterId,
+        adapterName,
+        ExtensionsLogProvider.INSTANCE.getMetricInfosForResource(adapterId)
+                                      .getMessagesOut()
+                                      .getCounter()
+    );
+  }
+
+
   public Map<String, AdapterDescription> getAllRunningInstancesAdapterDescriptions() {
     Map<String, AdapterDescription> result = new HashMap<>();
     List<AdapterDescription> allRunningInstancesAdapterDescription = this.adapterStorage.getAllAdapters();
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
index c8d16e4d3..6490a499f 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
@@ -21,6 +21,7 @@ package org.apache.streampipes.connect.management.management;
 import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException;
 import org.apache.streampipes.commons.exceptions.SepaParseException;
 import org.apache.streampipes.commons.exceptions.connect.AdapterException;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager;
 import org.apache.streampipes.connect.management.util.GroundingUtils;
 import org.apache.streampipes.connect.management.util.WorkerPaths;
 import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsLogProvider;
@@ -32,7 +33,7 @@ import org.apache.streampipes.resource.management.AdapterResourceManager;
 import org.apache.streampipes.resource.management.DataStreamResourceManager;
 import org.apache.streampipes.resource.management.SpResourceManager;
 import org.apache.streampipes.storage.api.IAdapterStorage;
-import org.apache.streampipes.storage.couchdb.impl.AdapterInstanceStorageImpl;
+import org.apache.streampipes.storage.management.StorageDispatcher;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -149,6 +150,10 @@ public class AdapterMasterManagement {
 
     WorkerRestClient.stopStreamAdapter(ad.getSelectedEndpointUrl(), ad);
     ExtensionsLogProvider.INSTANCE.reset(elementId);
+
+    // remove the adapter from the metrics manager so that
+    // no metrics for this adapter are exposed anymore
+    AdapterMetricsManager.INSTANCE.getAdapterMetrics().remove(ad.getElementId(), ad.getName());
   }
 
   public void startStreamAdapter(String elementId) throws AdapterException {
@@ -166,6 +171,9 @@ public class AdapterMasterManagement {
       // Invoke adapter instance
       WorkerRestClient.invokeStreamAdapter(baseUrl, elementId);
 
+      // register the adapter at the metrics manager so that the AdapterHealthCheck can send metrics
+      AdapterMetricsManager.INSTANCE.getAdapterMetrics().register(ad.getElementId(), ad.getName());
+
       LOG.info("Started adapter " + elementId + " on: " + baseUrl);
     } catch (NoServiceEndpointsAvailableException | URISyntaxException e) {
       throw new AdapterException("Could not start adapter due to unavailable service endpoint", e);
@@ -184,6 +192,6 @@ public class AdapterMasterManagement {
   }
 
   private IAdapterStorage getAdapterInstanceStorage() {
-    return new AdapterInstanceStorageImpl();
+    return StorageDispatcher.INSTANCE.getNoSqlStore().getAdapterInstanceStorage();
   }
 }