You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2023/12/06 13:12:26 UTC

(streampipes) branch prometheus-docker created (now ada31173f)

This is an automated email from the ASF dual-hosted git repository.

bossenti pushed a change to branch prometheus-docker
in repository https://gitbox.apache.org/repos/asf/streampipes.git


      at ada31173f feat: add monitoring metrics for adapters

This branch includes the following new commits:

     new ada31173f feat: add monitoring metrics for adapters

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(streampipes) 01/01: feat: add monitoring metrics for adapters

Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bossenti pushed a commit to branch prometheus-docker
in repository https://gitbox.apache.org/repos/asf/streampipes.git

commit ada31173ffed1896436b3b84d919474898af8553
Author: bossenti <bo...@posteo.de>
AuthorDate: Wed Dec 6 14:12:10 2023 +0100

    feat: add monitoring metrics for adapters
---
 .../prometheus/StreamPipesCollectorRegistry.java   |   8 ++
 .../commons/prometheus/adapter/AdapterMetrics.java | 126 +++++++++++++++++++++
 .../prometheus/adapter/AdapterMetricsManager.java} |  29 +++--
 .../prometheus/adapter/AdapterMetricsTest.java     |  63 +++++++++++
 .../commons/resources}/TestResources.java          |   4 +-
 .../management/health/AdapterHealthCheck.java      |  42 ++++++-
 .../management/AdapterMasterManagement.java        |  12 +-
 7 files changed, 268 insertions(+), 16 deletions(-)

diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
index 1795ea03d..242344b1f 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/StreamPipesCollectorRegistry.java
@@ -43,6 +43,14 @@ public class StreamPipesCollectorRegistry {
       .register(collectorRegistry);
   }
 
+  public static Gauge registerGauge(String name, String help, String... labelNames) {
+    return Gauge.build()
+                .name(name)
+                .help(help)
+                .labelNames(labelNames)
+                .register(collectorRegistry);
+  }
+
   public static Counter registerCounter(String name, String help) {
     return Counter.build()
       .name(name)
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java
new file mode 100644
index 000000000..174531fb6
--- /dev/null
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetrics.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.commons.prometheus.adapter;
+
+import org.apache.streampipes.commons.prometheus.StreamPipesCollectorRegistry;
+
+import io.prometheus.client.Gauge;
+
+import java.util.HashMap;
+import java.util.NoSuchElementException;
+
+/**
+ * AdapterMetrics class manages monitoring metrics related to StreamPipes adapters.
+ * It allows register and remove adapter instances as well as updating the values of metrics.
+ * Currently, the following metrics are supported:
+ * <ul>
+ *   <li><strong>adapter_events_published_total:</strong> The total number of events published per adapter</li>
+ * </ul>
+ * <p>Usage Example:
+ * <pre>{@code
+ * AdapterMetrics adapterMetrics = new AdapterMetrics();
+ * adapterMetrics.register("adapter-1", "Sample Adapter");
+ * adapterMetrics.updateTotalEventsPublished("adapter-1", "Sample Adapter", 100);
+ * adapterMetrics.remove("adapter-1", "Sample Adapter");
+ * }</pre>
+ */
+public class AdapterMetrics {
+
+  // Map to keep book about the registered adapters and to serve as mapping between adapter id and adapter name
+  // Both are used as dimensions for the metric
+  private final HashMap<String, String> registeredAdapters;
+
+  private static final String ELEMENT_NOT_FOUND_TEMPLATE = "No entry for adapter '%s' found. Please register it first.";
+
+  private final Gauge totalAdapterEventsPublishedMetric;
+
+  public AdapterMetrics() {
+    this.totalAdapterEventsPublishedMetric = StreamPipesCollectorRegistry.registerGauge(
+        "adapter_events_published_total",
+        "Total amount of events published per adapter",
+        "adapterId", "adapterName"
+    );
+    this.registeredAdapters = new HashMap<>();
+  }
+
+  /**
+   * Registers a new adapter with the given adapterId and adapterName.
+   *
+   * @param adapterId   The unique identifier for the adapter
+   * @param adapterName The name of the adapter
+   */
+  public void register(String adapterId, String adapterName) {
+    if (!contains(adapterId)) {
+      this.registeredAdapters.put(adapterId, adapterName);
+    }
+  }
+
+  /**
+   * Checks if an adapter with the specified adapterId is registered.
+   *
+   * @param adapterId The unique identifier for the adapter
+   * @return True if the adapter is registered, false otherwise
+   */
+  public boolean contains(String adapterId) {
+    return this.registeredAdapters.containsKey(adapterId);
+  }
+
+  /**
+   * Updates the total number of events published for a specific adapter.
+   *
+   * @param adapterId            The unique identifier for the adapter
+   * @param adapterName          The name of the adapter
+   * @param totalEventsPublished The total number of events published by the adapter
+   * @throws NoSuchElementException If the adapter is not registered yet
+   */
+  public void updateTotalEventsPublished(String adapterId, String adapterName, long totalEventsPublished) {
+    if (contains(adapterId)) {
+      // Order of labels needs to be preserved and match the one in AdapterMetrics#register
+      totalAdapterEventsPublishedMetric.labels(adapterId, adapterName)
+                                       .set(totalEventsPublished);
+    } else {
+      throw new NoSuchElementException(ELEMENT_NOT_FOUND_TEMPLATE.formatted(adapterId));
+    }
+  }
+
+  /**
+   * Removes the specified adapter from the registry.
+   * This results in no more metrics being published for this adapter.
+   *
+   * @param adapterId   The unique identifier for the adapter
+   * @param adapterName The name of the adapter
+   * @throws NoSuchElementException If the adapter is not found in the registry
+   */
+  public void remove(String adapterId, String adapterName) {
+    if (contains(adapterId)) {
+      this.totalAdapterEventsPublishedMetric.remove(adapterId, adapterName);
+      this.registeredAdapters.remove(adapterId);
+    } else {
+      throw new NoSuchElementException(ELEMENT_NOT_FOUND_TEMPLATE.formatted(adapterId));
+    }
+  }
+
+  /**
+   * Returns the number of registered adapters in the registry.
+   *
+   * @return The number of registered adapters
+   */
+  public int size() {
+    return registeredAdapters.size();
+  }
+}
diff --git a/streampipes-commons/src/test/java/TestResources.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
similarity index 56%
copy from streampipes-commons/src/test/java/TestResources.java
copy to streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
index ec7174e33..8e211ac7e 100644
--- a/streampipes-commons/src/test/java/TestResources.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsManager.java
@@ -16,21 +16,28 @@
  *
  */
 
-import org.apache.streampipes.commons.resources.Resources;
+package org.apache.streampipes.commons.prometheus.adapter;
 
-import org.junit.Test;
+/**
+ * The `AdapterMetricsManager` enum provides a single access point for retrieving the instance of
+ * the `AdapterMetrics` class and ensures that only one instance of the metrics manager exists throughout the
+ * application's lifecycle.
+ */
+public enum AdapterMetricsManager {
+
+  INSTANCE;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+  private final AdapterMetrics adapterMetrics;
 
-import static junit.framework.TestCase.assertEquals;
+  AdapterMetricsManager() {
+    this.adapterMetrics = new AdapterMetrics();
+  }
 
-public class TestResources {
+  public static AdapterMetricsManager getInstance() {
+    return INSTANCE;
+  }
 
-  @Test
-  public void testResourceLoader() throws IOException {
-    String filename = "test.txt";
-    String content = Resources.asString(filename, StandardCharsets.UTF_8);
-    assertEquals("HelloWorld", content.replaceAll("\\s+",""));
+  public AdapterMetrics getAdapterMetrics() {
+    return adapterMetrics;
   }
 }
diff --git a/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java b/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java
new file mode 100644
index 000000000..7068c5c5c
--- /dev/null
+++ b/streampipes-commons/src/test/java/org/apache/streampipes/commons/prometheus/adapter/AdapterMetricsTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.commons.prometheus.adapter;
+
+import org.junit.Test;
+
+import java.util.NoSuchElementException;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertThrows;
+
+public class AdapterMetricsTest {
+
+  @Test
+  public void register() {
+
+    var metrics = new AdapterMetrics();
+    var adapterId = "adapterId";
+    var adapterName = "adapterName";
+
+    assertEquals(0, metrics.size());
+
+    metrics.register(adapterId, adapterName);
+
+    assertEquals(1, metrics.size());
+    assertTrue(metrics.contains(adapterId));
+  }
+
+  @Test
+  public void removeNoSuchElement() {
+    var metrics = new AdapterMetrics();
+
+    assertThrows(NoSuchElementException.class, () -> metrics.remove("test-id", "test-name"));
+  }
+
+  @Test
+  public void updateTotalEventsPublishedNoSuchElement() {
+    var metrics = new AdapterMetrics();
+
+    assertThrows(
+        NoSuchElementException.class,
+        () -> metrics.updateTotalEventsPublished("test-id", "test-name", 0)
+    );
+
+  }
+}
diff --git a/streampipes-commons/src/test/java/TestResources.java b/streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
similarity index 90%
rename from streampipes-commons/src/test/java/TestResources.java
rename to streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
index ec7174e33..7db525a3a 100644
--- a/streampipes-commons/src/test/java/TestResources.java
+++ b/streampipes-commons/src/test/java/org/apache/streampipes/commons/resources/TestResources.java
@@ -16,7 +16,7 @@
  *
  */
 
-import org.apache.streampipes.commons.resources.Resources;
+package org.apache.streampipes.commons.resources;
 
 import org.junit.Test;
 
@@ -31,6 +31,6 @@ public class TestResources {
   public void testResourceLoader() throws IOException {
     String filename = "test.txt";
     String content = Resources.asString(filename, StandardCharsets.UTF_8);
-    assertEquals("HelloWorld", content.replaceAll("\\s+",""));
+    assertEquals("HelloWorld", content.replaceAll("\\s+", ""));
   }
 }
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
index fabf477af..8653ea080 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
@@ -19,9 +19,12 @@
 package org.apache.streampipes.connect.management.health;
 
 import org.apache.streampipes.commons.exceptions.connect.AdapterException;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetrics;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager;
 import org.apache.streampipes.connect.management.management.AdapterMasterManagement;
 import org.apache.streampipes.connect.management.management.WorkerRestClient;
 import org.apache.streampipes.connect.management.util.WorkerPaths;
+import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsLogProvider;
 import org.apache.streampipes.model.connect.adapter.AdapterDescription;
 import org.apache.streampipes.storage.api.IAdapterStorage;
 
@@ -57,12 +60,18 @@ public class AdapterHealthCheck implements Runnable {
    * In this method it is checked which adapters are currently running.
    * Then it calls all workers to validate if the adapter instance is
    * still running as expected. If the adapter is not running anymore a new worker instance is invoked.
+   * In addition, it publishes monitoring metrics for all running adapters (in line with
+   * {@link org.apache.streampipes.manager.health.PipelineHealthCheck}).
    */
   public void checkAndRestoreAdapters() {
-    // Get all adapters
+    // Get all running adapters
     Map<String, AdapterDescription> allRunningInstancesAdapterDescriptions =
         this.getAllRunningInstancesAdapterDescriptions();
 
+    if (!allRunningInstancesAdapterDescriptions.isEmpty()) {
+      updateMonitoringMetrics(allRunningInstancesAdapterDescriptions);
+    }
+
     // Get all worker containers that run adapters
     Map<String, List<AdapterDescription>> groupByWorker =
         this.getAllWorkersWithAdapters(allRunningInstancesAdapterDescriptions);
@@ -75,6 +84,37 @@ public class AdapterHealthCheck implements Runnable {
     this.recoverAdapters(allAdaptersToRecover);
   }
 
+  /**
+   * Updates the monitoring metrics based on the descriptions of running adapters.
+   *
+   * @param runningAdapterDescriptions A map containing the descriptions of running adapters, where the key is the
+   *                                   adapter's element ID and the value is the corresponding adapter description.
+   */
+  protected void updateMonitoringMetrics(Map<String, AdapterDescription> runningAdapterDescriptions) {
+
+    var adapterMetrics = AdapterMetricsManager.getInstance()
+                                              .getAdapterMetrics();
+    runningAdapterDescriptions.values()
+                              .forEach(
+                                  adapterDescription -> updateTotalEventsPublished(
+                                      adapterMetrics,
+                                      adapterDescription.getElementId(),
+                                      adapterDescription.getName()
+                                  ));
+    LOG.info("Monitoring {} adapter instances", adapterMetrics.size());
+  }
+
+  private void updateTotalEventsPublished(AdapterMetrics adapterMetrics, String adapterId, String adapterName) {
+    adapterMetrics.updateTotalEventsPublished(
+        adapterId,
+        adapterName,
+        ExtensionsLogProvider.INSTANCE.getMetricInfosForResource(adapterId)
+                                      .getMessagesOut()
+                                      .getCounter()
+    );
+  }
+
+
   public Map<String, AdapterDescription> getAllRunningInstancesAdapterDescriptions() {
     Map<String, AdapterDescription> result = new HashMap<>();
     List<AdapterDescription> allRunningInstancesAdapterDescription = this.adapterStorage.getAllAdapters();
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
index c8d16e4d3..6490a499f 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
@@ -21,6 +21,7 @@ package org.apache.streampipes.connect.management.management;
 import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException;
 import org.apache.streampipes.commons.exceptions.SepaParseException;
 import org.apache.streampipes.commons.exceptions.connect.AdapterException;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager;
 import org.apache.streampipes.connect.management.util.GroundingUtils;
 import org.apache.streampipes.connect.management.util.WorkerPaths;
 import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsLogProvider;
@@ -32,7 +33,7 @@ import org.apache.streampipes.resource.management.AdapterResourceManager;
 import org.apache.streampipes.resource.management.DataStreamResourceManager;
 import org.apache.streampipes.resource.management.SpResourceManager;
 import org.apache.streampipes.storage.api.IAdapterStorage;
-import org.apache.streampipes.storage.couchdb.impl.AdapterInstanceStorageImpl;
+import org.apache.streampipes.storage.management.StorageDispatcher;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -149,6 +150,10 @@ public class AdapterMasterManagement {
 
     WorkerRestClient.stopStreamAdapter(ad.getSelectedEndpointUrl(), ad);
     ExtensionsLogProvider.INSTANCE.reset(elementId);
+
+    // remove the adapter from the metrics manager so that
+    // no metrics for this adapter are exposed anymore
+    AdapterMetricsManager.INSTANCE.getAdapterMetrics().remove(ad.getElementId(), ad.getName());
   }
 
   public void startStreamAdapter(String elementId) throws AdapterException {
@@ -166,6 +171,9 @@ public class AdapterMasterManagement {
       // Invoke adapter instance
       WorkerRestClient.invokeStreamAdapter(baseUrl, elementId);
 
+      // register the adapter at the metrics manager so that the AdapterHealthCheck can send metrics
+      AdapterMetricsManager.INSTANCE.getAdapterMetrics().register(ad.getElementId(), ad.getName());
+
       LOG.info("Started adapter " + elementId + " on: " + baseUrl);
     } catch (NoServiceEndpointsAvailableException | URISyntaxException e) {
       throw new AdapterException("Could not start adapter due to unavailable service endpoint", e);
@@ -184,6 +192,6 @@ public class AdapterMasterManagement {
   }
 
   private IAdapterStorage getAdapterInstanceStorage() {
-    return new AdapterInstanceStorageImpl();
+    return StorageDispatcher.INSTANCE.getNoSqlStore().getAdapterInstanceStorage();
   }
 }