You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by bo...@apache.org on 2023/12/15 17:44:41 UTC

(streampipes) branch dev updated: fix: orchestrate adapter health check such that monitoring of adapters works as expected (#2335)

This is an automated email from the ASF dual-hosted git repository.

bossenti pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git


The following commit(s) were added to refs/heads/dev by this push:
     new 7d4c1bc12 fix: orchestrate adapter health check such that monitoring of adapters works as expected (#2335)
7d4c1bc12 is described below

commit 7d4c1bc12b31b614d087339e15456dea3a44b8c0
Author: Tim <50...@users.noreply.github.com>
AuthorDate: Fri Dec 15 18:44:30 2023 +0100

    fix: orchestrate adapter health check such that monitoring of adapters works as expected (#2335)
    
    * fix: orchestrate adapter health check such that monitoring of adapters works fine
    
    * style: fix formatting
---
 .../management/health/AdapterHealthCheck.java      | 137 ++++++++++++++-------
 .../management/health/AdapterHealthCheckTest.java  |  11 +-
 2 files changed, 98 insertions(+), 50 deletions(-)

diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
index f0882a1fc..1d8843a62 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java
@@ -31,7 +31,6 @@ import org.apache.streampipes.storage.api.IAdapterStorage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -66,28 +65,32 @@ public class AdapterHealthCheck implements Runnable {
    * {@link org.apache.streampipes.manager.health.PipelineHealthCheck}).
    */
   public void checkAndRestoreAdapters() {
-    // Get all running adapters
-    Map<String, AdapterDescription> allRunningInstancesAdapterDescriptions =
-        this.getAllRunningInstancesAdapterDescriptions();
+    // Get all adapters that are supposed to run according to the backend storage
+    Map<String, AdapterDescription> adapterInstancesSupposedToRun =
+        this.getAllAdaptersSupposedToRun();
 
-    // Get all worker containers that run adapters
+    // group all adapter instances supposed to run by their worker service URL
     Map<String, List<AdapterDescription>> groupByWorker =
-        this.getAllWorkersWithAdapters(allRunningInstancesAdapterDescriptions);
+        this.getAllWorkersWithAdapters(adapterInstancesSupposedToRun);
 
     // Get adapters that are not running anymore
     Map<String, AdapterDescription> allAdaptersToRecover =
-        this.getAdaptersToRecover(groupByWorker, allRunningInstancesAdapterDescriptions);
+        this.getAdaptersToRecover(groupByWorker, adapterInstancesSupposedToRun);
 
     try {
-      if (!allRunningInstancesAdapterDescriptions.isEmpty()) {
+      if (!adapterInstancesSupposedToRun.isEmpty()) {
         // Filter adapters so that only healthy and running adapters are updated in the metrics endpoint
-        updateMonitoringMetrics(
-            allRunningInstancesAdapterDescriptions
-                .entrySet()
-                .stream()
-                .filter((entry -> !allAdaptersToRecover.containsKey(entry.getKey())))
-                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
-        );
+        var adaptersToMonitor = adapterInstancesSupposedToRun
+            .entrySet()
+            .stream()
+            .filter((entry -> !allAdaptersToRecover.containsKey(entry.getKey())))
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        if (!adaptersToMonitor.isEmpty()) {
+          updateMonitoringMetrics(adaptersToMonitor);
+        } else {
+          LOG.info("No running adapter instances to monitor.");
+        }
       }
     } catch (NoSuchElementException e) {
       LOG.error("Could not update adapter metrics due to an invalid state. ({})", e.getMessage());
@@ -118,6 +121,15 @@ public class AdapterHealthCheck implements Runnable {
   }
 
   private void updateTotalEventsPublished(AdapterMetrics adapterMetrics, String adapterId, String adapterName) {
+
+    // Check if the adapter is already registered; if not, register it first.
+    // This step is crucial, especially when the StreamPipes Core service is restarted,
+    // and there are existing running adapters that need proper registration.
+    // Note: Proper registration is usually handled during the initial start of the adapter.
+    if (!adapterMetrics.contains(adapterId)) {
+      adapterMetrics.register(adapterId, adapterName);
+    }
+
     adapterMetrics.updateTotalEventsPublished(
         adapterId,
         adapterName,
@@ -128,7 +140,18 @@ public class AdapterHealthCheck implements Runnable {
   }
 
 
-  public Map<String, AdapterDescription> getAllRunningInstancesAdapterDescriptions() {
+  /**
+   * Retrieves a map of all adapter instances that are supposed to be running according to the backend storage.
+   * <p>
+   * This method queries the adapter storage to obtain information about all adapters
+   * and filters the running instances. The resulting map is keyed by the element ID
+   * of each running adapter, and the corresponding values are the respective
+   * {@link AdapterDescription} objects.
+   *
+   * @return A map containing all adapter instances supposed to be running according to the backend storage.
+   * The keys are element IDs, and the values are the corresponding adapter descriptions.
+   */
+  public Map<String, AdapterDescription> getAllAdaptersSupposedToRun() {
     Map<String, AdapterDescription> result = new HashMap<>();
     List<AdapterDescription> allRunningInstancesAdapterDescription = this.adapterStorage.getAllAdapters();
     allRunningInstancesAdapterDescription
@@ -144,41 +167,69 @@ public class AdapterHealthCheck implements Runnable {
   }
 
   public Map<String, List<AdapterDescription>> getAllWorkersWithAdapters(
-      Map<String, AdapterDescription> allRunningInstancesAdapterDescription) {
+      Map<String, AdapterDescription> adapterInstancesSupposedToRun
+  ) {
 
     Map<String, List<AdapterDescription>> groupByWorker = new HashMap<>();
-    allRunningInstancesAdapterDescription.values().forEach(ad -> {
-      String selectedEndpointUrl = ad.getSelectedEndpointUrl();
-      if (selectedEndpointUrl != null) {
-        if (groupByWorker.containsKey(selectedEndpointUrl)) {
-          groupByWorker.get(selectedEndpointUrl).add(ad);
-        } else {
-          List<AdapterDescription> tmp = new ArrayList<>();
-          tmp.add(ad);
-          groupByWorker.put(selectedEndpointUrl, tmp);
-        }
-      }
-    });
+    adapterInstancesSupposedToRun.values()
+                                 .forEach(ad -> {
+                                   String selectedEndpointUrl = ad.getSelectedEndpointUrl();
+                                   if (selectedEndpointUrl != null) {
+                                     if (groupByWorker.containsKey(selectedEndpointUrl)) {
+                                       groupByWorker.get(selectedEndpointUrl)
+                                                    .add(ad);
+                                     } else {
+                                       groupByWorker.put(selectedEndpointUrl, List.of(ad));
+                                     }
+                                   }
+                                 });
 
     return groupByWorker;
   }
 
+  /**
+   * Retrieves a map of adapters to recover by comparing the provided groupings of adapter instances
+   * with the instances supposed to run according to the storage.
+   * For every adapter instance it is verified that it actually runs on a worker node.
+   * If this is not the case, it is added to the output of adapters to recover.
+   *
+   * @param adapterInstancesGroupedByWorker A map grouping adapter instances by worker.
+   * @param adapterInstancesSupposedToRun   The map containing all adapter instances supposed to be running.
+   * @return A new map containing adapter instances to recover, filtered based on running instances.
+   */
   public Map<String, AdapterDescription> getAdaptersToRecover(
-      Map<String, List<AdapterDescription>> groupByWorker,
-      Map<String, AdapterDescription> allRunningInstancesAdapterDescription) {
-    groupByWorker.keySet().forEach(adapterEndpointUrl -> {
-      try {
-        List<AdapterDescription> allRunningInstancesOfOneWorker =
-            WorkerRestClient.getAllRunningAdapterInstanceDescriptions(
-                adapterEndpointUrl + WorkerPaths.getRunningAdaptersPath());
-        allRunningInstancesOfOneWorker.forEach(adapterDescription ->
-            allRunningInstancesAdapterDescription.remove(adapterDescription.getElementId()));
-      } catch (AdapterException e) {
-        LOG.info("Could not recover adapter at endpoint {} due to {}", adapterEndpointUrl, e.getMessage());
-      }
-    });
+      Map<String, List<AdapterDescription>> adapterInstancesGroupedByWorker,
+      Map<String, AdapterDescription> adapterInstancesSupposedToRun
+  ) {
 
-    return allRunningInstancesAdapterDescription;
+    // NOTE: This line is added to prevent modifying the existing map of instances supposed to run
+    // It looks like the parameter `adapterInstancesSupposedToRun` is not required at all,
+    // but this should be checked more carefully.
+    Map<String, AdapterDescription> adaptersToRecover = new HashMap<>(adapterInstancesSupposedToRun);
+
+    adapterInstancesGroupedByWorker.keySet()
+                                   .forEach(adapterEndpointUrl -> {
+                                     try {
+                                       List<AdapterDescription> allRunningInstancesOfOneWorker =
+                                           WorkerRestClient.getAllRunningAdapterInstanceDescriptions(
+                                               adapterEndpointUrl + WorkerPaths.getRunningAdaptersPath());
+
+                                       // only keep adapters where there is no running adapter instance
+                                       // therefore, all others are removed
+                                       allRunningInstancesOfOneWorker.forEach(
+                                           adapterDescription ->
+                                               adaptersToRecover.remove(
+                                                   adapterDescription.getElementId()));
+                                     } catch (AdapterException e) {
+                                       LOG.info(
+                                           "Could not recover adapter at endpoint {} due to {}",
+                                           adapterEndpointUrl,
+                                           e.getMessage()
+                                       );
+                                     }
+                                   });
+
+    return adaptersToRecover;
   }
 
 
diff --git a/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/health/AdapterHealthCheckTest.java b/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/health/AdapterHealthCheckTest.java
index 8eedbd725..e4f3c337c 100644
--- a/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/health/AdapterHealthCheckTest.java
+++ b/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/health/AdapterHealthCheckTest.java
@@ -23,7 +23,6 @@ import org.apache.streampipes.connect.management.management.AdapterMasterManagem
 import org.apache.streampipes.model.connect.adapter.AdapterDescription;
 import org.apache.streampipes.resource.management.SpResourceManager;
 import org.apache.streampipes.storage.api.IAdapterStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -51,14 +50,13 @@ public class AdapterHealthCheckTest {
     var healthCheck = new AdapterHealthCheck(
         adapterInstanceStorageMock,
         new AdapterMasterManagement(
-            StorageDispatcher.INSTANCE.getNoSqlStore()
-                                      .getAdapterInstanceStorage(),
+            adapterInstanceStorageMock,
             new SpResourceManager().manageAdapters(),
             new SpResourceManager().manageDataStreams(),
             AdapterMetricsManager.INSTANCE.getAdapterMetrics()
         )
     );
-    var result = healthCheck.getAllRunningInstancesAdapterDescriptions();
+    var result = healthCheck.getAllAdaptersSupposedToRun();
 
     assertTrue(result.isEmpty());
   }
@@ -82,14 +80,13 @@ public class AdapterHealthCheckTest {
     var healthCheck = new AdapterHealthCheck(
         adapterInstanceStorageMock,
         new AdapterMasterManagement(
-            StorageDispatcher.INSTANCE.getNoSqlStore()
-                                      .getAdapterInstanceStorage(),
+            adapterInstanceStorageMock,
             new SpResourceManager().manageAdapters(),
             new SpResourceManager().manageDataStreams(),
             AdapterMetricsManager.INSTANCE.getAdapterMetrics()
         )
     );
-    var result = healthCheck.getAllRunningInstancesAdapterDescriptions();
+    var result = healthCheck.getAllAdaptersSupposedToRun();
 
     assertEquals(1, result.size());
     assertTrue(result.containsKey(nameRunningAdapter));