You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2026/10/02 12:53:52 UTC
(streampipes) 02/02: Refactor remaining storage api uses
This is an automated email from the ASF dual-hosted git repository.
dominikriemer pushed a commit to branch refactor-remaining-storage-interfaces
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 0bd6f08b0bd18d9c110580605ed03de5a0b6cf6f
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Fri Oct 2 14:53:31 2026 +0200
Refactor remaining storage api uses
---
.../management/AdapterMasterManagement.java | 5 +-
.../management/WorkerAdministrationManagement.java | 5 +-
.../management/DataExplorerDispatcher.java | 7 +-
.../management/DatasetMetadataManagement.java | 10 +-
.../management/DatasetMetadataManagementTest.java | 39 +++++++-
.../streampipes/export/AssetLinkResolver.java | 5 +-
.../apache/streampipes/export/ExportManager.java | 9 +-
.../export/dataimport/PerformImportGenerator.java | 25 +++--
.../export/dataimport/PreviewImportGenerator.java | 2 +-
.../export/generator/ExportPackageGenerator.java | 16 ++--
.../export/resolver/AbstractResolver.java | 6 --
.../export/resolver/AdapterResolver.java | 7 +-
.../streampipes/export/resolver/ChartResolver.java | 5 +-
.../export/resolver/DashboardResolver.java | 5 +-
.../export/resolver/DataSourceResolver.java | 16 +++-
.../export/resolver/DocumentResolver.java | 3 +-
.../streampipes/export/resolver/FileResolver.java | 5 +-
.../resolver/GenericStorageDocumentResolver.java | 18 +++-
.../export/resolver/MeasurementResolver.java | 5 +-
.../export/resolver/PipelineResolver.java | 5 +-
.../health/monitoring/PipelineHealthCheck.java | 2 +-
.../loadbalance/impl/ExtensibleLoadManager.java | 3 +-
.../unit/ExtensionsServiceEndpointUtils.java | 12 ---
.../unit/PipelineElementPartitioner.java | 16 ++--
.../loadbalance/unit/ResourceUnitScanner.java | 22 +++--
.../loadbalance/unit/ResourceUnitStatsScanner.java | 5 +-
.../streampipes/manager/assets/AssetFetcher.java | 8 +-
.../streampipes/manager/assets/AssetManager.java | 6 +-
.../execution/PipelineExecutionTaskFactory.java | 2 +-
.../ExtensionsServiceEndpointGenerator.java | 12 ++-
.../execution/task/DiscoverEndpointsTask.java | 10 +-
.../streampipes/manager/file/FileManager.java | 5 +-
.../matching/PipelineVerificationHandlerV2.java | 8 +-
.../CustomTransformOutputSchemaGenerator.java | 12 ++-
.../matching/output/OutputSchemaFactory.java | 8 +-
.../matching/v2/pipeline/ComputeOutputStep.java | 7 +-
.../v2/pipeline/PipelineValidationSteps.java | 6 +-
.../pipeline/ExtensionsServiceLogExecutor.java | 3 +-
.../manager/pipeline/PipelineManager.java | 5 +-
.../compact/CompactPipelineManagement.java | 8 +-
.../pipeline/update/PipelineUpdateCoordinator.java | 4 +-
.../manager/preview/PipelinePreview.java | 4 +-
.../manager/recommender/ElementRecommender.java | 8 +-
.../remote/ContainerProvidedOptionsHandler.java | 2 +-
.../setup/PipelineElementInstallationStep.java | 2 +-
.../compact/CompactPipelineTemplateManagement.java | 8 +-
.../manager/verification/TypedElementVerifier.java | 6 +-
.../streampipes/manager/file/TestFileManager.java | 8 +-
.../CustomTransformOutputSchemaGeneratorTest.java | 2 +
.../update/PipelineUpdateCoordinatorTest.java | 14 +--
streampipes-resource-management/pom.xml | 5 +
.../management/AdapterResourceManager.java | 7 +-
.../resource/management/AssetResourceManager.java | 5 +-
.../resource/management/ChartResourceManager.java | 5 +-
.../resource/management/CrudResourceManager.java | 7 +-
.../management/DashboardResourceManager.java | 5 +-
.../management/DataStreamResourceManager.java | 15 ++-
.../management/PipelineResourceManager.java | 5 +-
.../management/ResourceDeletionManager.java | 54 +++++++++++
.../resource/management/SpResourceManager.java | 74 +++++++++++----
.../management/AdapterResourceManagerTest.java | 2 +-
.../management/DataStreamResourceManagerTest.java | 24 ++++-
.../management/ResourceDeletionManagerTest.java | 93 +++++++++++++++++++
.../rest/core/base/impl/AbstractRestResource.java | 11 ---
.../apache/streampipes/rest/ResetManagement.java | 17 ++--
.../rest/impl/CompactPipelineResource.java | 7 +-
.../apache/streampipes/rest/impl/FileResource.java | 8 +-
.../rest/impl/GenericStorageResource.java | 23 +++--
.../rest/impl/PipelineElementAsset.java | 7 +-
.../rest/impl/PipelineElementCategory.java | 12 ++-
.../streampipes/rest/impl/PipelineResource.java | 4 +-
.../streampipes/rest/impl/PipelineTemplate.java | 13 ++-
.../streampipes/rest/impl/ResetResource.java | 8 +-
.../streampipes/rest/impl/ServiceTagResource.java | 7 +-
.../impl/admin/ExtensionsInstallationResource.java | 2 +-
.../admin/ExtensionsServiceEndpointResource.java | 8 +-
.../impl/admin/ServiceRegistrationResource.java | 7 +-
.../rest/impl/connect/AdapterResource.java | 12 ++-
.../rest/impl/connect/CompactAdapterResource.java | 26 ++++--
.../rest/impl/connect/DescriptionResource.java | 2 +-
.../rest/impl/connect/GuessResource.java | 2 +-
.../impl/connect/RuntimeResolvableResource.java | 2 +-
.../TransformationScriptLanguageResource.java | 7 +-
.../rest/impl/dataset/AbstractDatasetResource.java | 3 +-
.../impl/connect/CompactAdapterResourceTest.java | 16 ++--
.../service/core/DatasetQueryConfiguration.java | 3 +-
.../core/PipelineManagementConfiguration.java | 13 +++
.../service/core/StreamPipesCoreApplication.java | 6 +-
.../ExtensionServiceRequestConfiguration.java | 21 ++++-
...ansportAwareExtensionServiceRequestManager.java | 11 ++-
.../core/migrations/AvailableMigrations.java | 24 +++--
.../v0980/FixImportedPermissionsMigration.java | 8 +-
.../v0980/ModifyAssetLinkTypesMigration.java | 7 +-
.../v0980/ModifyAssetLinksMigration.java | 7 +-
.../v099/ModifyAssetLinkIconMigration.java | 5 +-
.../migrations/v099/MoveAssetContentMigration.java | 7 +-
.../v099/RenameAssetLinkTypesMigration.java | 5 +-
.../service/core/scheduler/DatasetScheduler.java | 3 +-
.../core/storage/StorageApiConfiguration.java | 10 +-
...ortAwareExtensionServiceRequestManagerTest.java | 101 +++++++++++++++++++++
.../v099/RenameAssetLinkTypesMigrationTest.java | 3 +-
.../svcdiscovery/SpServiceDiscovery.java | 29 ------
.../storage/api/core/INoSqlStorage.java | 11 ---
.../storage/couchdb/CouchDbStorageManager.java | 58 ------------
.../couchdb/impl/core/DefaultCrudStorage.java | 14 +--
.../PipelineElementDescriptionStorageImpl.java | 10 ++
.../storage/management/StorageDispatcher.java | 31 -------
107 files changed, 846 insertions(+), 462 deletions(-)
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
index c21a4fdb35..74b30550ef 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java
@@ -35,7 +35,6 @@ import org.apache.streampipes.model.util.ElementIdGenerator;
import org.apache.streampipes.resource.management.AdapterResourceManager;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
import org.slf4j.Logger;
@@ -237,7 +236,7 @@ public class AdapterMasterManagement {
try {
try {
// Find endpoint to start adapter on
- var service = new ExtensionsServiceEndpointGenerator()
+ var service = new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery())
.selectService(ad.getAppId(), SpServiceUrlProvider.ADAPTER,
ad.getDeploymentConfiguration().getDesiredServiceTags());
@@ -269,7 +268,7 @@ public class AdapterMasterManagement {
}
private boolean installDataSource(SpDataStream stream, String principalSid) throws AdapterException {
- var storageApi = StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineElementDescriptionStorage();
+ var storageApi = resourceManager.getPipelineElementDescriptionStorage();
var verifier = new TypedElementVerifier<>(
stream,
storageApi,
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java
index 8f71808fb5..1d50d8f8f7 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java
@@ -28,6 +28,7 @@ import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.resource.management.UserResourceManager;
import org.apache.streampipes.storage.api.connect.IAdapterStorage;
import org.apache.streampipes.storage.api.user.IPermissionStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
import org.slf4j.Logger;
@@ -46,6 +47,7 @@ public class WorkerAdministrationManagement {
private final UserResourceManager userResourceManager;
private final ExtensionServiceRequestManager requestManager;
private final AssetManager assetManager;
+ private final ISpServiceDiscovery serviceDiscovery;
public WorkerAdministrationManagement(
IAdapterStorage adapterDescriptionStorage,
@@ -56,6 +58,7 @@ public class WorkerAdministrationManagement {
this.permissionStorage = resourceManager.managePermissions().getDb();
this.permissionResourceManager = resourceManager.managePermissions();
this.requestManager = requestManager;
+ this.serviceDiscovery = resourceManager.getServiceDiscovery();
this.assetManager = new AssetManager(resourceManager.getCoreConfigurationStorage());
}
@@ -68,7 +71,7 @@ public class WorkerAdministrationManagement {
if (!assetManager.existsAssetDir(adapter.getAppId())) {
try {
LOG.info("Updating assets for adapter {}", adapter.getAppId());
- assetManager.storeAsset(SpServiceUrlProvider.ADAPTER, adapter.getAppId(), requestManager);
+ assetManager.storeAsset(SpServiceUrlProvider.ADAPTER, adapter.getAppId(), requestManager, serviceDiscovery);
} catch (IOException | NoServiceEndpointsAvailableException e) {
LOG.error(
"Could not fetch asset for adapter {}, please try to manually update this adapter.",
diff --git a/streampipes-data-explorer-management/src/main/java/org/apache/streampipes/dataexplorer/management/DataExplorerDispatcher.java b/streampipes-data-explorer-management/src/main/java/org/apache/streampipes/dataexplorer/management/DataExplorerDispatcher.java
index 542d7fd0c2..956a71ad1c 100644
--- a/streampipes-data-explorer-management/src/main/java/org/apache/streampipes/dataexplorer/management/DataExplorerDispatcher.java
+++ b/streampipes-data-explorer-management/src/main/java/org/apache/streampipes/dataexplorer/management/DataExplorerDispatcher.java
@@ -25,6 +25,7 @@ import org.apache.streampipes.dataexplorer.influx.DataExplorerManagerInflux;
import org.apache.streampipes.dataexplorer.iotdb.DataExplorerManagerIotDb;
import org.apache.streampipes.manager.permission.DatasetPermissionManager;
import org.apache.streampipes.manager.pipeline.update.ChartSchemaUpdateCoordinator;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.storage.api.explorer.IDatasetMetadataStorage;
import org.apache.streampipes.storage.api.user.IPermissionStorage;
@@ -56,8 +57,10 @@ public class DataExplorerDispatcher {
public IDatasetMetadataManagement getSchemaManagement(ChartSchemaUpdateCoordinator charts,
IPermissionStorage permissions,
- IDatasetMetadataStorage datasets) {
- return new DatasetMetadataManagement(datasets, new DatasetPermissionManager(permissions), charts);
+ IDatasetMetadataStorage datasets,
+ ResourceDeletionManager resourceDeletionManager) {
+ return new DatasetMetadataManagement(datasets, new DatasetPermissionManager(permissions), charts,
+ resourceDeletionManager);
}
public DatasetServices getDatasetServices(
diff --git a/streampipes-data-explorer-management/src/main/java/org/apache/streampipes/dataexplorer/management/DatasetMetadataManagement.java b/streampipes-data-explorer-management/src/main/java/org/apache/streampipes/dataexplorer/management/DatasetMetadataManagement.java
index 5586eb1de5..b6b8aa285d 100644
--- a/streampipes-data-explorer-management/src/main/java/org/apache/streampipes/dataexplorer/management/DatasetMetadataManagement.java
+++ b/streampipes-data-explorer-management/src/main/java/org/apache/streampipes/dataexplorer/management/DatasetMetadataManagement.java
@@ -26,6 +26,7 @@ import org.apache.streampipes.model.dataset.DatasetMetadata;
import org.apache.streampipes.model.dataset.DatasetMetadataSchemaUpdateStrategy;
import org.apache.streampipes.model.schema.EventProperty;
import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.storage.api.explorer.IDatasetMetadataStorage;
import java.util.ArrayList;
@@ -40,13 +41,16 @@ import java.util.stream.Stream;
public class DatasetMetadataManagement implements IDatasetMetadataManagement {
private final IDatasetMetadataStorage dataLakeStorage;
+ private final ResourceDeletionManager resourceDeletionManager;
private final DatasetPermissionManager permissionManager;
private final ChartSchemaUpdateCoordinator chartSchemaUpdateCoordinator;
public DatasetMetadataManagement(IDatasetMetadataStorage dataLakeStorage,
DatasetPermissionManager permissionManager,
- ChartSchemaUpdateCoordinator chartSchemaUpdateCoordinator) {
+ ChartSchemaUpdateCoordinator chartSchemaUpdateCoordinator,
+ ResourceDeletionManager resourceDeletionManager) {
this.dataLakeStorage = dataLakeStorage;
+ this.resourceDeletionManager = resourceDeletionManager;
this.permissionManager = permissionManager;
this.chartSchemaUpdateCoordinator = chartSchemaUpdateCoordinator;
}
@@ -122,7 +126,7 @@ public class DatasetMetadataManagement implements IDatasetMetadataManagement {
@Override
public void deleteMeasurement(String elementId) {
if (dataLakeStorage.getElementById(elementId) != null) {
- dataLakeStorage.deleteElementById(elementId);
+ resourceDeletionManager.delete(dataLakeStorage, elementId);
} else {
throw new IllegalArgumentException("Could not find measure with this ID");
}
@@ -133,7 +137,7 @@ public class DatasetMetadataManagement implements IDatasetMetadataManagement {
var measureToDeleteOpt = getExistingMeasureByName(measureName);
return measureToDeleteOpt.map(measure -> {
- dataLakeStorage.deleteElementById(measure.getElementId());
+ resourceDeletionManager.delete(dataLakeStorage, measure.getElementId());
return true;
}).orElse(false);
}
diff --git a/streampipes-data-explorer-management/src/test/java/org/apache/streampipes/dataexplorer/management/DatasetMetadataManagementTest.java b/streampipes-data-explorer-management/src/test/java/org/apache/streampipes/dataexplorer/management/DatasetMetadataManagementTest.java
index 84d34fb951..f14340147d 100644
--- a/streampipes-data-explorer-management/src/test/java/org/apache/streampipes/dataexplorer/management/DatasetMetadataManagementTest.java
+++ b/streampipes-data-explorer-management/src/test/java/org/apache/streampipes/dataexplorer/management/DatasetMetadataManagementTest.java
@@ -23,6 +23,7 @@ import org.apache.streampipes.manager.pipeline.update.ChartSchemaUpdateCoordinat
import org.apache.streampipes.model.dataset.DatasetMetadata;
import org.apache.streampipes.model.dataset.DatasetMetadataSchemaUpdateStrategy;
import org.apache.streampipes.model.schema.EventProperty;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.storage.api.explorer.IDatasetMetadataStorage;
import org.apache.streampipes.storage.api.user.IPermissionStorage;
import org.apache.streampipes.test.generator.EventPropertyPrimitiveTestBuilder;
@@ -61,12 +62,39 @@ public class DatasetMetadataManagementTest {
this.chartSchemaUpdateCoordinator = mock(ChartSchemaUpdateCoordinator.class);
}
+ @Test
+ void deletionByIdUsesResourceDeletionManager() {
+ var deletionManager = mock(ResourceDeletionManager.class);
+ var management = new DatasetMetadataManagement(
+ dataLakeStorageMock, permissionManagerMock, chartSchemaUpdateCoordinator, deletionManager);
+ when(dataLakeStorageMock.getElementById("dataset-1")).thenReturn(new DatasetMetadata());
+
+ management.deleteMeasurement("dataset-1");
+
+ verify(deletionManager).delete(dataLakeStorageMock, "dataset-1");
+ }
+
+ @Test
+ void deletionByNameUsesResourceIdForAssetLinkCleanup() {
+ var deletionManager = mock(ResourceDeletionManager.class);
+ var management = new DatasetMetadataManagement(
+ dataLakeStorageMock, permissionManagerMock, chartSchemaUpdateCoordinator, deletionManager);
+ var dataset = new DatasetMetadata();
+ dataset.setElementId("dataset-1");
+ when(dataLakeStorageMock.getByMeasureName("temperature")).thenReturn(dataset);
+
+ assertTrue(management.deleteMeasurementByName("temperature"));
+
+ verify(deletionManager).delete(dataLakeStorageMock, "dataset-1");
+ }
+
@Test
public void createMeasurementThatNotExisted() {
var schemaManagement = new DatasetMetadataManagement(
dataLakeStorageMock,
permissionManagerMock,
- chartSchemaUpdateCoordinator
+ chartSchemaUpdateCoordinator,
+ mock(ResourceDeletionManager.class)
);
var oldMeasure = getSampleMeasure(
@@ -96,7 +124,8 @@ public class DatasetMetadataManagementTest {
var schemaManagement = new DatasetMetadataManagement(
dataLakeStorageMock,
permissionManagerMock,
- chartSchemaUpdateCoordinator
+ chartSchemaUpdateCoordinator,
+ mock(ResourceDeletionManager.class)
);
var newMeasure = getNewMeasure(DatasetMetadataSchemaUpdateStrategy.UPDATE_SCHEMA);
@@ -126,7 +155,8 @@ public class DatasetMetadataManagementTest {
var schemaManagement = new DatasetMetadataManagement(
dataLakeStorageMock,
permissionManagerMock,
- chartSchemaUpdateCoordinator
+ chartSchemaUpdateCoordinator,
+ mock(ResourceDeletionManager.class)
);
var newMeasure = getNewMeasure(DatasetMetadataSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA);
@@ -155,7 +185,8 @@ public class DatasetMetadataManagementTest {
var schemaManagement = new DatasetMetadataManagement(
dataLakeStorageMock,
permissionManagerMock,
- chartSchemaUpdateCoordinator
+ chartSchemaUpdateCoordinator,
+ mock(ResourceDeletionManager.class)
);
var newMeasure = getNewMeasure(DatasetMetadataSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA);
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
index 9ac21854ce..af8aa370d1 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
@@ -33,7 +33,6 @@ import org.apache.streampipes.model.assets.SpAssetModel;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.serializers.json.JacksonSerializer;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -89,7 +88,7 @@ public class AssetLinkResolver {
resourceManager.manageDashboards()
).resolve(getLinks(assetLinks, ResolvableAssetLinks.DASHBOARD)));
exportConfig.setDataSources(
- new DataSourceResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.DATA_STREAM)));
+ new DataSourceResolver(resourceManager.manageDataStreams().getDb()).resolve(getLinks(assetLinks, ResolvableAssetLinks.DATA_STREAM)));
exportConfig.setPipelines(
new PipelineResolver(extensionServiceRequestManager, pipelineManager, resourceManager.managePipelines())
.resolve(getLinks(assetLinks, ResolvableAssetLinks.PIPELINE))
@@ -117,7 +116,7 @@ public class AssetLinkResolver {
}
private SpAssetModel getAsset() throws IOException {
- return deserialize(StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage().findOne(this.assetId));
+ return deserialize(resourceManager.getGenericStorage().findOne(this.assetId));
}
private SpAssetModel deserialize(Map<String, Object> asset) {
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
index 19d6138db6..88c6d4f2d6 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
@@ -25,7 +25,7 @@ import org.apache.streampipes.model.assets.SpAssetModel;
import org.apache.streampipes.model.export.ExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.resource.management.SpResourceManager;
-import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import java.io.IOException;
import java.util.List;
@@ -44,7 +44,7 @@ public class ExportManager {
assetId, extensionServiceRequestManager, resourceManager, pipelineManager)
.resolveResources())
.collect(Collectors.toList());
- var genericStorageAppDocTypes = getGenericStorageAppDocTypes();
+ var genericStorageAppDocTypes = getGenericStorageAppDocTypes(resourceManager.getGenericStorage());
exportConfig.setAssetExportConfiguration(assetExportConfigurations);
exportConfig.setGenericStorageAppDocTypes(genericStorageAppDocTypes);
@@ -61,10 +61,9 @@ public class ExportManager {
.generateExportPackage();
}
- private static List<ExportItem> getGenericStorageAppDocTypes() {
+ private static List<ExportItem> getGenericStorageAppDocTypes(IGenericStorage genericStorage) {
try {
- return StorageDispatcher.INSTANCE.getNoSqlStore()
- .getGenericStorage()
+ return genericStorage
.getAllAppDocTypes()
.stream()
.filter(appDocType -> !SpAssetModel.APP_DOC_TYPE.equals(appDocType))
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
index e5fee3a454..c225b2956f 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
@@ -40,9 +40,8 @@ import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.resource.management.SpResourceManager;
-import org.apache.streampipes.storage.api.core.INoSqlStorage;
import org.apache.streampipes.storage.api.explorer.IDatasetMetadataStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -55,7 +54,7 @@ import java.util.Set;
public class PerformImportGenerator extends ImportGenerator<Void> {
private final AssetExportConfiguration config;
- private final INoSqlStorage storage;
+ private final IGenericStorage genericStorage;
private final Set<PermissionInfo> permissionsToStore = new HashSet<>();
private final String ownerSid;
private final ExtensionServiceRequestManager extensionServiceRequestManager;
@@ -69,7 +68,7 @@ public class PerformImportGenerator extends ImportGenerator<Void> {
SpResourceManager resourceManager,
PipelineManager pipelineManager) {
this.config = config;
- this.storage = StorageDispatcher.INSTANCE.getNoSqlStore();
+ this.genericStorage = resourceManager.getGenericStorage();
this.ownerSid = ownerSid;
this.extensionServiceRequestManager = extensionServiceRequestManager;
this.resourceManager = resourceManager;
@@ -81,14 +80,14 @@ public class PerformImportGenerator extends ImportGenerator<Void> {
protected void handleAsset(Map<String, byte[]> previewFiles, String assetId) throws IOException {
var document = asString(previewFiles.get(assetId));
try {
- var existing = storage.getGenericStorage().findOne(assetId);
+ var existing = genericStorage.findOne(assetId);
if (config.isOverwriteExistingDocuments()) {
- storage.getGenericStorage().delete(assetId, existing.get("_rev").toString());
+ genericStorage.delete(assetId, existing.get("_rev").toString());
}
} catch (IOException e) {
// Document not found, do nothing
}
- storage.getGenericStorage().create(document);
+ genericStorage.create(document);
permissionsToStore.add(new PermissionInfo(assetId, SpAssetModel.class));
}
@@ -123,8 +122,8 @@ public class PerformImportGenerator extends ImportGenerator<Void> {
@Override
protected void handleDataSource(String document, String dataSourceId) throws JsonProcessingException {
if (shouldStore(dataSourceId, config.getDataSources())) {
- writeDocument(document, new DataSourceResolver());
- var dataStream = new DataSourceResolver().deserializeDocument(document);
+ writeDocument(document, new DataSourceResolver(resourceManager.manageDataStreams().getDb()));
+ var dataStream = new DataSourceResolver(resourceManager.manageDataStreams().getDb()).deserializeDocument(document);
permissionsToStore.add(new PermissionInfo(dataStream.getElementId(), SpDataStream.class));
}
}
@@ -162,28 +161,28 @@ public class PerformImportGenerator extends ImportGenerator<Void> {
@Override
protected void handleLabel(String document, String labelId) throws JsonProcessingException {
if (shouldStore(labelId, config.getLabels())) {
- writeDocument(document, new GenericStorageDocumentResolver());
+ writeDocument(document, new GenericStorageDocumentResolver(genericStorage));
}
}
@Override
protected void handleSite(String document, String siteId) throws JsonProcessingException {
if (shouldStore(siteId, config.getSites())) {
- writeDocument(document, new GenericStorageDocumentResolver());
+ writeDocument(document, new GenericStorageDocumentResolver(genericStorage));
}
}
@Override
protected void handleGenericStorageDocument(String document, String documentId) throws JsonProcessingException {
if (shouldStore(documentId, config.getGenericStorageDocuments())) {
- writeDocument(document, new GenericStorageDocumentResolver());
+ writeDocument(document, new GenericStorageDocumentResolver(genericStorage));
}
}
private void writeDocument(String document,
AbstractResolver<?> resolver) throws JsonProcessingException {
if (config.isOverwriteExistingDocuments()) {
- resolver.deleteDocument(document);
+ resolver.deleteDocument(document, resourceManager.getResourceDeletionManager());
}
resolver.writeDocument(document, config);
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
index ba4fb87f59..6ba6c937ed 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
@@ -113,7 +113,7 @@ public class PreviewImportGenerator extends ImportGenerator<AssetExportConfigura
@Override
protected void handleDataSource(String document, String dataSourceId) throws JsonProcessingException {
- addExportItem(dataSourceId, new DataSourceResolver().readDocument(document).getName(), importConfig::addDataSource);
+ addExportItem(dataSourceId, new DataSourceResolver(resourceManager.manageDataStreams().getDb()).readDocument(document).getName(), importConfig::addDataSource);
}
@Override
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
index c12210de08..8709874696 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
@@ -37,7 +37,6 @@ import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.model.export.StreamPipesApplicationPackage;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.serializers.json.JacksonSerializer;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -103,7 +102,7 @@ public class ExportPackageGenerator {
config.getDataSources().forEach(item -> addDoc(builder,
item,
- new DataSourceResolver(),
+ new DataSourceResolver(resourceManager.manageDataStreams().getDb()),
manifest::addDataSource));
config.getDataLakeMeasures().forEach(item -> addDoc(builder,
@@ -137,17 +136,17 @@ public class ExportPackageGenerator {
config.getGenericStorageDocuments().forEach(item -> {
addGenericStorageDocument(builder,
item,
- new GenericStorageDocumentResolver(),
+ new GenericStorageDocumentResolver(resourceManager.getGenericStorage()),
manifest::addGenericStorageDocument,
exportedGenericStorageDocumentIds);
});
config.getLabels().forEach(item -> {
- addDoc(builder, item, new GenericStorageDocumentResolver(), manifest::addGenericStorageDocument);
+ addDoc(builder, item, new GenericStorageDocumentResolver(resourceManager.getGenericStorage()), manifest::addGenericStorageDocument);
});
config.getSites().forEach(item -> {
- addDoc(builder, item, new GenericStorageDocumentResolver(), manifest::addGenericStorageDocument);
+ addDoc(builder, item, new GenericStorageDocumentResolver(resourceManager.getGenericStorage()), manifest::addGenericStorageDocument);
});
config.getFiles().forEach(item -> {
@@ -217,11 +216,10 @@ public class ExportPackageGenerator {
String appDocType,
Consumer<String> function,
Set<String> exportedGenericStorageDocumentIds) {
- var resolver = new GenericStorageDocumentResolver();
+ var resolver = new GenericStorageDocumentResolver(resourceManager.getGenericStorage());
try {
- StorageDispatcher.INSTANCE.getNoSqlStore()
- .getGenericStorage()
+ resourceManager.getGenericStorage()
.findAll(appDocType)
.forEach(document -> addGenericStorageDocument(builder,
resolver.convert(document),
@@ -263,6 +261,6 @@ public class ExportPackageGenerator {
}
private Map<String, Object> getAsset(String assetId) throws IOException {
- return StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage().findOne(assetId);
+ return resourceManager.getGenericStorage().findOne(assetId);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
index 09aa1265f8..977d64bb3b 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
@@ -24,8 +24,6 @@ import org.apache.streampipes.model.assets.AssetLink;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.model.grounding.EventGrounding;
import org.apache.streampipes.serializers.json.JacksonSerializer;
-import org.apache.streampipes.storage.api.core.INoSqlStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -70,10 +68,6 @@ public abstract class AbstractResolver<T> implements DocumentResolver<T> {
}
}
- protected INoSqlStorage getNoSqlStore() {
- return StorageDispatcher.INSTANCE.getNoSqlStore();
- }
-
protected void overrideProtocol(EventGrounding grounding) {
var newProtocol = new EventGroundingProcessor().applyOverride(grounding.getTransportProtocol());
grounding.setTransportProtocol(newProtocol);
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
index db5b6caa6f..887994b53f 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
@@ -28,6 +28,7 @@ import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestMana
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.resource.management.secret.SecretProvider;
import org.apache.streampipes.storage.api.connect.IAdapterStorage;
@@ -92,7 +93,7 @@ public class AdapterResolver extends AbstractResolver<AdapterDescription> {
}
@Override
- public void deleteDocument(String document) throws JsonProcessingException {
+ public void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException {
var adapter = deserializeDocument(document);
var resourceId = adapter.getElementId();
var existingAdapter = adapterStorage.getElementById(resourceId);
@@ -103,7 +104,7 @@ public class AdapterResolver extends AbstractResolver<AdapterDescription> {
resourceManager,
AdapterMetricsManager.INSTANCE.getAdapterMetrics(),
new WorkerRestClient(extensionServiceRequestManager, resourceManager),
- getNoSqlStore().getExtensionsServiceStorage(),
+ resourceManager.getExtensionsServiceStorage(),
extensionServiceRequestManager,
new AdapterAuditRecorder(resourceManager.getAuditService())
).stopAdapter(resourceId, true);
@@ -111,7 +112,7 @@ public class AdapterResolver extends AbstractResolver<AdapterDescription> {
LOG.warn("Error when stopping adapter with id {} and name {}", resourceId, existingAdapter.getName());
}
}
- adapterStorage.deleteElementById(resourceId);
+ resourceDeletionManager.delete(adapterStorage, resourceId);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/ChartResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/ChartResolver.java
index 0ce2b2deb1..7e6aebe697 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/ChartResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/ChartResolver.java
@@ -21,6 +21,7 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.model.dataset.DataExplorerWidgetModel;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.serializers.json.JacksonSerializer;
import org.apache.streampipes.storage.api.core.CRUDStorage;
@@ -75,10 +76,10 @@ public class ChartResolver extends AbstractResolver<DataExplorerWidgetModel> {
}
@Override
- public void deleteDocument(String document) throws JsonProcessingException {
+ public void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException {
var chart = readDocument(document);
var resourceId = chart.getElementId();
- chartStorage.deleteElementById(resourceId);
+ resourceDeletionManager.delete(chartStorage, resourceId);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
index 3c9c2f2ebe..41cd3f98c7 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.model.dashboard.DashboardModel;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.resource.management.DashboardResourceManager;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -76,10 +77,10 @@ public class DashboardResolver extends AbstractResolver<DashboardModel> {
}
@Override
- public void deleteDocument(String document) throws JsonProcessingException {
+ public void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException {
var dashboard = readDocument(document);
var resourceId = dashboard.getElementId();
- resourceManager.getDb().deleteElementById(resourceId);
+ resourceDeletionManager.delete(resourceManager.getDb(), resourceId);
}
public List<String> getCharts(String resourceId) {
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
index 09f4769579..4052a8c867 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
@@ -21,14 +21,22 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
+import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
import com.fasterxml.jackson.core.JsonProcessingException;
public class DataSourceResolver extends AbstractResolver<SpDataStream> {
+ private final IDataStreamStorage dataStreamStorage;
+
+ public DataSourceResolver(IDataStreamStorage dataStreamStorage) {
+ this.dataStreamStorage = dataStreamStorage;
+ }
+
@Override
public SpDataStream findDocument(String resourceId) {
- return getNoSqlStore().getDataStreamStorage().getElementById(resourceId);
+ return dataStreamStorage.getElementById(resourceId);
}
@Override
@@ -56,7 +64,7 @@ public class DataSourceResolver extends AbstractResolver<SpDataStream> {
overrideProtocol(dataStream.getEventGrounding());
}
}
- getNoSqlStore().getDataStreamStorage().persist(dataStream);
+ dataStreamStorage.persist(dataStream);
}
@Override
@@ -65,9 +73,9 @@ public class DataSourceResolver extends AbstractResolver<SpDataStream> {
}
@Override
- public void deleteDocument(String document) throws JsonProcessingException {
+ public void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException {
var dataSource = readDocument(document);
var resourceId = dataSource.getElementId();
- getNoSqlStore().getDataStreamStorage().deleteElementById(resourceId);
+ resourceDeletionManager.delete(dataStreamStorage, resourceId);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DocumentResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DocumentResolver.java
index 2e4b4e35dc..81b47ea35a 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DocumentResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DocumentResolver.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.lightcouch.DocumentConflictException;
@@ -39,5 +40,5 @@ public interface DocumentResolver<T> {
T deserializeDocument(String document) throws JsonProcessingException;
- void deleteDocument(String document) throws JsonProcessingException;
+ void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException;
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
index 58c5713f6f..0fea75acf0 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
@@ -21,6 +21,7 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.model.file.FileMetadata;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.storage.api.system.IFileMetadataStorage;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -65,9 +66,9 @@ public class FileResolver extends AbstractResolver<FileMetadata> {
}
@Override
- public void deleteDocument(String document) throws JsonProcessingException {
+ public void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException {
var fileMetadata = readDocument(document);
var resourceId = fileMetadata.getElementId();
- fileMetadataStorage.deleteElementById(resourceId);
+ resourceDeletionManager.delete(fileMetadataStorage, resourceId);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
index 445cac1869..e3530ecb27 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
@@ -20,7 +20,9 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.serializers.json.JacksonSerializer;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
@@ -37,10 +39,16 @@ public class GenericStorageDocumentResolver extends AbstractResolver<Map<String,
private static final Logger LOG = LoggerFactory.getLogger(GenericStorageDocumentResolver.class);
+ private final IGenericStorage genericStorage;
+
+ public GenericStorageDocumentResolver(IGenericStorage genericStorage) {
+ this.genericStorage = genericStorage;
+ }
+
@Override
public Map<String, Object> findDocument(String resourceId) {
try {
- return getNoSqlStore().getGenericStorage().findOne(resourceId);
+ return genericStorage.findOne(resourceId);
} catch (IOException e) {
return null;
}
@@ -70,7 +78,7 @@ public class GenericStorageDocumentResolver extends AbstractResolver<Map<String,
public void writeDocument(String document, AssetExportConfiguration config)
throws JsonProcessingException, DocumentConflictException {
try {
- getNoSqlStore().getGenericStorage().create(document);
+ genericStorage.create(document);
} catch (IOException e) {
LOG.warn("Could not write document");
}
@@ -85,12 +93,12 @@ public class GenericStorageDocumentResolver extends AbstractResolver<Map<String,
}
@Override
- public void deleteDocument(String document) throws JsonProcessingException {
+ public void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException {
try {
var asset = readDocument(document);
var resourceId = asset.get("_id").toString();
- var storedAsset = getNoSqlStore().getGenericStorage().findOne(resourceId);
- getNoSqlStore().getGenericStorage().delete(resourceId, storedAsset.get("_rev").toString());
+ var storedAsset = genericStorage.findOne(resourceId);
+ genericStorage.delete(resourceId, storedAsset.get("_rev").toString());
} catch (IOException e) {
// Do nothing
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
index ff7452a6e0..d4aeb325fb 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
@@ -21,6 +21,7 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.model.dataset.DatasetMetadata;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.storage.api.explorer.IDatasetMetadataStorage;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -65,9 +66,9 @@ public class MeasurementResolver extends AbstractResolver<DatasetMetadata> {
}
@Override
- public void deleteDocument(String document) throws JsonProcessingException {
+ public void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException {
var measurement = readDocument(document);
var resourceId = measurement.getElementId();
- datasetStorage.deleteElementById(resourceId);
+ resourceDeletionManager.delete(datasetStorage, resourceId);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
index 1a315c1bdf..1c8969e87e 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.resource.management.PipelineResourceManager;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.resource.management.secret.SecretProvider;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -103,7 +104,7 @@ public class PipelineResolver extends AbstractResolver<Pipeline> {
}
@Override
- public void deleteDocument(String document) throws JsonProcessingException {
+ public void deleteDocument(String document, ResourceDeletionManager resourceDeletionManager) throws JsonProcessingException {
var pipeline = readDocument(document);
var resourceId = pipeline.getElementId();
var storedPipeline = pipelineManager.getPipeline(resourceId);
@@ -111,7 +112,7 @@ public class PipelineResolver extends AbstractResolver<Pipeline> {
if (storedPipeline.isRunning()) {
pipelineManager.stopPipeline(resourceId, true, requestManager);
}
- resourceManager.getDb().deleteElementById(resourceId);
+ resourceDeletionManager.delete(resourceManager.getDb(), resourceId);
}
}
}
diff --git a/streampipes-health-monitoring/src/main/java/org/apache/streampipes/health/monitoring/PipelineHealthCheck.java b/streampipes-health-monitoring/src/main/java/org/apache/streampipes/health/monitoring/PipelineHealthCheck.java
index c02e08bd39..18ae3ada08 100644
--- a/streampipes-health-monitoring/src/main/java/org/apache/streampipes/health/monitoring/PipelineHealthCheck.java
+++ b/streampipes-health-monitoring/src/main/java/org/apache/streampipes/health/monitoring/PipelineHealthCheck.java
@@ -225,7 +225,7 @@ public class PipelineHealthCheck implements HealthCheck {
protected boolean restorePipelineElement(InvocableStreamPipesEntity pipelineElement,
String pipelineId) {
try {
- var service = new ExtensionsServiceEndpointGenerator().selectService(
+ var service = new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery()).selectService(
pipelineElement.getAppId(),
ExtensionsServiceEndpointUtils.getPipelineElementType(pipelineElement),
Collections.emptySet()
diff --git a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/ExtensibleLoadManager.java b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/ExtensibleLoadManager.java
index 618e56e347..00e877fd0a 100644
--- a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/ExtensibleLoadManager.java
+++ b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/ExtensibleLoadManager.java
@@ -25,7 +25,6 @@ import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistratio
import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import java.util.List;
@@ -51,7 +50,7 @@ public class ExtensibleLoadManager implements LoadBalancer {
}
public void doLoadShedding() {
- IExtensionsServiceStorage storage = StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage();
+ IExtensionsServiceStorage storage = resourceManager.getExtensionsServiceStorage();
List<SpServiceRegistration> services = storage.findAll().stream().filter(s -> s.getStatus() == SpServiceStatus.HEALTHY)
.toList();
diff --git a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ExtensionsServiceEndpointUtils.java b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ExtensionsServiceEndpointUtils.java
index eedec8f50d..c48cabca4c 100644
--- a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ExtensionsServiceEndpointUtils.java
+++ b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ExtensionsServiceEndpointUtils.java
@@ -23,11 +23,8 @@ import org.apache.streampipes.model.graph.DataProcessorDescription;
import org.apache.streampipes.model.graph.DataProcessorInvocation;
import org.apache.streampipes.model.graph.DataSinkDescription;
import org.apache.streampipes.model.graph.DataSinkInvocation;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
-import java.util.NoSuchElementException;
-
public class ExtensionsServiceEndpointUtils {
public static SpServiceUrlProvider getPipelineElementType(NamedStreamPipesEntity entity) {
@@ -42,15 +39,6 @@ public class ExtensionsServiceEndpointUtils {
}
}
- public static SpServiceUrlProvider getPipelineElementType(String appId) {
- try {
- StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineElementDescriptionStorage().getDataProcessorByAppId(appId);
- return SpServiceUrlProvider.DATA_PROCESSOR;
- } catch (NoSuchElementException e) {
- return SpServiceUrlProvider.DATA_SINK;
- }
- }
-
private static boolean isDataProcessor(NamedStreamPipesEntity entity) {
return entity instanceof DataProcessorInvocation || entity instanceof DataProcessorDescription;
}
diff --git a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/PipelineElementPartitioner.java b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/PipelineElementPartitioner.java
index 4e540fc28a..c5b40fe9a5 100644
--- a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/PipelineElementPartitioner.java
+++ b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/PipelineElementPartitioner.java
@@ -25,7 +25,7 @@ import org.apache.streampipes.model.graph.DataProcessorInvocation;
import org.apache.streampipes.model.graph.DataSinkInvocation;
import org.apache.streampipes.model.loadbalancer.LoadBalanceResourceUnit;
import org.apache.streampipes.model.pipeline.Pipeline;
-import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
import java.util.ArrayList;
@@ -125,7 +125,7 @@ public class PipelineElementPartitioner {
* @param pipeline Pipeline to partition
* @return Partition result containing resource units and their compatible services
*/
- public static PartitionResult partitionPipeline(Pipeline pipeline) {
+ public static PartitionResult partitionPipeline(Pipeline pipeline, ISpServiceDiscovery serviceDiscovery) {
if (pipeline == null) {
throw new IllegalArgumentException("Pipeline cannot be null");
}
@@ -133,7 +133,7 @@ public class PipelineElementPartitioner {
List<DataSinkInvocation> sinks = pipeline.getActions();
List<DataProcessorInvocation> processors = pipeline.getSepas();
- PartitionResult result = partitionElements(sinks, processors, pipeline.getLabels());
+ PartitionResult result = partitionElements(sinks, processors, pipeline.getLabels(), serviceDiscovery);
return result;
}
@@ -148,13 +148,14 @@ public class PipelineElementPartitioner {
public static PartitionResult partitionElements(
List<DataSinkInvocation> sinks,
List<DataProcessorInvocation> processors,
- List<String> labels) {
+ List<String> labels,
+ ISpServiceDiscovery serviceDiscovery) {
if ((sinks == null || sinks.isEmpty()) && (processors == null || processors.isEmpty())) {
return new PartitionResult(new ArrayList<>());
}
- List<SpServiceRegistration> allServices = SpServiceDiscovery.getServiceDiscovery().findAll();
+ List<SpServiceRegistration> allServices = serviceDiscovery.findAll();
// Initialize data structures
Map<String, InvocableStreamPipesEntity> elementMap = new HashMap<>();
@@ -180,7 +181,8 @@ public class PipelineElementPartitioner {
* @param adapter Adapter description
* @return Adapter resource unit with compatible services
*/
- public static AdapterResourceUnitWithServices createAdapterResourceUnit(AdapterDescription adapter) {
+ public static AdapterResourceUnitWithServices createAdapterResourceUnit(AdapterDescription adapter,
+ ISpServiceDiscovery serviceDiscovery) {
if (adapter == null) {
throw new IllegalArgumentException("Adapter cannot be null");
}
@@ -189,7 +191,7 @@ public class PipelineElementPartitioner {
resourceUnit.addElement(adapter);
resourceUnit.setLabels(Collections.EMPTY_LIST);
- List<SpServiceRegistration> allServices = SpServiceDiscovery.getServiceDiscovery().findAll();
+ List<SpServiceRegistration> allServices = serviceDiscovery.findAll();
List<SpServiceRegistration> compatibleServices = findCompatibleServices(
SpServiceUrlProvider.ADAPTER.getServiceTag(adapter.getAppId()).asString(),
allServices);
diff --git a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitScanner.java b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitScanner.java
index 90124c56f5..8ee4695e4b 100644
--- a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitScanner.java
+++ b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitScanner.java
@@ -26,6 +26,7 @@ import org.apache.streampipes.model.loadbalancer.LoadBalanceResourceUnit;
import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.pipeline.IPipelineStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,7 +88,7 @@ public class ResourceUnitScanner {
// Scan and partition pipeline elements
List<PipelineElementPartitioner.PartitionResult> pipelineUnits =
- scanAndPartitionPipeline(service, resourceManager.managePipelines().getDb());
+ scanAndPartitionPipeline(service, resourceManager.managePipelines().getDb(), resourceManager.getServiceDiscovery());
// Scan and create adapter units
List<PipelineElementPartitioner.AdapterResourceUnitWithServices> adapterUnits =
@@ -107,7 +108,8 @@ public class ResourceUnitScanner {
*/
private static List<LoadBalanceResourceUnit<InvocableStreamPipesEntity>> scanAndPartitionPipelineElements(
SpServiceRegistration service,
- IPipelineStorage pipelineStorage) {
+ IPipelineStorage pipelineStorage,
+ ISpServiceDiscovery serviceDiscovery) {
String serviceUrl = service.getServiceUrl();
List<Pipeline> allPipelines = pipelineStorage.findAll();
@@ -134,7 +136,7 @@ public class ResourceUnitScanner {
// Use PipelineElementPartitioner to partition these elements
PipelineElementPartitioner.PartitionResult partitionResult = PipelineElementPartitioner
- .partitionElements(serviceSinks, serviceProcessors, pipeline.getLabels());
+ .partitionElements(serviceSinks, serviceProcessors, pipeline.getLabels(), serviceDiscovery);
// Add partitioned units with service ID set
for (PipelineElementPartitioner.ResourceUnitWithServices unitWithServices : partitionResult
@@ -158,7 +160,8 @@ public class ResourceUnitScanner {
*/
private static List<PipelineElementPartitioner.PartitionResult> scanAndPartitionPipeline(
SpServiceRegistration service,
- IPipelineStorage pipelineStorage) {
+ IPipelineStorage pipelineStorage,
+ ISpServiceDiscovery serviceDiscovery) {
String serviceUrl = service.getServiceUrl();
List<Pipeline> allPipelines = pipelineStorage.findAll();
@@ -185,7 +188,7 @@ public class ResourceUnitScanner {
// Use PipelineElementPartitioner to partition these elements
PipelineElementPartitioner.PartitionResult partitionResult = PipelineElementPartitioner
- .partitionElements(serviceSinks, serviceProcessors, pipeline.getLabels());
+ .partitionElements(serviceSinks, serviceProcessors, pipeline.getLabels(), serviceDiscovery);
resourceUnits.add(partitionResult);
}
@@ -257,7 +260,7 @@ public class ResourceUnitScanner {
List<PipelineElementPartitioner.AdapterResourceUnitWithServices> adapterUnits =
new ArrayList<>();
for (AdapterDescription adapter : serviceAdapters) {
- adapterUnits.add(PipelineElementPartitioner.createAdapterResourceUnit(adapter));
+ adapterUnits.add(PipelineElementPartitioner.createAdapterResourceUnit(adapter, resourceManager.getServiceDiscovery()));
}
return adapterUnits;
@@ -305,8 +308,9 @@ public class ResourceUnitScanner {
*/
public static List<LoadBalanceResourceUnit<InvocableStreamPipesEntity>> findResourceUnitsForService(
SpServiceRegistration service,
- IPipelineStorage pipelineStorage) {
- return scanAndPartitionPipelineElements(service, pipelineStorage);
+ IPipelineStorage pipelineStorage,
+ ISpServiceDiscovery serviceDiscovery) {
+ return scanAndPartitionPipelineElements(service, pipelineStorage, serviceDiscovery);
}
/**
@@ -334,7 +338,7 @@ public class ResourceUnitScanner {
var pipelineStorage = resourceManager.managePipelines().getDb();
return sourceServices.stream().collect(Collectors
.toMap(SpServiceRegistration::getSvcId, svcreg ->
- ResourceUnitScanner.findResourceUnitsForService(svcreg, pipelineStorage)));
+ ResourceUnitScanner.findResourceUnitsForService(svcreg, pipelineStorage, resourceManager.getServiceDiscovery())));
}
/**
diff --git a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitStatsScanner.java b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitStatsScanner.java
index b5698f5f33..7cf21fa779 100644
--- a/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitStatsScanner.java
+++ b/streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitStatsScanner.java
@@ -30,7 +30,6 @@ import org.apache.streampipes.model.monitoring.SpMetricsEntry;
import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.pipeline.IPipelineStorage;
-import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,7 +65,7 @@ public class ResourceUnitStatsScanner {
// Generate stats for pipeline elements (sinks and processors)
List<LoadBalanceResourceUnit<InvocableStreamPipesEntity>> pipelineUnits =
- ResourceUnitScanner.findResourceUnitsForService(service, resourceManager.managePipelines().getDb());
+ ResourceUnitScanner.findResourceUnitsForService(service, resourceManager.managePipelines().getDb(), resourceManager.getServiceDiscovery());
for (LoadBalanceResourceUnit<InvocableStreamPipesEntity> unit : pipelineUnits) {
LoadBalanceResourceUnitStats<InvocableStreamPipesEntity> stats =
@@ -247,7 +246,7 @@ public class ResourceUnitStatsScanner {
// Get all services
List<SpServiceRegistration> services =
- SpServiceDiscovery.getServiceDiscovery().getService(true);
+ resourceManager.getServiceDiscovery().getService(true);
if (services.isEmpty()) {
logger.debug("No services found for metrics collection");
return;
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetFetcher.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetFetcher.java
index 920c312957..08e28c0ede 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetFetcher.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetFetcher.java
@@ -22,6 +22,7 @@ import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestMana
import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestTargets;
import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequests;
import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
import java.io.ByteArrayInputStream;
@@ -31,20 +32,23 @@ import java.util.Set;
public class AssetFetcher {
+ private final ISpServiceDiscovery serviceDiscovery;
private final SpServiceUrlProvider spServiceUrlProvider;
private final String appId;
private final ExtensionServiceRequestManager requestManager;
- public AssetFetcher(SpServiceUrlProvider spServiceUrlProvider,
+ public AssetFetcher(ISpServiceDiscovery serviceDiscovery,
+ SpServiceUrlProvider spServiceUrlProvider,
String appId,
ExtensionServiceRequestManager requestManager) {
+ this.serviceDiscovery = serviceDiscovery;
this.spServiceUrlProvider = spServiceUrlProvider;
this.appId = appId;
this.requestManager = requestManager;
}
public InputStream fetchPipelineElementAssets() throws IOException, NoServiceEndpointsAvailableException {
- var service = new ExtensionsServiceEndpointGenerator().selectService(appId, spServiceUrlProvider, Set.of());
+ var service = new ExtensionsServiceEndpointGenerator(serviceDiscovery).selectService(appId, spServiceUrlProvider, Set.of());
var requestTarget = ExtensionServiceRequestTargets.pipelineElementAssets(service, spServiceUrlProvider, appId);
var response = requestManager.request(ExtensionServiceRequests.pipelineElementAssets(requestTarget));
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetManager.java
index 61d7ea5ca8..525aa67a73 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetManager.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetManager.java
@@ -21,6 +21,7 @@ import org.apache.streampipes.commons.constants.GlobalStreamPipesConstants;
import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException;
import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestManager;
import org.apache.streampipes.storage.api.system.ISpCoreConfigurationStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
import org.apache.commons.io.FileUtils;
@@ -59,8 +60,9 @@ public class AssetManager {
public void storeAsset(SpServiceUrlProvider spServiceUrlProvider,
String appId,
- ExtensionServiceRequestManager requestManager) throws IOException, NoServiceEndpointsAvailableException {
- InputStream assetStream = new AssetFetcher(spServiceUrlProvider, appId, requestManager)
+ ExtensionServiceRequestManager requestManager,
+ ISpServiceDiscovery serviceDiscovery) throws IOException, NoServiceEndpointsAvailableException {
+ InputStream assetStream = new AssetFetcher(serviceDiscovery, spServiceUrlProvider, appId, requestManager)
.fetchPipelineElementAssets();
new AssetExtractor(assetStream, appId, coreConfigurationStorage).extractAssetContents();
}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/PipelineExecutionTaskFactory.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/PipelineExecutionTaskFactory.java
index 2fc33ffaf5..11a765fa62 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/PipelineExecutionTaskFactory.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/PipelineExecutionTaskFactory.java
@@ -43,7 +43,7 @@ public class PipelineExecutionTaskFactory {
return List.of(
new UpdateGroupIdTask(),
new SecretEncryptionTask(SecretProvider.getDecryptionService()),
- new DiscoverEndpointsTask(),
+ new DiscoverEndpointsTask(resourceManager.getServiceDiscovery()),
new SubmitRequestTask(new InvokePipelineElementSubmitter(pipeline, requestManager, resourceManager)),
new SecretEncryptionTask(SecretProvider.getEncryptionService()),
new AfterInvocationTask(PipelineStatusMessageType.PIPELINE_STARTED),
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java
index 32a0955885..54789a45f0 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java
@@ -24,7 +24,7 @@ import org.apache.streampipes.loadbalance.LoadManager;
import org.apache.streampipes.manager.api.extensions.IExtensionsServiceEndpointGenerator;
import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration;
import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag;
-import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
@@ -41,7 +41,11 @@ public class ExtensionsServiceEndpointGenerator implements IExtensionsServiceEnd
private static final Logger LOG =
LoggerFactory.getLogger(ExtensionsServiceEndpointGenerator.class);
- public ExtensionsServiceEndpointGenerator() {}
+ private final ISpServiceDiscovery serviceDiscovery;
+
+ public ExtensionsServiceEndpointGenerator(ISpServiceDiscovery serviceDiscovery) {
+ this.serviceDiscovery = serviceDiscovery;
+ }
public SpServiceRegistration selectService(String appId,
SpServiceUrlProvider spServiceUrlProvider,
@@ -74,7 +78,7 @@ public class ExtensionsServiceEndpointGenerator implements IExtensionsServiceEnd
private List<SpServiceRegistration> getServiceEndpoints(String appId,
SpServiceUrlProvider spServiceUrlProvider,
Set<SpServiceTag> customServiceTags) {
- return SpServiceDiscovery.getServiceDiscovery()
+ return serviceDiscovery
.getService(DefaultSpServiceTypes.EXT, true,
ExtensionsServiceEndpointUtils.getDesiredServiceTags(appId, spServiceUrlProvider, customServiceTags));
}
@@ -82,7 +86,7 @@ public class ExtensionsServiceEndpointGenerator implements IExtensionsServiceEnd
private SpServiceRegistration getServiceURL(String appId, SpServiceUrlProvider spServiceUrlProvider,
Set<SpServiceTag> customServiceTags) {
List<SpServiceRegistration> services =
- SpServiceDiscovery.getServiceDiscovery().getService(true).stream()
+ serviceDiscovery.getService(true).stream()
.filter(s -> filtersSupported(s, spServiceUrlProvider.getServiceTag(appId).asString()))
.toList();
if (services.isEmpty()) {
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/task/DiscoverEndpointsTask.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/task/DiscoverEndpointsTask.java
index d8b47ceddb..5191c15305 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/task/DiscoverEndpointsTask.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/task/DiscoverEndpointsTask.java
@@ -26,16 +26,24 @@ import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistratio
import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.model.pipeline.PipelineElementStatus;
import org.apache.streampipes.model.pipeline.PipelineOperationStatus;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
public class DiscoverEndpointsTask implements PipelineExecutionTask {
+
+ private final ISpServiceDiscovery serviceDiscovery;
+
+ public DiscoverEndpointsTask(ISpServiceDiscovery serviceDiscovery) {
+ this.serviceDiscovery = serviceDiscovery;
+ }
+
@Override
public void executeTask(Pipeline pipeline,
PipelineExecutionInfo executionInfo) {
- for (PipelineElementPartitioner.ResourceUnitWithServices unit : PipelineElementPartitioner.partitionPipeline(pipeline).getResourceUnits()){
+ for (PipelineElementPartitioner.ResourceUnitWithServices unit : PipelineElementPartitioner.partitionPipeline(pipeline, serviceDiscovery).getResourceUnits()){
SpServiceRegistration service = LoadManager.allocation(unit.getCompatibleServices(), pipeline.getLabels());
if (Objects.nonNull(service)) {
unit.getResourceUnit().getElements().forEach(el -> {
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java
index db75b5a556..af2b7f49b7 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java
@@ -19,6 +19,7 @@ package org.apache.streampipes.manager.file;
import org.apache.streampipes.commons.file.FileHasher;
import org.apache.streampipes.model.file.FileMetadata;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.sdk.helpers.Filetypes;
import org.apache.streampipes.storage.api.system.IFileMetadataStorage;
import org.apache.streampipes.storage.api.system.ISpCoreConfigurationStorage;
@@ -91,11 +92,11 @@ public class FileManager {
}
- public void deleteFile(String id) {
+ public void deleteFile(String id, ResourceDeletionManager resourceDeletionManager) {
var fileMetadata = fileMetadataStorage.getElementById(id);
if (fileMetadata != null) {
fileHandler.deleteFile(fileMetadata.getFilename());
- fileMetadataStorage.deleteElementById(id);
+ resourceDeletionManager.delete(fileMetadataStorage, id);
}
}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineVerificationHandlerV2.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineVerificationHandlerV2.java
index d57c4af51f..8995b7a300 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineVerificationHandlerV2.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineVerificationHandlerV2.java
@@ -33,6 +33,7 @@ import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.model.pipeline.PipelineModification;
import org.apache.streampipes.model.pipeline.PipelineModificationResult;
import org.apache.streampipes.model.pipeline.PipelineVerificationResult;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import java.util.ArrayList;
import java.util.Collections;
@@ -41,18 +42,21 @@ import java.util.Optional;
public class PipelineVerificationHandlerV2 {
+ private final ISpServiceDiscovery serviceDiscovery;
private final Pipeline pipeline;
private final ExtensionServiceRequestManager requestManager;
- public PipelineVerificationHandlerV2(Pipeline pipeline,
+ public PipelineVerificationHandlerV2(ISpServiceDiscovery serviceDiscovery,
+ Pipeline pipeline,
ExtensionServiceRequestManager requestManager) {
+ this.serviceDiscovery = serviceDiscovery;
this.pipeline = pipeline;
this.requestManager = requestManager;
}
public PipelineModificationMessage verifyPipeline() {
PipelineGraph graph = new PipelineGraphBuilder(pipeline).buildGraph();
- var steps = new PipelineValidationSteps().collect(requestManager);
+ var steps = new PipelineValidationSteps().collect(requestManager, serviceDiscovery);
return new PipelineModificationGenerator(graph, steps).buildPipelineModificationMessage();
}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java
index 70e1e35d76..c9ee72c879 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java
@@ -31,6 +31,7 @@ import org.apache.streampipes.resource.management.secret.SecretDecrypter;
import org.apache.streampipes.resource.management.secret.SecretService;
import org.apache.streampipes.sdk.helpers.Tuple2;
import org.apache.streampipes.serializers.json.JacksonSerializer;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
import com.google.gson.JsonSyntaxException;
@@ -40,22 +41,27 @@ import java.util.Set;
public class CustomTransformOutputSchemaGenerator extends OutputSchemaGenerator<CustomTransformOutputStrategy> {
+ private final ISpServiceDiscovery serviceDiscovery;
private final DataProcessorInvocation dataProcessorInvocation;
private final ExtensionServiceRequestManager requestManager;
- public static CustomTransformOutputSchemaGenerator from(OutputStrategy strategy,
+ public static CustomTransformOutputSchemaGenerator from(ISpServiceDiscovery serviceDiscovery,
+ OutputStrategy strategy,
DataProcessorInvocation invocation,
ExtensionServiceRequestManager requestManager) {
return new CustomTransformOutputSchemaGenerator(
+ serviceDiscovery,
(CustomTransformOutputStrategy) strategy,
invocation,
requestManager);
}
- public CustomTransformOutputSchemaGenerator(CustomTransformOutputStrategy strategy,
+ public CustomTransformOutputSchemaGenerator(ISpServiceDiscovery serviceDiscovery,
+ CustomTransformOutputStrategy strategy,
DataProcessorInvocation invocation,
ExtensionServiceRequestManager requestManager) {
super(strategy);
+ this.serviceDiscovery = serviceDiscovery;
this.dataProcessorInvocation = invocation;
this.requestManager = requestManager;
}
@@ -75,7 +81,7 @@ public class CustomTransformOutputSchemaGenerator extends OutputSchemaGenerator<
private EventSchema makeRequest() {
try {
String httpRequestBody = makeRequestBody();
- var service = new ExtensionsServiceEndpointGenerator().selectService(
+ var service = new ExtensionsServiceEndpointGenerator(serviceDiscovery).selectService(
dataProcessorInvocation.getAppId(),
SpServiceUrlProvider.DATA_PROCESSOR,
Set.of()
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/OutputSchemaFactory.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/OutputSchemaFactory.java
index 41bf674be8..72b3c61104 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/OutputSchemaFactory.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/OutputSchemaFactory.java
@@ -29,15 +29,19 @@ import org.apache.streampipes.model.output.ListOutputStrategy;
import org.apache.streampipes.model.output.OutputStrategy;
import org.apache.streampipes.model.output.TransformOutputStrategy;
import org.apache.streampipes.model.output.UserDefinedOutputStrategy;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
public class OutputSchemaFactory {
+ private final ISpServiceDiscovery serviceDiscovery;
private final OutputStrategy outputStrategy;
private final DataProcessorInvocation dataProcessorInvocation;
private final ExtensionServiceRequestManager requestManager;
- public OutputSchemaFactory(DataProcessorInvocation dataProcessorInvocation,
+ public OutputSchemaFactory(ISpServiceDiscovery serviceDiscovery,
+ DataProcessorInvocation dataProcessorInvocation,
ExtensionServiceRequestManager requestManager) {
+ this.serviceDiscovery = serviceDiscovery;
this.dataProcessorInvocation = dataProcessorInvocation;
this.outputStrategy = dataProcessorInvocation.getOutputStrategies().get(0);
this.requestManager = requestManager;
@@ -57,7 +61,7 @@ public class OutputSchemaFactory {
} else if (outputStrategy instanceof TransformOutputStrategy) {
return TransformOutputSchemaGenerator.from(outputStrategy, dataProcessorInvocation);
} else if (outputStrategy instanceof CustomTransformOutputStrategy) {
- return CustomTransformOutputSchemaGenerator.from(outputStrategy, dataProcessorInvocation, requestManager);
+ return CustomTransformOutputSchemaGenerator.from(serviceDiscovery, outputStrategy, dataProcessorInvocation, requestManager);
} else if (outputStrategy instanceof UserDefinedOutputStrategy) {
return UserDefinedOutputSchemaGenerator.from(outputStrategy);
} else {
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/ComputeOutputStep.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/ComputeOutputStep.java
index f33fa5dad1..a588b70dda 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/ComputeOutputStep.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/ComputeOutputStep.java
@@ -28,6 +28,7 @@ import org.apache.streampipes.model.output.OutputStrategy;
import org.apache.streampipes.model.pipeline.PipelineElementValidationInfo;
import org.apache.streampipes.model.schema.EventSchema;
import org.apache.streampipes.sdk.helpers.Tuple2;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import java.util.Collections;
import java.util.HashMap;
@@ -37,9 +38,11 @@ import java.util.Set;
public class ComputeOutputStep extends AbstractPipelineValidationStep {
+ private final ISpServiceDiscovery serviceDiscovery;
private final ExtensionServiceRequestManager requestManager;
- public ComputeOutputStep(ExtensionServiceRequestManager requestManager) {
+ public ComputeOutputStep(ISpServiceDiscovery serviceDiscovery, ExtensionServiceRequestManager requestManager) {
+ this.serviceDiscovery = serviceDiscovery;
this.requestManager = requestManager;
}
@@ -54,7 +57,7 @@ public class ComputeOutputStep extends AbstractPipelineValidationStep {
if (target instanceof DataProcessorInvocation) {
DataProcessorInvocation pe = (DataProcessorInvocation) target;
Tuple2<EventSchema, ? extends OutputStrategy> outputSettings;
- OutputSchemaGenerator<?> schemaGenerator = new OutputSchemaFactory(pe, requestManager)
+ OutputSchemaGenerator<?> schemaGenerator = new OutputSchemaFactory(serviceDiscovery, pe, requestManager)
.getOuputSchemaGenerator();
if (target.getInputStreams().size() == 1) {
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/PipelineValidationSteps.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/PipelineValidationSteps.java
index fb8e1c1681..ec5ac836c1 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/PipelineValidationSteps.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/PipelineValidationSteps.java
@@ -19,13 +19,15 @@
package org.apache.streampipes.manager.matching.v2.pipeline;
import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestManager;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import java.util.Arrays;
import java.util.List;
public class PipelineValidationSteps {
- public List<IPipelineValidationStep> collect(ExtensionServiceRequestManager requestManager) {
+ public List<IPipelineValidationStep> collect(ExtensionServiceRequestManager requestManager,
+ ISpServiceDiscovery serviceDiscovery) {
return Arrays.asList(
new MeasurementChangeValidationStep(),
new PrepareStep(),
@@ -33,7 +35,7 @@ public class PipelineValidationSteps {
new SchemaValidationStep(),
new UpdateStaticPropertiesStep(),
new UpdateOutputStrategiesStep(),
- new ComputeOutputStep(requestManager),
+ new ComputeOutputStep(serviceDiscovery, requestManager),
new CheckCompletedStep()
);
}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/ExtensionsServiceLogExecutor.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/ExtensionsServiceLogExecutor.java
index 5bac74bd1f..7a780bccbf 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/ExtensionsServiceLogExecutor.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/ExtensionsServiceLogExecutor.java
@@ -33,7 +33,6 @@ import org.apache.streampipes.model.graph.DataSinkInvocation;
import org.apache.streampipes.model.monitoring.SpEndpointMonitoringInfo;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.serializers.json.JacksonSerializer;
-import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -170,7 +169,7 @@ public class ExtensionsServiceLogExecutor implements Runnable {
}
private List<SpServiceRegistration> getActiveExtensionsEndpoints() {
- return SpServiceDiscovery.getServiceDiscovery().getService(DefaultSpServiceTypes.EXT, true, List.of());
+ return resourceManager.getServiceDiscovery().getService(DefaultSpServiceTypes.EXT, true, List.of());
}
private SpEndpointMonitoringInfo parseLogResponse(String response)
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/PipelineManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/PipelineManager.java
index c1615688f6..b033705414 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/PipelineManager.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/PipelineManager.java
@@ -28,7 +28,6 @@ import org.apache.streampipes.model.client.user.Permission;
import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.model.pipeline.PipelineHealthStatus;
import org.apache.streampipes.model.pipeline.PipelineOperationStatus;
-import org.apache.streampipes.resource.management.CrudResourceManager;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.pipeline.IPipelineStorage;
@@ -129,9 +128,7 @@ public class PipelineManager {
* @param pipelineId of pipeline to be deleted
*/
public void deletePipeline(String pipelineId) {
- var pipelineCrudResourceManager = new CrudResourceManager<>(
- pipelineStorage, Pipeline.class, resourceManager.managePermissions()
- );
+ var pipelineCrudResourceManager = resourceManager.managePipelines();
var pipeline = getPipeline(pipelineId);
if (Objects.nonNull(pipeline)) {
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java
index ba853aeefe..39ebb57f19 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java
@@ -27,16 +27,20 @@ import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.model.pipeline.PipelineModificationResult;
import org.apache.streampipes.model.pipeline.compact.CompactPipeline;
import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import java.util.UUID;
public class CompactPipelineManagement {
+ private final ISpServiceDiscovery serviceDiscovery;
private final IPipelineElementDescriptionStorage storage;
private final ExtensionServiceRequestManager requestManager;
- public CompactPipelineManagement(IPipelineElementDescriptionStorage storage,
+ public CompactPipelineManagement(ISpServiceDiscovery serviceDiscovery,
+ IPipelineElementDescriptionStorage storage,
ExtensionServiceRequestManager requestManager) {
+ this.serviceDiscovery = serviceDiscovery;
this.storage = storage;
this.requestManager = requestManager;
}
@@ -47,7 +51,7 @@ public class CompactPipelineManagement {
new PipelineElementConfigurationStep(storage).apply(pipeline, compactPipeline);
- return new PipelineVerificationHandlerV2(pipeline, requestManager).makeModifiedPipeline();
+ return new PipelineVerificationHandlerV2(serviceDiscovery, pipeline, requestManager).makeModifiedPipeline();
}
public CompactPipeline convertPipeline(Pipeline pipeline) {
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/update/PipelineUpdateCoordinator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/update/PipelineUpdateCoordinator.java
index 6b95a49770..7405727c07 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/update/PipelineUpdateCoordinator.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/update/PipelineUpdateCoordinator.java
@@ -98,7 +98,7 @@ public class PipelineUpdateCoordinator {
var updatedPipeline = updatePipeline(storedPipeline, affectedElementId, updatedStreamName, updatedEventSchema);
try {
- var verificationHandler = new PipelineVerificationHandlerV2(updatedPipeline, requestManager);
+ var verificationHandler = new PipelineVerificationHandlerV2(resourceManager.getServiceDiscovery(), updatedPipeline, requestManager);
var modificationMessage = verificationHandler.verifyPipeline();
var updateInfo = makeUpdateInfo(modificationMessage, updatedPipeline);
var modifiedPipeline = verificationHandler.makeModifiedPipeline(modificationMessage).pipeline();
@@ -142,7 +142,7 @@ public class PipelineUpdateCoordinator {
affectedPipelines.forEach(pipeline -> {
var updatedPipeline = updatePipeline(pipeline, affectedElementId, updatedStreamName, updatedEventSchema);
try {
- var verificationHandler = new PipelineVerificationHandlerV2(updatedPipeline, requestManager);
+ var verificationHandler = new PipelineVerificationHandlerV2(resourceManager.getServiceDiscovery(), updatedPipeline, requestManager);
var modificationMessage = verificationHandler.verifyPipeline();
var updateInfo = makeUpdateInfo(modificationMessage, updatedPipeline);
updateInfo.setChartSchemaUpdateInfos(
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java
index e59fd27559..a704193e2b 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java
@@ -62,7 +62,7 @@ public class PipelinePreview {
var elementIdMappings = new HashMap<String, String>();
pipeline.setActions(new ArrayList<>());
List<NamedStreamPipesEntity> pipelineElements = new ArrayList<>(
- new PipelineVerificationHandlerV2(pipeline, requestManager)
+ new PipelineVerificationHandlerV2(resourceManager.getServiceDiscovery(), pipeline, requestManager)
.verifyAndBuildGraphs(true)
.modifiedPipelineElements()
);
@@ -138,7 +138,7 @@ public class PipelinePreview {
}
private SpServiceRegistration findSelectedService(InvocableStreamPipesEntity g) throws NoServiceEndpointsAvailableException {
- return new ExtensionsServiceEndpointGenerator()
+ return new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery())
.selectService(
g.getAppId(),
ExtensionsServiceEndpointUtils.getPipelineElementType(g),
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/recommender/ElementRecommender.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/recommender/ElementRecommender.java
index b705b6eb5a..52240b52d7 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/recommender/ElementRecommender.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/recommender/ElementRecommender.java
@@ -34,6 +34,7 @@ import org.apache.streampipes.model.pipeline.PipelineModification;
import org.apache.streampipes.resource.management.DataProcessorResourceManager;
import org.apache.streampipes.resource.management.DataSinkResourceManager;
import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,6 +48,7 @@ public class ElementRecommender {
private static final Logger LOG = LoggerFactory.getLogger(ElementRecommender.class);
+ private final ISpServiceDiscovery serviceDiscovery;
private final IPipelineElementDescriptionStorage descriptionStorage;
private final Pipeline pipeline;
private final String baseRecDomId;
@@ -55,12 +57,14 @@ public class ElementRecommender {
private final DataProcessorResourceManager dataProcessorResourceManager;
private final DataSinkResourceManager dataSinkResourceManager;
- public ElementRecommender(Pipeline partialPipeline,
+ public ElementRecommender(ISpServiceDiscovery serviceDiscovery,
+ Pipeline partialPipeline,
String baseRecDomId,
ExtensionServiceRequestManager requestManager,
DataProcessorResourceManager dataProcessorResourceManager,
DataSinkResourceManager dataSinkResourceManager,
IPipelineElementDescriptionStorage descriptionStorage) {
+ this.serviceDiscovery = serviceDiscovery;
this.descriptionStorage = descriptionStorage;
this.pipeline = partialPipeline;
this.baseRecDomId = baseRecDomId;
@@ -133,7 +137,7 @@ public class ElementRecommender {
} else {
Pipeline partialPipeline =
new PartialPipelineGenerator(this.baseRecDomId, elementsProvider).makePartialPipeline();
- var modifications = new PipelineVerificationHandlerV2(partialPipeline, requestManager).verifyPipeline();
+ var modifications = new PipelineVerificationHandlerV2(serviceDiscovery, partialPipeline, requestManager).verifyPipeline();
return modifications.getPipelineModifications()
.stream()
.filter(m -> m.getDomId().equals(this.baseRecDomId))
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/remote/ContainerProvidedOptionsHandler.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/remote/ContainerProvidedOptionsHandler.java
index e49277ee19..1f0191bfd8 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/remote/ContainerProvidedOptionsHandler.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/remote/ContainerProvidedOptionsHandler.java
@@ -98,7 +98,7 @@ public class ContainerProvidedOptionsHandler {
private ExtensionServiceRequestTarget getEndpointRequestTarget(String appId)
throws NoServiceEndpointsAvailableException {
SpServiceUrlProvider provider = ExtensionsServiceEndpointUtils.getPipelineElementType(appId, descriptionStorage);
- var service = new ExtensionsServiceEndpointGenerator().selectService(appId, provider, Set.of());
+ var service = new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery()).selectService(appId, provider, Set.of());
return ExtensionServiceRequestTargets.containerProvidedOptions(service, provider, appId);
}
}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/PipelineElementInstallationStep.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/PipelineElementInstallationStep.java
index 08660aaf57..4bfb394f57 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/PipelineElementInstallationStep.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/PipelineElementInstallationStep.java
@@ -56,7 +56,7 @@ public class PipelineElementInstallationStep extends InstallationStep {
public void install() {
var installationReq = ExtensionItemInstallationRequest.fromDescription(extensionItem, true);
try {
- var service = new ExtensionsServiceEndpointGenerator().selectService(
+ var service = new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery()).selectService(
installationReq.appId(),
SpServiceUrlProvider.valueOf(installationReq.serviceTagPrefix().name()),
Set.of()
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/CompactPipelineTemplateManagement.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/CompactPipelineTemplateManagement.java
index a449b2a967..dd99076301 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/CompactPipelineTemplateManagement.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/template/compact/CompactPipelineTemplateManagement.java
@@ -28,6 +28,7 @@ import org.apache.streampipes.model.template.CompactPipelineTemplate;
import org.apache.streampipes.model.template.PipelineTemplateGenerationRequest;
import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import java.util.List;
import java.util.Map;
@@ -39,13 +40,16 @@ import static org.apache.streampipes.manager.pipeline.compact.generation.Invocab
public class CompactPipelineTemplateManagement {
+ private final ISpServiceDiscovery serviceDiscovery;
private final IPipelineElementDescriptionStorage storage;
private final ICompactPipelineTemplateStorage templateStorage;
private final ExtensionServiceRequestManager requestManager;
- public CompactPipelineTemplateManagement(ICompactPipelineTemplateStorage templateStorage,
+ public CompactPipelineTemplateManagement(ISpServiceDiscovery serviceDiscovery,
+ ICompactPipelineTemplateStorage templateStorage,
IPipelineElementDescriptionStorage descriptionStorage,
ExtensionServiceRequestManager requestManager) {
+ this.serviceDiscovery = serviceDiscovery;
this.templateStorage = templateStorage;
this.storage = descriptionStorage;
this.requestManager = requestManager;
@@ -68,7 +72,7 @@ public class CompactPipelineTemplateManagement {
}
var pipeline = makePipeline(template);
- return new PipelineVerificationHandlerV2(pipeline, requestManager).makeModifiedPipeline();
+ return new PipelineVerificationHandlerV2(serviceDiscovery, pipeline, requestManager).makeModifiedPipeline();
}
private Pipeline makePipeline(CompactPipelineTemplate template) throws Exception {
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/TypedElementVerifier.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/TypedElementVerifier.java
index f00bf3364c..b8bb1ac341 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/TypedElementVerifier.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/verification/TypedElementVerifier.java
@@ -23,6 +23,7 @@ import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestMana
import org.apache.streampipes.model.base.NamedStreamPipesEntity;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
import java.io.IOException;
@@ -36,6 +37,7 @@ public class TypedElementVerifier<T extends NamedStreamPipesEntity> extends Elem
private final Consumer<T> updateOperation;
private final SpServiceUrlProvider serviceUrlProvider;
private final ExtensionServiceRequestManager requestManager;
+ private final ISpServiceDiscovery serviceDiscovery;
public TypedElementVerifier(
String graphData,
@@ -54,6 +56,7 @@ public class TypedElementVerifier<T extends NamedStreamPipesEntity> extends Elem
this.updateOperation = updateOperation;
this.serviceUrlProvider = serviceUrlProvider;
this.requestManager = requestManager;
+ this.serviceDiscovery = resourceManager.getServiceDiscovery();
}
public TypedElementVerifier(
@@ -72,6 +75,7 @@ public class TypedElementVerifier<T extends NamedStreamPipesEntity> extends Elem
this.updateOperation = updateOperation;
this.serviceUrlProvider = serviceUrlProvider;
this.requestManager = requestManager;
+ this.serviceDiscovery = resourceManager.getServiceDiscovery();
}
@Override
@@ -91,7 +95,7 @@ public class TypedElementVerifier<T extends NamedStreamPipesEntity> extends Elem
@Override
protected void storeAssets() throws IOException, NoServiceEndpointsAvailableException {
if (elementDescription.isIncludesAssets()) {
- assetManager.storeAsset(serviceUrlProvider, elementDescription.getAppId(), requestManager);
+ assetManager.storeAsset(serviceUrlProvider, elementDescription.getAppId(), requestManager, serviceDiscovery);
}
}
}
diff --git a/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java
index 4e83c4cc15..0ab095a885 100644
--- a/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java
+++ b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java
@@ -19,7 +19,9 @@ package org.apache.streampipes.manager.file;
import org.apache.streampipes.commons.file.FileHasher;
import org.apache.streampipes.model.file.FileMetadata;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.storage.api.system.IFileMetadataStorage;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.BeforeEach;
@@ -199,10 +201,10 @@ public class TestFileManager {
when(fileMetadataStorage.getElementById(id)).thenReturn(fileMetadata);
- fileManager.deleteFile(id);
+ fileManager.deleteFile(id, new ResourceDeletionManager(mock(IGenericStorage.class)));
verify(fileHandler, times(1)).deleteFile(fileMetadata.getFilename());
- verify(fileMetadataStorage, times(1)).deleteElementById(id);
+ verify(fileMetadataStorage, times(1)).deleteElement(fileMetadata);
}
@Test
@@ -211,7 +213,7 @@ public class TestFileManager {
when(fileMetadataStorage.getElementById(id)).thenReturn(null);
- fileManager.deleteFile(id);
+ fileManager.deleteFile(id, new ResourceDeletionManager(mock(IGenericStorage.class)));
verify(fileHandler, times(0)).deleteFile(anyString());
verify(fileMetadataStorage, times(0)).deleteElementById(id);
diff --git a/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGeneratorTest.java b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGeneratorTest.java
index 4e33b8d7c8..a9986ab55c 100644
--- a/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGeneratorTest.java
+++ b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGeneratorTest.java
@@ -23,6 +23,7 @@ import org.apache.streampipes.model.graph.DataProcessorInvocation;
import org.apache.streampipes.model.output.CustomTransformOutputStrategy;
import org.apache.streampipes.model.staticproperty.SecretStaticProperty;
import org.apache.streampipes.serializers.json.JacksonSerializer;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.user.management.encryption.SecretEncryptionManager;
import org.junit.jupiter.api.Test;
@@ -46,6 +47,7 @@ class CustomTransformOutputSchemaGeneratorTest {
invocation.setStaticProperties(List.of(secretProperty));
var generator = new CustomTransformOutputSchemaGenerator(
+ mock(ISpServiceDiscovery.class),
new CustomTransformOutputStrategy(),
invocation,
mock(ExtensionServiceRequestManager.class)
diff --git a/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/pipeline/update/PipelineUpdateCoordinatorTest.java b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/pipeline/update/PipelineUpdateCoordinatorTest.java
index 4968a574bb..9d59dfbd11 100644
--- a/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/pipeline/update/PipelineUpdateCoordinatorTest.java
+++ b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/pipeline/update/PipelineUpdateCoordinatorTest.java
@@ -81,7 +81,7 @@ class PipelineUpdateCoordinatorTest {
try (MockedConstruction<PipelineVerificationHandlerV2> verificationHandlerConstruction =
mockConstruction(PipelineVerificationHandlerV2.class, (mock, context) -> {
- verifiedPipelines.add((Pipeline) context.arguments().get(0));
+ verifiedPipelines.add((Pipeline) context.arguments().get(1));
when(mock.verifyPipeline()).thenReturn(modificationMessage);
when(mock.makeModifiedPipeline(modificationMessage))
.thenReturn(new PipelineModificationResult(modifiedPipeline, List.of()));
@@ -122,7 +122,7 @@ class PipelineUpdateCoordinatorTest {
try (MockedConstruction<PipelineVerificationHandlerV2> verificationHandlerConstruction =
mockConstruction(PipelineVerificationHandlerV2.class, (mock, context) -> {
- verifiedPipelines.add((Pipeline) context.arguments().get(0));
+ verifiedPipelines.add((Pipeline) context.arguments().get(1));
when(mock.verifyPipeline()).thenReturn(modificationMessage);
when(mock.makeModifiedPipeline(modificationMessage))
.thenReturn(new PipelineModificationResult(modifiedPipeline, List.of()));
@@ -203,10 +203,10 @@ class PipelineUpdateCoordinatorTest {
try (MockedConstruction<PipelineVerificationHandlerV2> verificationHandlerConstruction =
mockConstruction(PipelineVerificationHandlerV2.class, (mock, context) -> {
- verifiedPipelines.add((Pipeline) context.arguments().get(0));
+ verifiedPipelines.add((Pipeline) context.arguments().get(1));
when(mock.verifyPipeline()).thenReturn(modificationMessage);
when(mock.makeModifiedPipeline(modificationMessage))
- .thenReturn(new PipelineModificationResult((Pipeline) context.arguments().get(0), List.of()));
+ .thenReturn(new PipelineModificationResult((Pipeline) context.arguments().get(1), List.of()));
})) {
var result = coordinator.checkPipelineMigrations(dataStream);
@@ -240,10 +240,10 @@ class PipelineUpdateCoordinatorTest {
try (MockedConstruction<PipelineVerificationHandlerV2> verificationHandlerConstruction =
mockConstruction(PipelineVerificationHandlerV2.class, (mock, context) -> {
- verifiedPipelines.add((Pipeline) context.arguments().get(0));
+ verifiedPipelines.add((Pipeline) context.arguments().get(1));
when(mock.verifyPipeline()).thenReturn(modificationMessage);
when(mock.makeModifiedPipeline(modificationMessage))
- .thenReturn(new PipelineModificationResult((Pipeline) context.arguments().get(0), List.of()));
+ .thenReturn(new PipelineModificationResult((Pipeline) context.arguments().get(1), List.of()));
})) {
var result = coordinator.checkPipelineMigrations(dataStream);
@@ -283,7 +283,7 @@ class PipelineUpdateCoordinatorTest {
mockConstruction(PipelineVerificationHandlerV2.class, (mock, context) -> {
when(mock.verifyPipeline()).thenReturn(modificationMessage);
when(mock.makeModifiedPipeline(modificationMessage))
- .thenReturn(new PipelineModificationResult((Pipeline) context.arguments().get(0), List.of()));
+ .thenReturn(new PipelineModificationResult((Pipeline) context.arguments().get(1), List.of()));
})) {
var result = coordinator.checkPipelineMigrations(dataStream);
diff --git a/streampipes-resource-management/pom.xml b/streampipes-resource-management/pom.xml
index d460743c7a..8f0786e023 100644
--- a/streampipes-resource-management/pom.xml
+++ b/streampipes-resource-management/pom.xml
@@ -27,6 +27,11 @@
<artifactId>streampipes-resource-management</artifactId>
<dependencies>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-service-discovery-api</artifactId>
+ <version>0.99.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-audit-api</artifactId>
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AdapterResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AdapterResourceManager.java
index b03d2e6248..4199953166 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AdapterResourceManager.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AdapterResourceManager.java
@@ -32,13 +32,16 @@ import org.springframework.security.core.Authentication;
public class AdapterResourceManager extends AbstractResourceManager<IAdapterStorage> {
private final ICertificateStorage certificateStorage;
+ private final ResourceDeletionManager resourceDeletionManager;
private final SpPermissionEvaluator permissionEvaluator;
public AdapterResourceManager(IAdapterStorage adapterStorage,
ICertificateStorage certificateStorage,
- PermissionResourceManager permissionResourceManager) {
+ PermissionResourceManager permissionResourceManager,
+ ResourceDeletionManager resourceDeletionManager) {
super(adapterStorage);
this.certificateStorage = certificateStorage;
+ this.resourceDeletionManager = resourceDeletionManager;
this.permissionEvaluator = new SpPermissionEvaluator(permissionResourceManager.getDb());
}
@@ -105,7 +108,7 @@ public class AdapterResourceManager extends AbstractResourceManager<IAdapterStor
});
// Then delete the adapter
- db.deleteElementById(elementId);
+ resourceDeletionManager.delete(db, elementId);
}
/**
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AssetResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AssetResourceManager.java
index b01892003e..1c6a9d8685 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AssetResourceManager.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/AssetResourceManager.java
@@ -31,8 +31,9 @@ public class AssetResourceManager extends CrudResourceManager<SpAssetModel, IAss
private final SpPermissionEvaluator permissionEvaluator;
public AssetResourceManager(IAssetStorage assetStorage,
- PermissionResourceManager permissionResourceManager) {
- super(assetStorage, SpAssetModel.class, permissionResourceManager);
+ PermissionResourceManager permissionResourceManager,
+ ResourceDeletionManager resourceDeletionManager) {
+ super(assetStorage, SpAssetModel.class, permissionResourceManager, resourceDeletionManager);
this.permissionEvaluator = new SpPermissionEvaluator(permissionResourceManager.getDb());
}
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/ChartResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/ChartResourceManager.java
index 7b15372d25..e655c72c41 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/ChartResourceManager.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/ChartResourceManager.java
@@ -36,8 +36,9 @@ public class ChartResourceManager
public ChartResourceManager(DashboardResourceManager dashboardManager,
IChartStorage db,
- PermissionResourceManager permissionResourceManager) {
- super(db, DataExplorerWidgetModel.class, permissionResourceManager);
+ PermissionResourceManager permissionResourceManager,
+ ResourceDeletionManager resourceDeletionManager) {
+ super(db, DataExplorerWidgetModel.class, permissionResourceManager, resourceDeletionManager);
this.dashboardManager = dashboardManager;
}
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/CrudResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/CrudResourceManager.java
index 38072a624f..46b7e97fdb 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/CrudResourceManager.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/CrudResourceManager.java
@@ -29,14 +29,17 @@ public class CrudResourceManager<T extends Storable, SeT extends CRUDStorage<T>>
extends AbstractResourceManager<SeT> {
private final Class<T> elementClass;
+ private final ResourceDeletionManager resourceDeletionManager;
protected final SpPermissionEvaluator permissionEvaluator;
protected final PermissionResourceManager permissionResourceManager;
public CrudResourceManager(SeT db,
Class<T> elementClass,
- PermissionResourceManager permissionResourceManager) {
+ PermissionResourceManager permissionResourceManager,
+ ResourceDeletionManager resourceDeletionManager) {
super(db);
this.elementClass = elementClass;
+ this.resourceDeletionManager = resourceDeletionManager;
this.permissionEvaluator = new SpPermissionEvaluator(permissionResourceManager.getDb());
this.permissionResourceManager = permissionResourceManager;
}
@@ -50,7 +53,7 @@ public class CrudResourceManager<T extends Storable, SeT extends CRUDStorage<T>>
}
public void delete(String elementId) {
- db.deleteElementById(elementId);
+ resourceDeletionManager.delete(db, elementId);
deletePermissions(elementId);
}
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/DashboardResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/DashboardResourceManager.java
index cc736e4856..4873b3f618 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/DashboardResourceManager.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/DashboardResourceManager.java
@@ -39,8 +39,9 @@ public class DashboardResourceManager extends CrudResourceManager<DashboardModel
public DashboardResourceManager(IDashboardStorage dashboardStorage,
IChartStorage widgetStorage,
IDatasetMetadataStorage datasetMetadataStorage,
- PermissionResourceManager permissionResourceManager) {
- super(dashboardStorage, DashboardModel.class, permissionResourceManager);
+ PermissionResourceManager permissionResourceManager,
+ ResourceDeletionManager resourceDeletionManager) {
+ super(dashboardStorage, DashboardModel.class, permissionResourceManager, resourceDeletionManager);
this.widgetStorage = widgetStorage;
this.datasetMetadataStorage = datasetMetadataStorage;
}
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/DataStreamResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/DataStreamResourceManager.java
index 035e56023f..bba7114f76 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/DataStreamResourceManager.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/DataStreamResourceManager.java
@@ -19,18 +19,25 @@ package org.apache.streampipes.resource.management;
import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
public class DataStreamResourceManager
extends AbstractPipelineElementResourceManager<IDataStreamStorage, SpDataStream, SpDataStream> {
+ private final ResourceDeletionManager resourceDeletionManager;
+
public DataStreamResourceManager(IDataStreamStorage db,
- PermissionResourceManager permissionResourceManager) {
+ PermissionResourceManager permissionResourceManager,
+ ResourceDeletionManager resourceDeletionManager) {
super(db, permissionResourceManager);
+ this.resourceDeletionManager = resourceDeletionManager;
}
- public DataStreamResourceManager(PermissionResourceManager permissionResourceManager) {
- super(StorageDispatcher.INSTANCE.getNoSqlStore().getDataStreamStorage(), permissionResourceManager);
+ @Override
+ public void delete(String elementId) {
+ if (find(elementId) != null) {
+ resourceDeletionManager.removeAssetLinks(elementId);
+ super.delete(elementId);
+ }
}
@Override
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/PipelineResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/PipelineResourceManager.java
index d593d5058f..e79349aa8c 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/PipelineResourceManager.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/PipelineResourceManager.java
@@ -27,8 +27,9 @@ import org.springframework.security.core.Authentication;
public class PipelineResourceManager extends CrudResourceManager<Pipeline, IPipelineStorage> {
public PipelineResourceManager(IPipelineStorage pipelineStorage,
- PermissionResourceManager permissionResourceManager) {
- super(pipelineStorage, Pipeline.class, permissionResourceManager);
+ PermissionResourceManager permissionResourceManager,
+ ResourceDeletionManager resourceDeletionManager) {
+ super(pipelineStorage, Pipeline.class, permissionResourceManager, resourceDeletionManager);
}
public ResourceSummaryDto<PipelineSummaryDto> getSummary(Authentication auth) {
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/ResourceDeletionManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/ResourceDeletionManager.java
new file mode 100644
index 0000000000..8de6a0a666
--- /dev/null
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/ResourceDeletionManager.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.resource.management;
+
+import org.apache.streampipes.storage.api.core.CRUDStorage;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/** Coordinates document deletion and asset-link cleanup for linkable resources. */
+public class ResourceDeletionManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ResourceDeletionManager.class);
+
+ private final IGenericStorage genericStorage;
+
+ public ResourceDeletionManager(IGenericStorage genericStorage) {
+ this.genericStorage = genericStorage;
+ }
+
+ public <T> void delete(CRUDStorage<T> storage, String resourceId) {
+ var resource = storage.getElementById(resourceId);
+ if (resource != null) {
+ removeAssetLinks(resourceId);
+ storage.deleteElement(resource);
+ }
+ }
+
+ public void removeAssetLinks(String resourceId) {
+ try {
+ genericStorage.deleteAssetLinkToResource(resourceId);
+ } catch (IOException e) {
+ LOG.error("Asset links for {} could not be deleted", resourceId, e);
+ }
+ }
+}
diff --git a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/SpResourceManager.java b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/SpResourceManager.java
index 425bf3dcfa..3a311a452d 100644
--- a/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/SpResourceManager.java
+++ b/streampipes-resource-management/src/main/java/org/apache/streampipes/resource/management/SpResourceManager.java
@@ -22,13 +22,16 @@ import org.apache.streampipes.storage.api.connect.IAdapterStorage;
import org.apache.streampipes.storage.api.explorer.IChartStorage;
import org.apache.streampipes.storage.api.explorer.IDashboardStorage;
import org.apache.streampipes.storage.api.explorer.IDatasetMetadataStorage;
-import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
import org.apache.streampipes.storage.api.pipeline.IDataProcessorStorage;
import org.apache.streampipes.storage.api.pipeline.IDataSinkStorage;
+import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
+import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
import org.apache.streampipes.storage.api.pipeline.IPipelineStorage;
import org.apache.streampipes.storage.api.system.IAssetStorage;
import org.apache.streampipes.storage.api.system.ICertificateStorage;
+import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
import org.apache.streampipes.storage.api.system.IFileMetadataStorage;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import org.apache.streampipes.storage.api.system.ISpCoreConfigurationStorage;
import org.apache.streampipes.storage.api.user.IPasswordRecoveryTokenStorage;
import org.apache.streampipes.storage.api.user.IPermissionStorage;
@@ -37,7 +40,7 @@ import org.apache.streampipes.storage.api.user.IRoleStorage;
import org.apache.streampipes.storage.api.user.IUserActivationTokenStorage;
import org.apache.streampipes.storage.api.user.IUserGroupStorage;
import org.apache.streampipes.storage.api.user.IUserStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import java.util.Objects;
@@ -60,9 +63,15 @@ public class SpResourceManager {
private final IUserStorage userStorage;
private final IDataProcessorStorage dataProcessorStorage;
private final IDataSinkStorage dataSinkStorage;
+ private final IDataStreamStorage dataStreamStorage;
+ private final IPipelineElementDescriptionStorage descriptionStorage;
+ private final IGenericStorage genericStorage;
+ private final ResourceDeletionManager resourceDeletionManager;
private final IUserActivationTokenStorage userActivationTokenStorage;
private final IPasswordRecoveryTokenStorage passwordRecoveryTokenStorage;
private final ICertificateStorage certificateStorage;
+ private final IExtensionsServiceStorage extensionsServiceStorage;
+ private final ISpServiceDiscovery serviceDiscovery;
public SpResourceManager(IPermissionStorage permissionStorage,
IChartStorage chartStorage,
@@ -80,14 +89,20 @@ public class SpResourceManager {
IUserStorage userStorage,
IDataProcessorStorage dataProcessorStorage,
IDataSinkStorage dataSinkStorage,
+ IDataStreamStorage dataStreamStorage,
+ IPipelineElementDescriptionStorage descriptionStorage,
+ IGenericStorage genericStorage,
IUserActivationTokenStorage userActivationTokenStorage,
IPasswordRecoveryTokenStorage passwordRecoveryTokenStorage,
- ICertificateStorage certificateStorage) {
+ ICertificateStorage certificateStorage,
+ IExtensionsServiceStorage extensionsServiceStorage,
+ ISpServiceDiscovery serviceDiscovery) {
this(permissionStorage, chartStorage, adapterStorage, adapterDescriptionStorage,
assetStorage, dashboardStorage, pipelineStorage,
datasetStorage, coreConfigurationStorage, fileMetadataStorage, roleStorage, userGroupStorage,
- privilegeStorage, userStorage, dataProcessorStorage, dataSinkStorage, userActivationTokenStorage,
- passwordRecoveryTokenStorage, certificateStorage,
+ privilegeStorage, userStorage, dataProcessorStorage, dataSinkStorage, dataStreamStorage, descriptionStorage, genericStorage,
+ userActivationTokenStorage,
+ passwordRecoveryTokenStorage, certificateStorage, extensionsServiceStorage, serviceDiscovery,
AuditService.disabled());
}
@@ -107,9 +122,14 @@ public class SpResourceManager {
IUserStorage userStorage,
IDataProcessorStorage dataProcessorStorage,
IDataSinkStorage dataSinkStorage,
+ IDataStreamStorage dataStreamStorage,
+ IPipelineElementDescriptionStorage descriptionStorage,
+ IGenericStorage genericStorage,
IUserActivationTokenStorage userActivationTokenStorage,
IPasswordRecoveryTokenStorage passwordRecoveryTokenStorage,
ICertificateStorage certificateStorage,
+ IExtensionsServiceStorage extensionsServiceStorage,
+ ISpServiceDiscovery serviceDiscovery,
AuditService auditService) {
this.auditService = Objects.requireNonNull(auditService);
this.permissionStorage = permissionStorage;
@@ -128,9 +148,35 @@ public class SpResourceManager {
this.userStorage = userStorage;
this.dataProcessorStorage = dataProcessorStorage;
this.dataSinkStorage = dataSinkStorage;
+ this.dataStreamStorage = dataStreamStorage;
+ this.descriptionStorage = descriptionStorage;
+ this.genericStorage = genericStorage;
+ this.resourceDeletionManager = new ResourceDeletionManager(genericStorage);
this.userActivationTokenStorage = userActivationTokenStorage;
this.passwordRecoveryTokenStorage = passwordRecoveryTokenStorage;
this.certificateStorage = certificateStorage;
+ this.extensionsServiceStorage = extensionsServiceStorage;
+ this.serviceDiscovery = serviceDiscovery;
+ }
+
+ public ResourceDeletionManager getResourceDeletionManager() {
+ return resourceDeletionManager;
+ }
+
+ public IGenericStorage getGenericStorage() {
+ return genericStorage;
+ }
+
+ public IPipelineElementDescriptionStorage getPipelineElementDescriptionStorage() {
+ return descriptionStorage;
+ }
+
+ public ISpServiceDiscovery getServiceDiscovery() {
+ return serviceDiscovery;
+ }
+
+ public IExtensionsServiceStorage getExtensionsServiceStorage() {
+ return extensionsServiceStorage;
}
public AuditService getAuditService() {
@@ -150,15 +196,15 @@ public class SpResourceManager {
}
public DataStreamResourceManager manageDataStreams() {
- return new DataStreamResourceManager(managePermissions());
+ return new DataStreamResourceManager(dataStreamStorage, managePermissions(), resourceDeletionManager);
}
public AssetResourceManager manageAssets() {
- return new AssetResourceManager(assetStorage, managePermissions());
+ return new AssetResourceManager(assetStorage, managePermissions(), resourceDeletionManager);
}
public AdapterResourceManager manageAdapters() {
- return new AdapterResourceManager(adapterStorage, certificateStorage, managePermissions());
+ return new AdapterResourceManager(adapterStorage, certificateStorage, managePermissions(), resourceDeletionManager);
}
public DatasetMetadataResourceManager manageDataLakeMeasures() {
@@ -170,26 +216,22 @@ public class SpResourceManager {
}
public DashboardResourceManager manageDashboards() {
- return new DashboardResourceManager(dashboardStorage, chartStorage, datasetStorage, managePermissions());
+ return new DashboardResourceManager(dashboardStorage, chartStorage, datasetStorage, managePermissions(),
+ resourceDeletionManager);
}
public ChartResourceManager manageCharts() {
- return new ChartResourceManager(manageDashboards(), chartStorage, managePermissions());
+ return new ChartResourceManager(manageDashboards(), chartStorage, managePermissions(), resourceDeletionManager);
}
public PipelineResourceManager managePipelines() {
- return new PipelineResourceManager(pipelineStorage, managePermissions()
- );
+ return new PipelineResourceManager(pipelineStorage, managePermissions(), resourceDeletionManager);
}
public ISpCoreConfigurationStorage getCoreConfigurationStorage() {
return coreConfigurationStorage;
}
- public ICompactPipelineTemplateStorage getPipelineTemplateStorage() {
- return StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineTemplateStorage();
- }
-
public IFileMetadataStorage getFileMetadataStorage() {
return fileMetadataStorage;
}
diff --git a/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/AdapterResourceManagerTest.java b/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/AdapterResourceManagerTest.java
index 8e6fbf767f..3e8d475438 100644
--- a/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/AdapterResourceManagerTest.java
+++ b/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/AdapterResourceManagerTest.java
@@ -44,7 +44,7 @@ public class AdapterResourceManagerTest {
void setUp() {
storage = mock(IAdapterStorage.class);
PermissionResourceManager permissionResourceManager = mock(PermissionResourceManager.class);
- adapterResourceManager = new AdapterResourceManager(storage, null, permissionResourceManager);
+ adapterResourceManager = new AdapterResourceManager(storage, null, permissionResourceManager, mock(ResourceDeletionManager.class));
}
@Test
diff --git a/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/DataStreamResourceManagerTest.java b/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/DataStreamResourceManagerTest.java
index 15fafd3861..df79bd6b92 100644
--- a/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/DataStreamResourceManagerTest.java
+++ b/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/DataStreamResourceManagerTest.java
@@ -20,21 +20,43 @@ package org.apache.streampipes.resource.management;
import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import org.junit.jupiter.api.Test;
+import java.io.IOException;
+
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class DataStreamResourceManagerTest {
+ @Test
+ void deletionCleansAssetLinksAndPermissions() throws IOException {
+ var storage = mock(IDataStreamStorage.class);
+ var permissions = mock(PermissionResourceManager.class);
+ var genericStorage = mock(IGenericStorage.class);
+ var stream = new SpDataStream();
+ stream.setElementId("stream-1");
+ when(storage.getElementById("stream-1")).thenReturn(stream);
+
+ new DataStreamResourceManager(storage, permissions, new ResourceDeletionManager(genericStorage))
+ .delete("stream-1");
+
+ verify(genericStorage).deleteAssetLinkToResource("stream-1");
+ verify(permissions).findForObjectId("stream-1");
+ verify(storage).deleteElement(stream);
+ }
+
@Test
public void update() {
IDataStreamStorage storage = mock(IDataStreamStorage.class);
PermissionResourceManager permissionResourceManager = mock(PermissionResourceManager.class);
- DataStreamResourceManager dataStreamResourceManager = new DataStreamResourceManager(storage, permissionResourceManager);
+ DataStreamResourceManager dataStreamResourceManager = new DataStreamResourceManager(
+ storage, permissionResourceManager, mock(ResourceDeletionManager.class));
dataStreamResourceManager.update(new SpDataStream());
verify(storage, times(1)).updateElement(any());
diff --git a/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/ResourceDeletionManagerTest.java b/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/ResourceDeletionManagerTest.java
new file mode 100644
index 0000000000..f5b173a0b1
--- /dev/null
+++ b/streampipes-resource-management/src/test/java/org/apache/streampipes/resource/management/ResourceDeletionManagerTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.resource.management;
+
+import org.apache.streampipes.model.assets.SpAssetModel;
+import org.apache.streampipes.model.client.user.Permission;
+import org.apache.streampipes.storage.api.system.IAssetStorage;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+class ResourceDeletionManagerTest {
+
+ private final IGenericStorage genericStorage = mock(IGenericStorage.class);
+ private final IAssetStorage storage = mock(IAssetStorage.class);
+ private final ResourceDeletionManager manager = new ResourceDeletionManager(genericStorage);
+ private final SpAssetModel asset = new SpAssetModel();
+
+ @BeforeEach
+ void setUp() {
+ asset.setElementId("asset-1");
+ when(storage.getElementById("asset-1")).thenReturn(asset);
+ }
+
+ @Test
+ void removesAssetLinksBeforeDeletingTheDocument() throws IOException {
+ manager.delete(storage, "asset-1");
+
+ var order = inOrder(genericStorage, storage);
+ order.verify(genericStorage).deleteAssetLinkToResource("asset-1");
+ order.verify(storage).deleteElement(asset);
+ }
+
+ @Test
+ void stillDeletesTheDocumentWhenAssetLinkCleanupFails() throws IOException {
+ doThrow(new IOException("unavailable")).when(genericStorage).deleteAssetLinkToResource("asset-1");
+
+ manager.delete(storage, "asset-1");
+
+ verify(storage).deleteElement(asset);
+ }
+
+ @Test
+ void resourceManagerDeletesPermissionsAfterDeletingTheDocument() throws IOException {
+ var permissions = mock(PermissionResourceManager.class);
+ var permission = new Permission();
+ when(permissions.findForObjectId("asset-1")).thenReturn(List.of(permission));
+
+ new AssetResourceManager(storage, permissions, manager).delete("asset-1");
+
+ var order = inOrder(genericStorage, storage, permissions);
+ order.verify(genericStorage).deleteAssetLinkToResource("asset-1");
+ order.verify(storage).deleteElement(asset);
+ order.verify(permissions).delete(permission);
+ }
+
+ @Test
+ void doesNotCleanLinksWhenTheResourceDoesNotExist() {
+ when(storage.getElementById("asset-1")).thenReturn(null);
+
+ manager.delete(storage, "asset-1");
+
+ verifyNoInteractions(genericStorage);
+ verify(storage, never()).deleteElement(asset);
+ }
+}
diff --git a/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java b/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java
index 8848ac0ba2..a8df9dbc9b 100644
--- a/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java
+++ b/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java
@@ -23,22 +23,11 @@ import org.apache.streampipes.model.message.Message;
import org.apache.streampipes.model.message.Notification;
import org.apache.streampipes.model.message.SuccessMessage;
import org.apache.streampipes.rest.shared.impl.AbstractSharedRestInterface;
-import org.apache.streampipes.storage.api.core.INoSqlStorage;
-import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import org.springframework.http.ResponseEntity;
public class AbstractRestResource extends AbstractSharedRestInterface {
- protected IPipelineElementDescriptionStorage getPipelineElementStorage() {
- return getNoSqlStorage().getPipelineElementDescriptionStorage();
- }
-
- protected INoSqlStorage getNoSqlStorage() {
- return StorageDispatcher.INSTANCE.getNoSqlStore();
- }
-
protected ResponseEntity<Message> constructSuccessMessage(Notification... notifications) {
return statusMessage(new SuccessMessage(notifications));
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
index e77f1b853f..e60ad33d53 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
@@ -39,7 +39,6 @@ import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.pipeline.IPipelineElementTemplateStorage;
import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
import org.apache.streampipes.storage.api.system.IGenericStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,6 +52,7 @@ public class ResetManagement {
// dependency between this package and streampipes-pipeline-management
// See in issue [STREAMPIPES-405]
+ private final IGenericStorage genericStorage;
private final WorkerRestClient workerRestClient;
private final IExtensionsServiceStorage extensionsServiceStorage;
private final ExtensionServiceRequestManager requestManager;
@@ -62,6 +62,7 @@ public class ResetManagement {
private final IPipelineElementTemplateStorage pipelineElementTemplateStorage;
public ResetManagement(WorkerRestClient workerRestClient,
+ IGenericStorage genericStorage,
IExtensionsServiceStorage extensionsServiceStorage,
ExtensionServiceRequestManager requestManager,
PipelineManager pipelineManager,
@@ -69,6 +70,7 @@ public class ResetManagement {
IPipelineElementTemplateStorage pipelineElementTemplateStorage,
ChartSchemaUpdateCoordinator chartSchemaUpdateCoordinator) {
this.workerRestClient = workerRestClient;
+ this.genericStorage = genericStorage;
this.extensionsServiceStorage = extensionsServiceStorage;
this.requestManager = requestManager;
this.pipelineManager = pipelineManager;
@@ -158,7 +160,7 @@ public class ResetManagement {
resourceManager.getFileMetadataStorage()
);
List<FileMetadata> allFiles = fileManager.getAllFiles();
- allFiles.forEach(fileMetadata -> fileManager.deleteFile(fileMetadata.getFileId()));
+ allFiles.forEach(fileMetadata -> fileManager.deleteFile(fileMetadata.getFileId(), resourceManager.getResourceDeletionManager()));
}
private void removeAllDataInDataLake() {
@@ -166,7 +168,8 @@ public class ResetManagement {
.getSchemaManagement(
chartSchemaUpdateCoordinator,
resourceManager.managePermissions().getDb(),
- resourceManager.manageDataLakeMeasures().getDb());
+ resourceManager.manageDataLakeMeasures().getDb(),
+ resourceManager.getResourceDeletionManager());
var dataExplorerQueryManagement = new DataExplorerDispatcher()
.getDatasetServices(datasetMetadataManagement).administration();
List<DatasetMetadata> allMeasurements = datasetMetadataManagement.getAllMeasurements();
@@ -182,18 +185,16 @@ public class ResetManagement {
private void removeAllDataViewWidgets() {
resourceManager.manageCharts().findAll()
.forEach(widget ->
- resourceManager.manageCharts().getDb().deleteElementById(widget.getElementId()));
+ resourceManager.manageCharts().delete(widget.getElementId()));
}
private void removeAllDataViews() {
resourceManager.manageDashboards().findAll()
.forEach(dashboard ->
- resourceManager.manageDashboards().getDb().deleteElementById(dashboard.getElementId()));
+ resourceManager.manageDashboards().delete(dashboard.getElementId()));
}
private void removeAllAssets(String username) {
- IGenericStorage genericStorage = StorageDispatcher.INSTANCE.getNoSqlStore()
- .getGenericStorage();
try {
for (Map<String, Object> asset : genericStorage.findAll("asset-management")) {
genericStorage.delete((String) asset.get("_id"), (String) asset.get("_rev"));
@@ -216,8 +217,6 @@ public class ResetManagement {
"asset-sites",
"sp-labels"
);
- var genericStorage = StorageDispatcher.INSTANCE.getNoSqlStore()
- .getGenericStorage();
appDocTypesToDelete.forEach(docType -> {
try {
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/CompactPipelineResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/CompactPipelineResource.java
index 44718c542c..d346ebee6f 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/CompactPipelineResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/CompactPipelineResource.java
@@ -29,6 +29,7 @@ import org.apache.streampipes.model.pipeline.compact.CompactPipeline;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
import org.apache.streampipes.rest.security.AuthConstants;
+import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -49,9 +50,11 @@ public class CompactPipelineResource extends AbstractAuthGuardedRestResource {
private final PipelineManager pipelineManager;
public CompactPipelineResource(ExtensionServiceRequestManager requestManager,
- SpResourceManager resourceManager) {
+ SpResourceManager resourceManager,
+ IPipelineElementDescriptionStorage descriptionStorage) {
this.compactPipelineManagement = new CompactPipelineManagement(
- getPipelineElementStorage(),
+ resourceManager.getServiceDiscovery(),
+ descriptionStorage,
requestManager
);
this.requestManager = requestManager;
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java
index 3a31a9b166..b80cd36cce 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.rest.impl;
import org.apache.streampipes.manager.file.FileManager;
import org.apache.streampipes.model.file.FileMetadata;
import org.apache.streampipes.model.message.Notifications;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
import org.apache.streampipes.rest.security.AuthConstants;
import org.apache.streampipes.rest.shared.exception.SpMessageException;
@@ -57,10 +58,13 @@ import static org.springframework.http.HttpStatus.NOT_FOUND;
public class FileResource extends AbstractAuthGuardedRestResource {
private final FileManager fileManager;
+ private final ResourceDeletionManager resourceDeletionManager;
public FileResource(ISpCoreConfigurationStorage coreConfigurationStorage,
- IFileMetadataStorage fileMetadataStorage) {
+ IFileMetadataStorage fileMetadataStorage,
+ ResourceDeletionManager resourceDeletionManager) {
this.fileManager = new FileManager(coreConfigurationStorage, fileMetadataStorage);
+ this.resourceDeletionManager = resourceDeletionManager;
}
@PostMapping(
@@ -91,7 +95,7 @@ public class FileResource extends AbstractAuthGuardedRestResource {
@DeleteMapping(path = "{fileId}")
@PreAuthorize(AuthConstants.HAS_WRITE_FILE_PRIVILEGE)
public ResponseEntity<Void> deleteFile(@PathVariable("fileId") String fileId) {
- fileManager.deleteFile(fileId);
+ fileManager.deleteFile(fileId, resourceDeletionManager);
return ok();
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/GenericStorageResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/GenericStorageResource.java
index 06ec807f5b..76be2ca1ee 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/GenericStorageResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/GenericStorageResource.java
@@ -22,7 +22,6 @@ import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResourc
import org.apache.streampipes.rest.security.AuthConstants;
import org.apache.streampipes.rest.shared.exception.SpMessageException;
import org.apache.streampipes.storage.api.system.IGenericStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,6 +46,12 @@ import java.util.Map;
@RequestMapping("/api/v2/storage-generic")
public class GenericStorageResource extends AbstractAuthGuardedRestResource {
+ private final IGenericStorage genericStorage;
+
+ public GenericStorageResource(IGenericStorage genericStorage) {
+ this.genericStorage = genericStorage;
+ }
+
public static final String APP_DOC_NAME = "appDocName";
private static final Logger LOG = LoggerFactory.getLogger(GenericStorageResource.class);
@@ -55,7 +60,7 @@ public class GenericStorageResource extends AbstractAuthGuardedRestResource {
@PreAuthorize(AuthConstants.HAS_READ_GENERIC_STORAGE_PRIVILEGE)
public ResponseEntity<List<Map<String, Object>>> getAll(@PathVariable(APP_DOC_NAME) String appDocName) {
try {
- List<Map<String, Object>> assets = getGenericStorage().findAll(appDocName);
+ List<Map<String, Object>> assets = genericStorage.findAll(appDocName);
return ok(assets);
} catch (IOException e) {
LOG.error("Could not connect to storage", e);
@@ -71,7 +76,7 @@ public class GenericStorageResource extends AbstractAuthGuardedRestResource {
public ResponseEntity<Map<String, Object>> create(@PathVariable(APP_DOC_NAME) String appDocName,
@RequestBody String document) {
try {
- Map<String, Object> obj = getGenericStorage().create(document);
+ Map<String, Object> obj = genericStorage.create(document);
return ok(obj);
} catch (IOException e) {
LOG.error("Could not connect to storage", e);
@@ -87,7 +92,7 @@ public class GenericStorageResource extends AbstractAuthGuardedRestResource {
public ResponseEntity<List<Map<String, Object>>> find(@PathVariable(APP_DOC_NAME) String appDocName,
@RequestBody Map<String, Object> query) {
try {
- var docs = getGenericStorage().find(appDocName, query);
+ var docs = genericStorage.find(appDocName, query);
return ok(docs);
} catch (IOException e) {
LOG.error("Could not connect to storage", e);
@@ -100,7 +105,7 @@ public class GenericStorageResource extends AbstractAuthGuardedRestResource {
public ResponseEntity<Map<String, Object>> getCategory(@PathVariable(APP_DOC_NAME) String appDocName,
@PathVariable("id") String documentId) {
try {
- Map<String, Object> obj = getGenericStorage().findOne(documentId);
+ Map<String, Object> obj = genericStorage.findOne(documentId);
return ok(obj);
} catch (IOException e) {
LOG.error("Could not connect to storage", e);
@@ -117,7 +122,7 @@ public class GenericStorageResource extends AbstractAuthGuardedRestResource {
@PathVariable("id") String documentId,
@RequestBody String document) {
try {
- Map<String, Object> obj = getGenericStorage().update(documentId, document);
+ Map<String, Object> obj = genericStorage.update(documentId, document);
return ok(obj);
} catch (IOException e) {
LOG.error("Could not connect to storage", e);
@@ -131,7 +136,7 @@ public class GenericStorageResource extends AbstractAuthGuardedRestResource {
@PathVariable("id") String documentId,
@PathVariable("rev") String rev) {
try {
- getGenericStorage().delete(documentId, rev);
+ genericStorage.delete(documentId, rev);
return ok();
} catch (IOException e) {
LOG.error("Could not connect to storage", e);
@@ -139,8 +144,6 @@ public class GenericStorageResource extends AbstractAuthGuardedRestResource {
}
}
- private IGenericStorage getGenericStorage() {
- return StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage();
- }
+
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementAsset.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementAsset.java
index e51ccced37..3d970737b2 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementAsset.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementAsset.java
@@ -21,7 +21,6 @@ import org.apache.streampipes.commons.media.ImageMimeTypeDetector;
import org.apache.streampipes.manager.assets.AssetManager;
import org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.rest.core.base.impl.AbstractRestResource;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,9 +66,9 @@ public class PipelineElementAsset extends AbstractRestResource {
//it indicates usage by the asset-overview view where the data stream ID is supplied.
//In such cases, the appId of the adapter description needs retrieval for successful documentation loading.
if (appId.contains("sp:spdatastream")) {
- var dataStream = StorageDispatcher.INSTANCE
- .getNoSqlStore()
- .getDataStreamStorage()
+ var dataStream = resourceManager
+ .manageDataStreams()
+ .getDb()
.getDataStreamByAppId(appId);
var adapterDescription = resourceManager
.manageAdapters()
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementCategory.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementCategory.java
index a7d2db3195..753ad4eaad 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementCategory.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementCategory.java
@@ -23,7 +23,7 @@ import org.apache.streampipes.model.DataSinkType;
import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.client.Category;
import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
-import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -38,11 +38,15 @@ import java.util.stream.Collectors;
@RequestMapping("/api/v2/categories")
public class PipelineElementCategory extends AbstractAuthGuardedRestResource {
+ private final IPipelineElementDescriptionStorage descriptionStorage;
+
+ public PipelineElementCategory(IPipelineElementDescriptionStorage descriptionStorage) {
+ this.descriptionStorage = descriptionStorage;
+ }
+
@GetMapping(path = "/ep", produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<List<Category>> getEps() {
- return ok(makeCategories(StorageDispatcher.INSTANCE.getNoSqlStore()
- .getPipelineElementDescriptionStorage()
- .getAllDataStreams()));
+ return ok(makeCategories(descriptionStorage.getAllDataStreams()));
}
@GetMapping(path = "/epa", produces = MediaType.APPLICATION_JSON_VALUE)
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java
index f9bc9b91df..b93f016064 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java
@@ -106,6 +106,7 @@ public class PipelineResource extends AbstractAuthGuardedRestResource {
this.resourceManager = resourceManager;
this.descriptionStorage = descriptionStorage;
this.compactPipelineManagement = new CompactPipelineManagement(
+ resourceManager.getServiceDiscovery(),
descriptionStorage,
requestManager
);
@@ -246,6 +247,7 @@ public class PipelineResource extends AbstractAuthGuardedRestResource {
@PathVariable("recId") String baseRecElement) {
try {
return new ElementRecommender(
+ resourceManager.getServiceDiscovery(),
pipeline, baseRecElement, requestManager, dataProcessorResourceManager, dataSinkResourceManager, descriptionStorage
).findRecommendedElements();
} catch (JsonSyntaxException e) {
@@ -276,7 +278,7 @@ public class PipelineResource extends AbstractAuthGuardedRestResource {
@PreAuthorize("this.hasWriteAuthority()")
public ResponseEntity<?> validatePipeline(@RequestBody Pipeline pipeline) {
try {
- return ok(new PipelineVerificationHandlerV2(pipeline, requestManager).verifyPipeline());
+ return ok(new PipelineVerificationHandlerV2(resourceManager.getServiceDiscovery(), pipeline, requestManager).verifyPipeline());
} catch (JsonSyntaxException e) {
return badRequest(new Notification(NotificationType.UNKNOWN_ERROR, e.getMessage()));
} catch (Exception e) {
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineTemplate.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineTemplate.java
index 28408cf9a8..6f2d1ba3cb 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineTemplate.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineTemplate.java
@@ -27,7 +27,8 @@ import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResourc
import org.apache.streampipes.rest.shared.constants.SpMediaType;
import org.apache.streampipes.rest.shared.exception.BadRequestException;
import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -52,11 +53,15 @@ public class PipelineTemplate extends AbstractAuthGuardedRestResource {
private final ICompactPipelineTemplateStorage storage;
private final CompactPipelineTemplateManagement templateManagement;
- public PipelineTemplate(ExtensionServiceRequestManager extensionServiceRequestManager) {
- storage = StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineTemplateStorage();
+ public PipelineTemplate(ExtensionServiceRequestManager extensionServiceRequestManager,
+ ISpServiceDiscovery serviceDiscovery,
+ ICompactPipelineTemplateStorage storage,
+ IPipelineElementDescriptionStorage descriptionStorage) {
+ this.storage = storage;
templateManagement = new CompactPipelineTemplateManagement(
+ serviceDiscovery,
storage,
- StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineElementDescriptionStorage(),
+ descriptionStorage,
extensionServiceRequestManager
);
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ResetResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ResetResource.java
index 8c853e3f0b..3e02f7f39b 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ResetResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ResetResource.java
@@ -31,8 +31,8 @@ import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResourc
import org.apache.streampipes.rest.security.AuthConstants;
import org.apache.streampipes.storage.api.pipeline.IPipelineElementTemplateStorage;
import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import org.apache.streampipes.storage.api.system.ITransformationScriptTemplateStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import io.swagger.v3.oas.annotations.Operation;
import org.springframework.context.annotation.Conditional;
@@ -59,8 +59,9 @@ public class ResetResource extends AbstractAuthGuardedRestResource {
ExtensionServiceRequestManager requestManager,
SpResourceManager resourceManager,
ITransformationScriptTemplateStorage templateStorage,
- IPipelineElementTemplateStorage pipelineElementTemplateStorage) {
- IExtensionsServiceStorage extensionsServiceStorage = StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage();
+ IPipelineElementTemplateStorage pipelineElementTemplateStorage,
+ IExtensionsServiceStorage extensionsServiceStorage,
+ IGenericStorage genericStorage) {
this.resourceManager = resourceManager;
this.templateStorage = templateStorage;
var pipelineManager = new PipelineManager(
@@ -68,6 +69,7 @@ public class ResetResource extends AbstractAuthGuardedRestResource {
);
this.resetManagement = new ResetManagement(
workerRestClient,
+ genericStorage,
extensionsServiceStorage,
requestManager,
pipelineManager,
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ServiceTagResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ServiceTagResource.java
index 536933f950..6434cfafcb 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ServiceTagResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ServiceTagResource.java
@@ -20,7 +20,6 @@ package org.apache.streampipes.rest.impl;
import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag;
import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
-import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.springframework.http.MediaType;
@@ -34,7 +33,11 @@ import java.util.Set;
@RequestMapping("/api/v2/service-tags")
public class ServiceTagResource extends AbstractAuthGuardedRestResource {
- private final ISpServiceDiscovery serviceDiscovery = SpServiceDiscovery.getServiceDiscovery();
+ private final ISpServiceDiscovery serviceDiscovery;
+
+ public ServiceTagResource(ISpServiceDiscovery serviceDiscovery) {
+ this.serviceDiscovery = serviceDiscovery;
+ }
@GetMapping(produces = MediaType.APPLICATION_JSON_VALUE)
public Set<SpServiceTag> getCustomServiceTags() {
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsInstallationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsInstallationResource.java
index bcc3f5a5d2..2bbf1bf201 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsInstallationResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsInstallationResource.java
@@ -136,7 +136,7 @@ public class ExtensionsInstallationResource extends AbstractAuthGuardedRestResou
}
private SpServiceRegistration findSupportedService(ExtensionItemInstallationRequest installationReq) throws NoServiceEndpointsAvailableException {
- return new ExtensionsServiceEndpointGenerator().selectService(
+ return new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery()).selectService(
installationReq.appId(),
SpServiceUrlProvider.valueOf(installationReq.serviceTagPrefix().name()),
Set.of()
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java
index d516458786..107bf41a8d 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExtensionsServiceEndpointResource.java
@@ -31,6 +31,7 @@ import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResourc
import org.apache.streampipes.rest.security.AuthConstants;
import org.apache.streampipes.rest.shared.exception.SpMessageException;
import org.apache.streampipes.storage.api.system.ISpCoreConfigurationStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
import org.springframework.http.HttpStatus;
@@ -52,13 +53,16 @@ import java.util.Set;
@PreAuthorize(AuthConstants.IS_ADMIN_ROLE)
public class ExtensionsServiceEndpointResource extends AbstractAuthGuardedRestResource {
+ private final ISpServiceDiscovery serviceDiscovery;
private final ExtensionServiceRequestManager extensionServiceRequestManager;
private final AssetManager assetManager;
private final AvailableExtensionsProvider availableExtensionsProvider;
- public ExtensionsServiceEndpointResource(ExtensionServiceRequestManager extensionServiceRequestManager,
+ public ExtensionsServiceEndpointResource(ISpServiceDiscovery serviceDiscovery,
+ ExtensionServiceRequestManager extensionServiceRequestManager,
ISpCoreConfigurationStorage coreConfigurationStorage,
AvailableExtensionsProvider availableExtensionsProvider) {
+ this.serviceDiscovery = serviceDiscovery;
this.extensionServiceRequestManager = extensionServiceRequestManager;
this.availableExtensionsProvider = availableExtensionsProvider;
this.assetManager = new AssetManager(coreConfigurationStorage);
@@ -93,7 +97,7 @@ public class ExtensionsServiceEndpointResource extends AbstractAuthGuardedRestRe
try {
var urlProvider = SpServiceUrlProvider.valueOf(extensionItemDescription.getServiceTagPrefix().name());
- var service = new ExtensionsServiceEndpointGenerator().selectService(
+ var service = new ExtensionsServiceEndpointGenerator(serviceDiscovery).selectService(
extensionItemDescription.getAppId(),
urlProvider,
Set.of()
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java
index eadb100e3d..5de6bd0f41 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java
@@ -52,15 +52,16 @@ import java.util.List;
@PreAuthorize(AuthConstants.IS_ADMIN_ROLE)
public class ServiceRegistrationResource extends AbstractAuthGuardedRestResource {
- private final IExtensionsServiceStorage extensionsServiceStorage =
- getNoSqlStorage().getExtensionsServiceStorage();
+ private final IExtensionsServiceStorage extensionsServiceStorage;
private final ExtensionServiceRequestManager extensionServiceRequestManager;
private final SpResourceManager resourceManager;
public ServiceRegistrationResource(ExtensionServiceRequestManager extensionServiceRequestManager,
- SpResourceManager resourceManager) {
+ SpResourceManager resourceManager,
+ IExtensionsServiceStorage extensionsServiceStorage) {
this.extensionServiceRequestManager = extensionServiceRequestManager;
this.resourceManager = resourceManager;
+ this.extensionsServiceStorage = extensionsServiceStorage;
}
@GetMapping(produces = MediaType.APPLICATION_JSON_VALUE)
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/AdapterResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/AdapterResource.java
index b92b67f1d0..cbacb0482e 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/AdapterResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/AdapterResource.java
@@ -46,7 +46,7 @@ import org.apache.streampipes.rest.security.AuthConstants;
import org.apache.streampipes.rest.shared.constants.SpMediaType;
import org.apache.streampipes.storage.api.connect.IAdapterStorage;
import org.apache.streampipes.storage.api.pipeline.IPipelineStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.http.HttpStatus;
@@ -89,8 +89,9 @@ public class AdapterResource extends AbstractAdapterResource<AdapterMasterManage
public AdapterResource(WorkerRestClient workerRestClient,
ExtensionServiceRequestManager requestManager,
SpResourceManager resourceManager,
- @Qualifier("adapterDescriptionStorage") IAdapterStorage adapterDescriptionStorage) {
- this(workerRestClient, requestManager, null, resourceManager, adapterDescriptionStorage);
+ @Qualifier("adapterDescriptionStorage") IAdapterStorage adapterDescriptionStorage,
+ IExtensionsServiceStorage extensionsServiceStorage) {
+ this(workerRestClient, requestManager, null, resourceManager, adapterDescriptionStorage, extensionsServiceStorage);
}
@Autowired
@@ -98,12 +99,13 @@ public class AdapterResource extends AbstractAdapterResource<AdapterMasterManage
ExtensionServiceRequestManager requestManager,
ApplicationEventPublisher eventPublisher,
SpResourceManager resourceManager,
- @Qualifier("adapterDescriptionStorage") IAdapterStorage adapterDescriptionStorage) {
+ @Qualifier("adapterDescriptionStorage") IAdapterStorage adapterDescriptionStorage,
+ IExtensionsServiceStorage extensionsServiceStorage) {
super(() -> new AdapterMasterManagement(
resourceManager,
AdapterMetricsManager.INSTANCE.getAdapterMetrics(),
workerRestClient,
- StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage(),
+ extensionsServiceStorage,
requestManager,
new AdapterAuditRecorder(resourceManager.getAuditService())));
this.requestManager = requestManager;
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResource.java
index df0748cf2b..6d1b2bf3b7 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResource.java
@@ -41,7 +41,10 @@ import org.apache.streampipes.rest.shared.constants.SpMediaType;
import org.apache.streampipes.rest.shared.exception.BadRequestException;
import org.apache.streampipes.rest.shared.exception.SpMessageException;
import org.apache.streampipes.storage.api.connect.IAdapterStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
+import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
+import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,26 +70,36 @@ public class CompactAdapterResource extends AbstractAdapterResource<AdapterMaste
private final AdapterUpdateManagement adapterUpdateManagement;
private final ExtensionServiceRequestManager requestManager;
private final PipelineManager pipelineManager;
+ private final ISpServiceDiscovery serviceDiscovery;
+ private final ICompactPipelineTemplateStorage pipelineTemplateStorage;
+ private final IPipelineElementDescriptionStorage descriptionStorage;
public CompactAdapterResource(WorkerRestClient workerRestClient,
ExtensionServiceRequestManager requestManager,
ApplicationEventPublisher eventPublisher,
SpResourceManager resourceManager,
- @Qualifier("adapterDescriptionStorage") IAdapterStorage adapterDescriptionStorage) {
+ @Qualifier("adapterDescriptionStorage") IAdapterStorage adapterDescriptionStorage,
+ IExtensionsServiceStorage extensionsServiceStorage,
+ ISpServiceDiscovery serviceDiscovery,
+ ICompactPipelineTemplateStorage pipelineTemplateStorage,
+ IPipelineElementDescriptionStorage descriptionStorage) {
super(() -> new AdapterMasterManagement(
resourceManager,
AdapterMetricsManager.INSTANCE.getAdapterMetrics(),
workerRestClient,
- StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage(),
+ extensionsServiceStorage,
requestManager,
new AdapterAuditRecorder(resourceManager.getAuditService())
));
var guessManagement = new GuessManagement(
- new ExtensionsServiceEndpointGenerator(),
+ new ExtensionsServiceEndpointGenerator(serviceDiscovery),
requestManager,
resourceManager
);
this.requestManager = requestManager;
+ this.serviceDiscovery = serviceDiscovery;
+ this.pipelineTemplateStorage = pipelineTemplateStorage;
+ this.descriptionStorage = descriptionStorage;
this.compactAdapterManagement = new CompactAdapterManagement(
new AdapterGenerationSteps(guessManagement).getGenerators(), adapterDescriptionStorage
);
@@ -143,9 +156,10 @@ public class CompactAdapterResource extends AbstractAdapterResource<AdapterMaste
.persist()) {
var storedAdapter = managementService.getAdapter(adapterId);
new PersistPipelineHandler(
- getNoSqlStorage().getPipelineTemplateStorage(),
+ pipelineTemplateStorage,
new CompactPipelineManagement(
- getNoSqlStorage().getPipelineElementDescriptionStorage(),
+ serviceDiscovery,
+ descriptionStorage,
requestManager
),
pipelineManager,
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java
index e30e7e4ddf..4356a7acc2 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java
@@ -59,7 +59,7 @@ public class DescriptionResource extends AbstractAdapterResource<DescriptionMana
SpResourceManager resourceManager,
@Qualifier("adapterDescriptionStorage") IAdapterStorage adapterDescriptionStorage) {
super(() -> new DescriptionManagement(workerRestClient, resourceManager.manageAdapters(), adapterDescriptionStorage));
- endpointGenerator = new ExtensionsServiceEndpointGenerator();
+ endpointGenerator = new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery());
}
@GetMapping(path = "/adapters", produces = MediaType.APPLICATION_JSON_VALUE)
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/GuessResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/GuessResource.java
index 2da499d326..d1ab90c183 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/GuessResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/GuessResource.java
@@ -50,7 +50,7 @@ public class GuessResource extends AbstractAdapterResource<GuessManagement> {
public GuessResource(ExtensionServiceRequestManager extensionServiceRequestManager,
SpResourceManager resourceManager) {
super(() -> new GuessManagement(
- new ExtensionsServiceEndpointGenerator(),
+ new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery()),
extensionServiceRequestManager,
resourceManager)
);
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/RuntimeResolvableResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/RuntimeResolvableResource.java
index f4a560467c..af581f5064 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/RuntimeResolvableResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/RuntimeResolvableResource.java
@@ -55,7 +55,7 @@ public class RuntimeResolvableResource extends AbstractAdapterResource<Void> {
public RuntimeResolvableResource(WorkerRestClient workerRestClient,
SpResourceManager resourceManager) {
super();
- this.endpointGenerator = new ExtensionsServiceEndpointGenerator();
+ this.endpointGenerator = new ExtensionsServiceEndpointGenerator(resourceManager.getServiceDiscovery());
this.workerRestClient = workerRestClient;
this.resourceManager = resourceManager;
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/TransformationScriptLanguageResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/TransformationScriptLanguageResource.java
index 5067621f70..f1e8b77f84 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/TransformationScriptLanguageResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/TransformationScriptLanguageResource.java
@@ -25,7 +25,6 @@ import org.apache.streampipes.model.connect.ScriptMetadata;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration;
import org.apache.streampipes.rest.security.AuthConstants;
-import org.apache.streampipes.svcdiscovery.SpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes;
import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider;
@@ -46,11 +45,7 @@ public class TransformationScriptLanguageResource {
private final ISpServiceDiscovery serviceDiscovery;
- public TransformationScriptLanguageResource() {
- this(SpServiceDiscovery.getServiceDiscovery());
- }
-
- TransformationScriptLanguageResource(ISpServiceDiscovery serviceDiscovery) {
+ public TransformationScriptLanguageResource(ISpServiceDiscovery serviceDiscovery) {
this.serviceDiscovery = serviceDiscovery;
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/dataset/AbstractDatasetResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/dataset/AbstractDatasetResource.java
index d9c8af6c1b..1fb92d1205 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/dataset/AbstractDatasetResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/dataset/AbstractDatasetResource.java
@@ -45,7 +45,8 @@ public class AbstractDatasetResource extends AbstractAuthGuardedRestResource {
this.datasetMetadataManagement = new DataExplorerDispatcher().getSchemaManagement(
chartSchemaUpdateCoordinator,
resourceManager.managePermissions().getDb(),
- resourceManager.manageDataLakeMeasures().getDb());
+ resourceManager.manageDataLakeMeasures().getDb(),
+ resourceManager.getResourceDeletionManager());
}
/**
diff --git a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResourceTest.java b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResourceTest.java
index cdf8e851dc..640f957c40 100644
--- a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResourceTest.java
+++ b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/connect/CompactAdapterResourceTest.java
@@ -25,8 +25,8 @@ import org.apache.streampipes.model.client.user.DefaultRole;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
import org.apache.streampipes.model.connect.adapter.compact.CompactAdapter;
import org.apache.streampipes.model.connect.adapter.compact.CreateOptions;
-import org.apache.streampipes.storage.api.core.INoSqlStorage;
import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
+import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -56,14 +56,14 @@ class CompactAdapterResourceTest {
private AdapterMasterManagement adapterManagement;
private CompactAdapterManagement compactAdapterManagement;
- private INoSqlStorage noSqlStorage;
+ private IPipelineElementDescriptionStorage descriptionStorage;
private CompactAdapterResource resource;
@BeforeEach
void setUp() throws Exception {
adapterManagement = mock(AdapterMasterManagement.class);
compactAdapterManagement = mock(CompactAdapterManagement.class);
- noSqlStorage = mock(INoSqlStorage.class);
+ descriptionStorage = mock(IPipelineElementDescriptionStorage.class);
var adapterDescription = new AdapterDescription();
adapterDescription.setElementId(ADAPTER_ID);
@@ -71,18 +71,18 @@ class CompactAdapterResourceTest {
when(compactAdapterManagement.convertToAdapterDescription(any(), eq(USER_SID)))
.thenReturn(adapterDescription);
when(adapterManagement.getAdapter(ADAPTER_ID)).thenReturn(adapterDescription);
- // no persist pipeline template stored, so the persist step ends with an exception
- when(noSqlStorage.getPipelineTemplateStorage()).thenReturn(mock(ICompactPipelineTemplateStorage.class));
// the protected base class methods cannot be stubbed from this package
resource = mock(CompactAdapterResource.class, withSettings().defaultAnswer(invocation ->
switch (invocation.getMethod().getName()) {
case "getAuthenticatedUserSid" -> USER_SID;
- case "getNoSqlStorage" -> noSqlStorage;
default -> invocation.callRealMethod();
}));
resource.managementService = adapterManagement;
setField(resource, "compactAdapterManagement", compactAdapterManagement);
+ setField(resource, "descriptionStorage", descriptionStorage);
+ // No persist pipeline template is stored, so the persist step ends with an exception.
+ setField(resource, "pipelineTemplateStorage", mock(ICompactPipelineTemplateStorage.class));
}
@AfterEach
@@ -98,7 +98,7 @@ class CompactAdapterResourceTest {
assertEquals(HttpStatus.FORBIDDEN, response.getStatusCode());
assertNull(response.getBody());
- verifyNoInteractions(compactAdapterManagement, adapterManagement, noSqlStorage);
+ verifyNoInteractions(compactAdapterManagement, adapterManagement, descriptionStorage);
}
@Test
@@ -110,7 +110,7 @@ class CompactAdapterResourceTest {
assertEquals(HttpStatus.OK, response.getStatusCode());
verify(adapterManagement).addAdapter(any(), eq(ADAPTER_ID), eq(USER_SID));
verify(adapterManagement).startAdapter(ADAPTER_ID, USER_SID);
- verifyNoInteractions(noSqlStorage);
+ verifyNoInteractions(descriptionStorage);
}
@Test
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/DatasetQueryConfiguration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/DatasetQueryConfiguration.java
index d60a71d863..48b761657f 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/DatasetQueryConfiguration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/DatasetQueryConfiguration.java
@@ -34,7 +34,8 @@ public class DatasetQueryConfiguration {
public DatasetServices datasetServices(IChartStorage charts, SpResourceManager resources) {
var dispatcher = new DataExplorerDispatcher();
var catalog = dispatcher.getSchemaManagement(new ChartSchemaUpdateCoordinator(charts),
- resources.managePermissions().getDb(), resources.manageDataLakeMeasures().getDb());
+ resources.managePermissions().getDb(), resources.manageDataLakeMeasures().getDb(),
+ resources.getResourceDeletionManager());
return dispatcher.getDatasetServices(catalog);
}
}
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PipelineManagementConfiguration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PipelineManagementConfiguration.java
index 84128bc06c..77e23708b8 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PipelineManagementConfiguration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/PipelineManagementConfiguration.java
@@ -20,11 +20,14 @@ package org.apache.streampipes.service.core;
import org.apache.streampipes.manager.extensions.AvailableExtensionsProvider;
import org.apache.streampipes.manager.setup.InstallationConfiguration;
+import org.apache.streampipes.resource.management.ResourceDeletionManager;
import org.apache.streampipes.storage.api.connect.IAdapterStorage;
import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
import org.apache.streampipes.storage.api.system.IGenericStorage;
+import org.apache.streampipes.svcdiscovery.SpServiceDiscoveryCore;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
@@ -33,6 +36,16 @@ import org.springframework.context.annotation.Configuration;
@Configuration
public class PipelineManagementConfiguration {
+ @Bean
+ public ResourceDeletionManager resourceDeletionManager(IGenericStorage genericStorage) {
+ return new ResourceDeletionManager(genericStorage);
+ }
+
+ @Bean
+ public ISpServiceDiscovery serviceDiscovery(IExtensionsServiceStorage extensionsServiceStorage) {
+ return new SpServiceDiscoveryCore(extensionsServiceStorage);
+ }
+
@Bean
public AvailableExtensionsProvider availableExtensionsProvider(
IExtensionsServiceStorage extensionsServiceStorage,
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java
index 6eb563d7b8..7ce3b9658c 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java
@@ -63,6 +63,7 @@ import org.apache.streampipes.service.core.migrations.MigrationsHandler;
import org.apache.streampipes.service.core.storage.StorageApiConfiguration;
import org.apache.streampipes.storage.api.connect.IAdapterStorage;
import org.apache.streampipes.storage.api.function.IFunctionStateStorage;
+import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
import org.apache.streampipes.storage.api.pipeline.IPipelineStorage;
import org.apache.streampipes.storage.api.system.ICertificateStorage;
import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
@@ -113,6 +114,9 @@ public class StreamPipesCoreApplication extends StreamPipesServiceBase {
@Autowired
protected ICertificateStorage certificateStorage;
+ @Autowired
+ private ICompactPipelineTemplateStorage pipelineTemplateStorage;
+
@Autowired
protected ExtensionServiceRequestManager extensionServiceRequestManager;
@@ -274,7 +278,7 @@ public class StreamPipesCoreApplication extends StreamPipesServiceBase {
}
protected List<Migration> getMigrations() {
- return new AvailableMigrations(resourceManager, certificateStorage).getAvailableMigrations();
+ return new AvailableMigrations(resourceManager, certificateStorage, pipelineTemplateStorage).getAvailableMigrations();
}
protected List<HealthCheck> getRegisteredExtensionHealthChecks() {
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/extensions/ExtensionServiceRequestConfiguration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/extensions/ExtensionServiceRequestConfiguration.java
index e4cc09ffa4..582e797868 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/extensions/ExtensionServiceRequestConfiguration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/extensions/ExtensionServiceRequestConfiguration.java
@@ -30,10 +30,14 @@ import org.apache.streampipes.storage.api.explorer.IDashboardStorage;
import org.apache.streampipes.storage.api.explorer.IDatasetMetadataStorage;
import org.apache.streampipes.storage.api.pipeline.IDataProcessorStorage;
import org.apache.streampipes.storage.api.pipeline.IDataSinkStorage;
+import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
+import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
import org.apache.streampipes.storage.api.pipeline.IPipelineStorage;
import org.apache.streampipes.storage.api.system.IAssetStorage;
import org.apache.streampipes.storage.api.system.ICertificateStorage;
+import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
import org.apache.streampipes.storage.api.system.IFileMetadataStorage;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import org.apache.streampipes.storage.api.system.ISpCoreConfigurationStorage;
import org.apache.streampipes.storage.api.user.IPasswordRecoveryTokenStorage;
import org.apache.streampipes.storage.api.user.IPermissionStorage;
@@ -43,6 +47,7 @@ import org.apache.streampipes.storage.api.user.IRoleStorage;
import org.apache.streampipes.storage.api.user.IUserActivationTokenStorage;
import org.apache.streampipes.storage.api.user.IUserGroupStorage;
import org.apache.streampipes.storage.api.user.IUserStorage;
+import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
import org.apache.streampipes.user.management.service.RefreshTokenService;
import org.slf4j.Logger;
@@ -87,7 +92,8 @@ public class ExtensionServiceRequestConfiguration {
@Bean
@Primary
public ExtensionServiceRequestManager extensionServiceRequestManager(
- NatsExtensionServiceRequestManager natsExtensionServiceRequestManager
+ NatsExtensionServiceRequestManager natsExtensionServiceRequestManager,
+ IExtensionsServiceStorage extensionsServiceStorage
) {
var env = Environments.getEnvironment();
@@ -100,7 +106,8 @@ public class ExtensionServiceRequestConfiguration {
return new TransportAwareExtensionServiceRequestManager(
new HttpExtensionServiceRequestManager(),
natsExtensionServiceRequestManager,
- transportMode
+ transportMode,
+ extensionsServiceStorage
);
}
@@ -121,9 +128,14 @@ public class ExtensionServiceRequestConfiguration {
IUserStorage userStorage,
IDataProcessorStorage dataProcessorStorage,
IDataSinkStorage dataSinkStorage,
+ IDataStreamStorage dataStreamStorage,
+ IPipelineElementDescriptionStorage descriptionStorage,
+ IGenericStorage genericStorage,
IUserActivationTokenStorage userActivationTokenStorage,
IPasswordRecoveryTokenStorage passwordRecoveryTokenStorage,
ICertificateStorage certificateStorage,
+ IExtensionsServiceStorage extensionsServiceStorage,
+ ISpServiceDiscovery serviceDiscovery,
AuditService auditService) {
return new SpResourceManager(
permissionStorage,
@@ -142,9 +154,14 @@ public class ExtensionServiceRequestConfiguration {
userStorage,
dataProcessorStorage,
dataSinkStorage,
+ dataStreamStorage,
+ descriptionStorage,
+ genericStorage,
userActivationTokenStorage,
passwordRecoveryTokenStorage,
certificateStorage,
+ extensionsServiceStorage,
+ serviceDiscovery,
auditService
);
}
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/extensions/TransportAwareExtensionServiceRequestManager.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/extensions/TransportAwareExtensionServiceRequestManager.java
index 12c69cd499..fd78d4315a 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/extensions/TransportAwareExtensionServiceRequestManager.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/extensions/TransportAwareExtensionServiceRequestManager.java
@@ -25,7 +25,7 @@ import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestTarg
import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix;
import org.apache.streampipes.model.extensions.transport.ExtensionServiceBrokerOperations;
import org.apache.streampipes.model.extensions.transport.ExtensionServiceBrokerTopics;
-import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,15 +40,18 @@ public class TransportAwareExtensionServiceRequestManager implements ExtensionSe
private final ExtensionServiceRequestManager httpRequestManager;
private final NatsExtensionServiceRequestManager natsRequestManager;
private final CoreExtensionTransportMode transportMode;
+ private final IExtensionsServiceStorage extensionsServiceStorage;
public TransportAwareExtensionServiceRequestManager(
ExtensionServiceRequestManager httpRequestManager,
NatsExtensionServiceRequestManager natsRequestManager,
- CoreExtensionTransportMode transportMode
+ CoreExtensionTransportMode transportMode,
+ IExtensionsServiceStorage extensionsServiceStorage
) {
this.httpRequestManager = httpRequestManager;
this.natsRequestManager = natsRequestManager;
this.transportMode = transportMode;
+ this.extensionsServiceStorage = extensionsServiceStorage;
}
@Override
@@ -87,9 +90,7 @@ public class TransportAwareExtensionServiceRequestManager implements ExtensionSe
}
private boolean serviceSupportsNats(ExtensionServiceRequestTarget target) {
- var service = StorageDispatcher.INSTANCE
- .getNoSqlStore()
- .getExtensionsServiceStorage()
+ var service = extensionsServiceStorage
.getElementById(target.serviceId());
if (service == null || service.getTags() == null) {
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java
index cbeb812836..ee32070934 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java
@@ -53,9 +53,11 @@ import org.apache.streampipes.storage.api.explorer.IDashboardStorage;
import org.apache.streampipes.storage.api.explorer.IDatasetMetadataStorage;
import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
import org.apache.streampipes.storage.api.pipeline.IDataSinkStorage;
+import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
import org.apache.streampipes.storage.api.pipeline.IPipelineStorage;
import org.apache.streampipes.storage.api.system.IAssetStorage;
import org.apache.streampipes.storage.api.system.ICertificateStorage;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
import org.apache.streampipes.storage.api.system.ISpCoreConfigurationStorage;
import org.apache.streampipes.storage.api.user.IPermissionStorage;
import org.apache.streampipes.storage.api.user.IPrivilegeStorage;
@@ -83,16 +85,21 @@ public class AvailableMigrations {
private final IPrivilegeStorage privilegeStorage;
private final IUserStorage userStorage;
private final ICertificateStorage certificateStorage;
+ private final IDataStreamStorage dataStreamStorage;
+ private final IGenericStorage genericStorage;
- public AvailableMigrations(SpResourceManager resourceManager, ICertificateStorage certificateStorage) {
+ public AvailableMigrations(SpResourceManager resourceManager,
+ ICertificateStorage certificateStorage,
+ ICompactPipelineTemplateStorage pipelineTemplateStorage) {
this.certificateStorage = certificateStorage;
+ this.genericStorage = resourceManager.getGenericStorage();
this.chartStorage = resourceManager.manageCharts().getDb();
this.permissionStorage = resourceManager.managePermissions().getDb();
this.adapterStorage = resourceManager.manageAdapters().getDb();
this.dashboardStorage = resourceManager.manageDashboards().getDb();
this.assetStorage = resourceManager.manageAssets().getDb();
this.dataSinkStorage = resourceManager.manageDataSinks().getDb();
- this.pipelineTemplateStorage = resourceManager.getPipelineTemplateStorage();
+ this.pipelineTemplateStorage = pipelineTemplateStorage;
this.pipelineStorage = resourceManager.managePipelines().getDb();
this.datasetStorage = resourceManager.manageDataLakeMeasures().getDb();
this.coreConfigStorage = resourceManager.getCoreConfigurationStorage();
@@ -100,17 +107,18 @@ public class AvailableMigrations {
this.userGroupStorage = resourceManager.getUserGroupStorage();
this.privilegeStorage = resourceManager.getPrivilegeStorage();
this.userStorage = resourceManager.manageUsers().getDb();
+ this.dataStreamStorage = resourceManager.manageDataStreams().getDb();
}
public List<Migration> getAvailableMigrations() {
return Arrays.asList(
- new ModifyAssetLinksMigration(),
- new ModifyAssetLinkTypesMigration(),
+ new ModifyAssetLinksMigration(genericStorage),
+ new ModifyAssetLinkTypesMigration(genericStorage),
new AddDatasetMetadataViewMigration(),
new AddDefaultExportProviderMigration(coreConfigStorage),
- new FixImportedPermissionsMigration(chartStorage, dashboardStorage, permissionStorage),
+ new FixImportedPermissionsMigration(chartStorage, dataStreamStorage, dashboardStorage, permissionStorage),
new AddAssetManagementViewMigration(),
- new MoveAssetContentMigration(),
+ new MoveAssetContentMigration(genericStorage),
new CreateAssetPermissionMigration(permissionStorage, assetStorage),
new MigrateDataLakeDatabaseToDatasetMigration(),
new CreateDatasetPermissionMigration(permissionStorage, pipelineStorage, datasetStorage),
@@ -121,7 +129,7 @@ public class AvailableMigrations {
new ComputeCertificateThumbprintMigration(certificateStorage),
new MigrateAdaptersToUseScript(adapterStorage),
new MigratePlc4xS7AdaptersToGenericAdapter(adapterStorage),
- new ModifyAssetLinkIconMigration(),
+ new ModifyAssetLinkIconMigration(genericStorage),
new RemoveDuplicatedAssetPermissions(permissionStorage, assetStorage),
new AddFunctionStateViewMigration(),
new AddRefreshTokenViewsMigration(),
@@ -130,7 +138,7 @@ public class AvailableMigrations {
new MigrateDatasetMetadataMigration(datasetStorage, permissionStorage),
new MigrateDataLakePersistPipelineTemplateMigration(pipelineTemplateStorage),
new ReplaceDefaultServiceSecretMigration(userStorage),
- new RenameAssetLinkTypesMigration()
+ new RenameAssetLinkTypesMigration(genericStorage)
);
}
}
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/FixImportedPermissionsMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/FixImportedPermissionsMigration.java
index f20e203e99..1839621690 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/FixImportedPermissionsMigration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/FixImportedPermissionsMigration.java
@@ -22,8 +22,8 @@ import org.apache.streampipes.model.shared.api.Storable;
import org.apache.streampipes.service.core.migrations.Migration;
import org.apache.streampipes.storage.api.explorer.IChartStorage;
import org.apache.streampipes.storage.api.explorer.IDashboardStorage;
+import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
import org.apache.streampipes.storage.api.user.IPermissionStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,11 +40,14 @@ public class FixImportedPermissionsMigration implements Migration {
private final IChartStorage chartStorage;
private final IPermissionStorage permissionStorage;
private final IDashboardStorage dashboardStorage;
+ private final IDataStreamStorage dataStreamStorage;
public FixImportedPermissionsMigration(IChartStorage chartStorage,
+ IDataStreamStorage dataStreamStorage,
IDashboardStorage dashboardStorage,
IPermissionStorage permissionStorage) {
this.chartStorage = chartStorage;
+ this.dataStreamStorage = dataStreamStorage;
this.dashboardStorage = dashboardStorage;
this.permissionStorage = permissionStorage;
}
@@ -79,9 +82,6 @@ public class FixImportedPermissionsMigration implements Migration {
private void migrateDataStreamPermissions() {
LOG.debug("Start migrate permissions for data streams");
- var dataStreamStorage =
- StorageDispatcher.INSTANCE.getNoSqlStore()
- .getDataStreamStorage();
var dataStreams = dataStreamStorage.findAll();
migrateResourcePermissions(dataStreams);
LOG.debug("Finished migrate permissions for data streams");
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/ModifyAssetLinkTypesMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/ModifyAssetLinkTypesMigration.java
index 983b3f76d3..1313ac172d 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/ModifyAssetLinkTypesMigration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/ModifyAssetLinkTypesMigration.java
@@ -22,7 +22,6 @@ import org.apache.streampipes.commons.constants.GenericDocTypes;
import org.apache.streampipes.serializers.json.JacksonSerializer;
import org.apache.streampipes.service.core.migrations.Migration;
import org.apache.streampipes.storage.api.system.IGenericStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -32,7 +31,11 @@ import java.util.Map;
public class ModifyAssetLinkTypesMigration implements Migration {
- private final IGenericStorage storage = StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage();
+ private final IGenericStorage storage;
+
+ public ModifyAssetLinkTypesMigration(IGenericStorage storage) {
+ this.storage = storage;
+ }
@Override
public boolean shouldExecute() {
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/ModifyAssetLinksMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/ModifyAssetLinksMigration.java
index 1e01e4612b..965e8225ef 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/ModifyAssetLinksMigration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/ModifyAssetLinksMigration.java
@@ -22,7 +22,6 @@ import org.apache.streampipes.commons.constants.GenericDocTypes;
import org.apache.streampipes.serializers.json.JacksonSerializer;
import org.apache.streampipes.service.core.migrations.Migration;
import org.apache.streampipes.storage.api.system.IGenericStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -35,7 +34,11 @@ import java.util.stream.Collectors;
public class ModifyAssetLinksMigration implements Migration {
- private final IGenericStorage storage = StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage();
+ private final IGenericStorage storage;
+
+ public ModifyAssetLinksMigration(IGenericStorage storage) {
+ this.storage = storage;
+ }
@Override
public boolean shouldExecute() {
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/ModifyAssetLinkIconMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/ModifyAssetLinkIconMigration.java
index 287738875a..ddcf6f63f3 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/ModifyAssetLinkIconMigration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/ModifyAssetLinkIconMigration.java
@@ -21,7 +21,6 @@ package org.apache.streampipes.service.core.migrations.v099;
import org.apache.streampipes.commons.constants.GenericDocTypes;
import org.apache.streampipes.service.core.migrations.Migration;
import org.apache.streampipes.storage.api.system.IGenericStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -53,8 +52,8 @@ public class ModifyAssetLinkIconMigration implements Migration {
private final IGenericStorage genericStorage;
- public ModifyAssetLinkIconMigration() {
- this.genericStorage = StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage();
+ public ModifyAssetLinkIconMigration(IGenericStorage genericStorage) {
+ this.genericStorage = genericStorage;
}
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/MoveAssetContentMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/MoveAssetContentMigration.java
index 0621fcf3f5..c1273f882c 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/MoveAssetContentMigration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/MoveAssetContentMigration.java
@@ -22,7 +22,6 @@ import org.apache.streampipes.model.assets.SpAssetModel;
import org.apache.streampipes.serializers.json.JacksonSerializer;
import org.apache.streampipes.service.core.migrations.Migration;
import org.apache.streampipes.storage.api.system.IGenericStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -39,7 +38,7 @@ public class MoveAssetContentMigration implements Migration {
private static final Logger LOG = LoggerFactory.getLogger(MoveAssetContentMigration.class);
- private IGenericStorage genericStorage;
+ private final IGenericStorage genericStorage;
private final ObjectMapper mapper = JacksonSerializer.getObjectMapper(Map.of(
DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true
));
@@ -60,8 +59,8 @@ public class MoveAssetContentMigration implements Migration {
"appDocType"
);
- public MoveAssetContentMigration() {
- this.genericStorage = StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage();
+ public MoveAssetContentMigration(IGenericStorage genericStorage) {
+ this.genericStorage = genericStorage;
}
@Override
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/RenameAssetLinkTypesMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/RenameAssetLinkTypesMigration.java
index bad0a53a71..c5f89dc3ce 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/RenameAssetLinkTypesMigration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v099/RenameAssetLinkTypesMigration.java
@@ -21,7 +21,6 @@ package org.apache.streampipes.service.core.migrations.v099;
import org.apache.streampipes.commons.constants.GenericDocTypes;
import org.apache.streampipes.service.core.migrations.Migration;
import org.apache.streampipes.storage.api.system.IGenericStorage;
-import org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -47,8 +46,8 @@ public class RenameAssetLinkTypesMigration implements Migration {
private final IGenericStorage genericStorage;
private final ObjectMapper mapper;
- public RenameAssetLinkTypesMigration() {
- this(StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage(), new ObjectMapper());
+ public RenameAssetLinkTypesMigration(IGenericStorage genericStorage) {
+ this(genericStorage, new ObjectMapper());
}
RenameAssetLinkTypesMigration(IGenericStorage genericStorage, ObjectMapper mapper) {
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/scheduler/DatasetScheduler.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/scheduler/DatasetScheduler.java
index e606621d1a..2e4bc86f22 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/scheduler/DatasetScheduler.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/scheduler/DatasetScheduler.java
@@ -51,7 +51,8 @@ public class DatasetScheduler implements SchedulingConfigurer {
.getSchemaManagement(
chartSchemaUpdateCoordinator,
resourceManager.managePermissions().getDb(),
- resourceManager.manageDataLakeMeasures().getDb());
+ resourceManager.manageDataLakeMeasures().getDb(),
+ resourceManager.getResourceDeletionManager());
this.datasetExportManager = new DatasetExportManager(
datasetMetadataManagement,
services,
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/storage/StorageApiConfiguration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/storage/StorageApiConfiguration.java
index 8a40d0b735..f76d5a909c 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/storage/StorageApiConfiguration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/storage/StorageApiConfiguration.java
@@ -82,6 +82,7 @@ import org.apache.streampipes.storage.couchdb.impl.user.UserGroupStorageImpl;
import org.apache.streampipes.storage.couchdb.impl.user.UserStorage;
import org.apache.streampipes.storage.couchdb.utils.Utils;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
@@ -244,8 +245,13 @@ public class StorageApiConfiguration {
}
@Bean
- public IPipelineElementDescriptionStorage pipelineElementDescriptionStorage() {
- return new PipelineElementDescriptionStorageImpl();
+ public IPipelineElementDescriptionStorage pipelineElementDescriptionStorage(
+ IDataProcessorStorage dataProcessorStorage,
+ IDataStreamStorage dataStreamStorage,
+ IDataSinkStorage dataSinkStorage,
+ @Qualifier("adapterDescriptionStorage") IAdapterStorage adapterDescriptionStorage) {
+ return new PipelineElementDescriptionStorageImpl(
+ dataProcessorStorage, dataStreamStorage, dataSinkStorage, adapterDescriptionStorage);
}
@Bean
diff --git a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/extensions/TransportAwareExtensionServiceRequestManagerTest.java b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/extensions/TransportAwareExtensionServiceRequestManagerTest.java
new file mode 100644
index 0000000000..55df7b7903
--- /dev/null
+++ b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/extensions/TransportAwareExtensionServiceRequestManagerTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.service.core.extensions;
+
+import org.apache.streampipes.manager.api.extensions.ExtensionServiceOperationResult;
+import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequest;
+import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestManager;
+import org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestTarget;
+import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration;
+import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag;
+import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix;
+import org.apache.streampipes.model.extensions.transport.ExtensionServiceBrokerTopics;
+import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+class TransportAwareExtensionServiceRequestManagerTest {
+
+ private final ExtensionServiceRequestManager http = mock(ExtensionServiceRequestManager.class);
+ private final NatsExtensionServiceRequestManager nats = mock(NatsExtensionServiceRequestManager.class);
+ private final IExtensionsServiceStorage storage = mock(IExtensionsServiceStorage.class);
+ private final ExtensionServiceRequest request = ExtensionServiceRequest.get(
+ ExtensionServiceRequestTarget.of("http://extension", "service-1", "monitoring"), null);
+ private final ExtensionServiceOperationResult response = new ExtensionServiceOperationResult(200, null);
+
+ @Test
+ void autoModeReflectsChangesInStoredTransportTags() throws IOException {
+ var registration = new SpServiceRegistration();
+ registration.setTags(Set.of(
+ SpServiceTag.create(SpServiceTagPrefix.CUSTOM, ExtensionServiceBrokerTopics.TRANSPORT_TAG_NATS)));
+ when(storage.getElementById("service-1")).thenReturn(registration);
+ when(nats.request(request)).thenReturn(response);
+ when(http.request(request)).thenReturn(response);
+ var manager = manager(CoreExtensionTransportMode.AUTO);
+
+ assertSame(response, manager.request(request));
+ registration.setTags(Set.of());
+ assertSame(response, manager.request(request));
+
+ verify(nats).request(request);
+ verify(http).request(request);
+ verify(storage, times(2)).getElementById("service-1");
+ }
+
+ @Test
+ void autoModeUsesHttpWhenServiceIsMissing() throws IOException {
+ when(http.request(request)).thenReturn(response);
+
+ assertSame(response, manager(CoreExtensionTransportMode.AUTO).request(request));
+
+ verifyNoInteractions(nats);
+ }
+
+ @Test
+ void httpModeDoesNotQueryStorage() throws IOException {
+ when(http.request(request)).thenReturn(response);
+
+ assertSame(response, manager(CoreExtensionTransportMode.HTTP).request(request));
+
+ verifyNoInteractions(storage, nats);
+ }
+
+ @Test
+ void natsModeDoesNotQueryStorage() throws IOException {
+ when(nats.request(request)).thenReturn(response);
+
+ assertSame(response, manager(CoreExtensionTransportMode.NATS).request(request));
+
+ verifyNoInteractions(storage, http);
+ }
+
+ private TransportAwareExtensionServiceRequestManager manager(CoreExtensionTransportMode mode) {
+ return new TransportAwareExtensionServiceRequestManager(http, nats, mode, storage);
+ }
+}
diff --git a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/RenameAssetLinkTypesMigrationTest.java b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/RenameAssetLinkTypesMigrationTest.java
index 17abb41d35..fdc3e6ccbe 100644
--- a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/RenameAssetLinkTypesMigrationTest.java
+++ b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/RenameAssetLinkTypesMigrationTest.java
@@ -21,7 +21,6 @@ package org.apache.streampipes.service.core.migrations.v099;
import org.apache.streampipes.commons.constants.GenericDocTypes;
import org.apache.streampipes.storage.api.system.IGenericStorage;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -47,7 +46,7 @@ class RenameAssetLinkTypesMigrationTest {
@BeforeEach
void setUp() {
genericStorage = mock(IGenericStorage.class);
- migration = new RenameAssetLinkTypesMigration(genericStorage, new ObjectMapper());
+ migration = new RenameAssetLinkTypesMigration(genericStorage);
}
@Test
diff --git a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscovery.java b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscovery.java
deleted file mode 100644
index 8f054e9063..0000000000
--- a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscovery.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.svcdiscovery;
-
-import org.apache.streampipes.storage.management.StorageDispatcher;
-import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery;
-
-public class SpServiceDiscovery {
-
- public static ISpServiceDiscovery getServiceDiscovery() {
- return new SpServiceDiscoveryCore(StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage());
- }
-
-}
diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/core/INoSqlStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/core/INoSqlStorage.java
index 820b17e6c2..80a4e79630 100644
--- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/core/INoSqlStorage.java
+++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/core/INoSqlStorage.java
@@ -17,21 +17,10 @@
*/
package org.apache.streampipes.storage.api.core;
-import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
-import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
-import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
-import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
import org.apache.streampipes.storage.api.system.IGenericStorage;
public interface INoSqlStorage {
IGenericStorage getGenericStorage();
- IPipelineElementDescriptionStorage getPipelineElementDescriptionStorage();
-
- IDataStreamStorage getDataStreamStorage();
-
- IExtensionsServiceStorage getExtensionsServiceStorage();
-
- ICompactPipelineTemplateStorage getPipelineTemplateStorage();
}
diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java
deleted file mode 100644
index 4727d6ce5a..0000000000
--- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.storage.couchdb;
-
-import org.apache.streampipes.storage.api.core.INoSqlStorage;
-import org.apache.streampipes.storage.api.pipeline.ICompactPipelineTemplateStorage;
-import org.apache.streampipes.storage.api.pipeline.IDataStreamStorage;
-import org.apache.streampipes.storage.api.pipeline.IPipelineElementDescriptionStorage;
-import org.apache.streampipes.storage.api.system.IExtensionsServiceStorage;
-import org.apache.streampipes.storage.api.system.IGenericStorage;
-import org.apache.streampipes.storage.couchdb.impl.pipeline.CompactPipelineTemplateStorageImpl;
-import org.apache.streampipes.storage.couchdb.impl.pipeline.DataStreamStorageImpl;
-import org.apache.streampipes.storage.couchdb.impl.pipeline.PipelineElementDescriptionStorageImpl;
-import org.apache.streampipes.storage.couchdb.impl.system.ExtensionsServiceStorageImpl;
-import org.apache.streampipes.storage.couchdb.impl.system.GenericStorageImpl;
-
-public class CouchDbStorageManager implements INoSqlStorage {
-
- @Override
- public IGenericStorage getGenericStorage() {
- return new GenericStorageImpl();
- }
-
- @Override
- public IPipelineElementDescriptionStorage getPipelineElementDescriptionStorage() {
- return new PipelineElementDescriptionStorageImpl();
- }
-
- @Override
- public IDataStreamStorage getDataStreamStorage() {
- return new DataStreamStorageImpl();
- }
-
- @Override
- public IExtensionsServiceStorage getExtensionsServiceStorage() {
- return new ExtensionsServiceStorageImpl();
- }
-
- @Override
- public ICompactPipelineTemplateStorage getPipelineTemplateStorage() {
- return new CompactPipelineTemplateStorageImpl();
- }
-}
diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/core/DefaultCrudStorage.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/core/DefaultCrudStorage.java
index 92fb459402..d749334d1f 100644
--- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/core/DefaultCrudStorage.java
+++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/core/DefaultCrudStorage.java
@@ -20,22 +20,15 @@ package org.apache.streampipes.storage.couchdb.impl.core;
import org.apache.streampipes.model.shared.api.Storable;
import org.apache.streampipes.storage.api.core.CRUDStorage;
-import org.apache.streampipes.storage.couchdb.CouchDbStorageManager;
import org.apache.streampipes.storage.couchdb.dao.AbstractDao;
import org.lightcouch.CouchDbClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.function.Supplier;
public class DefaultCrudStorage<T extends Storable> extends AbstractDao<T> implements CRUDStorage<T> {
- private static final Logger LOG = LoggerFactory.getLogger(DefaultCrudStorage.class);
-
- public DefaultCrudStorage(Supplier<CouchDbClient> couchDbClientSupplier,
- Class<T> clazz) {
+ public DefaultCrudStorage(Supplier<CouchDbClient> couchDbClientSupplier, Class<T> clazz) {
super(couchDbClientSupplier, clazz);
}
@@ -52,11 +45,6 @@ public class DefaultCrudStorage<T extends Storable> extends AbstractDao<T> imple
@Override
public void deleteElement(T element) {
- try {
- new CouchDbStorageManager().getGenericStorage().deleteAssetLinkToResource(element.getElementId());
- } catch (IOException e) {
- LOG.error("Asset link for " + element.getElementId() + " could not be deleted.");
- }
delete(element.getElementId());
}
}
diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/pipeline/PipelineElementDescriptionStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/pipeline/PipelineElementDescriptionStorageImpl.java
index f59bf102bf..08c4455ef3 100644
--- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/pipeline/PipelineElementDescriptionStorageImpl.java
+++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/pipeline/PipelineElementDescriptionStorageImpl.java
@@ -44,6 +44,16 @@ public class PipelineElementDescriptionStorageImpl implements IPipelineElementDe
this.adapterStorage = new AdapterDescriptionStorageImpl();
}
+ public PipelineElementDescriptionStorageImpl(IDataProcessorStorage dataProcessorStorage,
+ IDataStreamStorage dataStreamStorage,
+ IDataSinkStorage dataSinkStorage,
+ IAdapterStorage adapterStorage) {
+ this.dataProcessorStorage = dataProcessorStorage;
+ this.dataStreamStorage = dataStreamStorage;
+ this.dataSinkStorage = dataSinkStorage;
+ this.adapterStorage = adapterStorage;
+ }
+
@Override
public boolean storeDataStream(SpDataStream stream) {
this.dataStreamStorage.persist(stream);
diff --git a/streampipes-storage-management/src/main/java/org/apache/streampipes/storage/management/StorageDispatcher.java b/streampipes-storage-management/src/main/java/org/apache/streampipes/storage/management/StorageDispatcher.java
deleted file mode 100644
index 7fdc185b9c..0000000000
--- a/streampipes-storage-management/src/main/java/org/apache/streampipes/storage/management/StorageDispatcher.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.storage.management;
-
-import org.apache.streampipes.storage.api.core.INoSqlStorage;
-import org.apache.streampipes.storage.couchdb.CouchDbStorageManager;
-
-public enum StorageDispatcher {
-
- INSTANCE;
-
- public INoSqlStorage getNoSqlStore() {
- return new CouchDbStorageManager();
- }
-
-}