You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2022/08/26 08:52:07 UTC
[incubator-streampipes] branch rel/0.70.0 updated: [STREAMPIPES-565] Let export feature ignore deleted resources
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch rel/0.70.0
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/rel/0.70.0 by this push:
new c2c974c59 [STREAMPIPES-565] Let export feature ignore deleted resources
c2c974c59 is described below
commit c2c974c5992ffa3d2ee1ea24d6cbe6cd1426df55
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Fri Aug 26 10:51:54 2022 +0200
[STREAMPIPES-565] Let export feature ignore deleted resources
---
.../export/generator/ExportPackageGenerator.java | 13 +++++++++++--
.../streampipes/export/resolver/AbstractResolver.java | 18 ++++++++++++++++--
.../streampipes/export/resolver/AdapterResolver.java | 7 ++++++-
.../streampipes/export/resolver/DashboardResolver.java | 12 +++++++++++-
.../export/resolver/DashboardWidgetResolver.java | 6 +++++-
.../export/resolver/DataSourceResolver.java | 6 +++++-
.../streampipes/export/resolver/DataViewResolver.java | 12 +++++++++++-
.../export/resolver/DataViewWidgetResolver.java | 6 +++++-
.../streampipes/export/resolver/FileResolver.java | 6 +++++-
.../export/resolver/MeasurementResolver.java | 6 +++++-
.../streampipes/export/resolver/PipelineResolver.java | 6 +++++-
ui/src/app/assets/dialog/base-asset-links.directive.ts | 1 +
.../edit-asset-link-dialog.component.html | 2 +-
.../edit-asset-link-dialog.component.ts | 2 +-
14 files changed, 88 insertions(+), 15 deletions(-)
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
index 3350e69e8..8073dd567 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.export.generator;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streampipes.commons.exceptions.ElementNotFoundException;
import org.apache.streampipes.export.resolver.*;
import org.apache.streampipes.export.utils.SerializationUtils;
import org.apache.streampipes.manager.file.FileManager;
@@ -28,6 +29,8 @@ import org.apache.streampipes.model.export.ExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.model.export.StreamPipesApplicationPackage;
import org.apache.streampipes.storage.management.StorageDispatcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
@@ -38,6 +41,8 @@ import java.util.stream.Collectors;
public class ExportPackageGenerator {
+ private static final Logger LOG = LoggerFactory.getLogger(ExportPackageGenerator.class);
+
private final ExportConfiguration exportConfiguration;
private ObjectMapper defaultMapper;
private ObjectMapper spMapper;
@@ -138,8 +143,12 @@ public class ExportPackageGenerator {
var sanitizedResourceId = sanitize(resourceId);
builder.addText(sanitizedResourceId, resolver.getSerializedDocument(resourceId));
function.accept(sanitizedResourceId);
- } catch (JsonProcessingException e) {
- e.printStackTrace();
+ } catch (JsonProcessingException | ElementNotFoundException e) {
+ LOG.warn(
+ "Could not find document with resource id {} with resolver {}",
+ exportItem.getResourceId(),
+ resolver.getClass().getCanonicalName(),
+ e);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
index c0bae151f..d4e0ed2d4 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.export.resolver;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streampipes.commons.exceptions.ElementNotFoundException;
import org.apache.streampipes.export.utils.SerializationUtils;
import org.apache.streampipes.model.assets.AssetLink;
import org.apache.streampipes.model.export.ExportItem;
@@ -27,6 +28,7 @@ import org.apache.streampipes.storage.api.INoSqlStorage;
import org.apache.streampipes.storage.management.StorageDispatcher;
import org.lightcouch.DocumentConflictException;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -44,12 +46,22 @@ public abstract class AbstractResolver<T> {
return assetLinks
.stream()
.map(link -> findDocument(link.getResourceId()))
+ .filter(this::existsDoc)
.map(this::convert)
.collect(Collectors.toSet());
}
- public String getSerializedDocument(String resourceId) throws JsonProcessingException {
- return SerializationUtils.getSpObjectMapper().writeValueAsString(findDocument(resourceId));
+ protected boolean existsDoc(T doc) {
+ return Objects.nonNull(doc);
+ }
+
+ public String getSerializedDocument(String resourceId) throws JsonProcessingException, ElementNotFoundException {
+ var document = findDocument(resourceId);
+ if (document != null) {
+ return SerializationUtils.getSpObjectMapper().writeValueAsString(modifyDocumentForExport(document));
+ } else {
+ throw new ElementNotFoundException("Could not find element with resource id " + resourceId);
+ }
}
protected INoSqlStorage getNoSqlStore() {
@@ -58,6 +70,8 @@ public abstract class AbstractResolver<T> {
public abstract T findDocument(String resourceId);
+ public abstract T modifyDocumentForExport(T doc);
+
public abstract T readDocument(String serializedDoc) throws JsonProcessingException;
public abstract ExportItem convert(T document);
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
index 681afbaab..d3b7b2c29 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
@@ -30,12 +30,17 @@ public class AdapterResolver extends AbstractResolver<AdapterDescription> {
@Override
public AdapterDescription findDocument(String resourceId) {
- var doc = getNoSqlStore().getAdapterInstanceStorage().getAdapter(resourceId);
+ return getNoSqlStore().getAdapterInstanceStorage().getAdapter(resourceId);
+ }
+
+ @Override
+ public AdapterDescription modifyDocumentForExport(AdapterDescription doc) {
doc.setRev(null);
doc.setSelectedEndpointUrl(null);
if (doc instanceof AdapterStreamDescription) {
((AdapterStreamDescription) doc).setRunning(false);
}
+
return doc;
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
index 467c8aa07..a756c03d0 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
@@ -25,17 +25,27 @@ import org.apache.streampipes.model.dashboard.DashboardModel;
import org.apache.streampipes.model.export.ExportItem;
import java.util.List;
+import java.util.Objects;
import java.util.stream.Collectors;
public class DashboardResolver extends AbstractResolver<DashboardModel> {
@Override
public DashboardModel findDocument(String resourceId) {
- var doc = getNoSqlStore().getDashboardStorage().getDashboard(resourceId);
+ return getNoSqlStore().getDashboardStorage().getDashboard(resourceId);
+ }
+
+ @Override
+ public DashboardModel modifyDocumentForExport(DashboardModel doc) {
doc.setCouchDbRev(null);
return doc;
}
+ @Override
+ protected boolean existsDoc(DashboardModel doc) {
+ return Objects.nonNull(doc) && doc.getCouchDbId() != null;
+ }
+
@Override
public DashboardModel readDocument(String serializedDoc) throws JsonProcessingException {
return SerializationUtils.getSpObjectMapper().readValue(serializedDoc, DashboardModel.class);
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java
index b8eb3b867..91d4876bb 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java
@@ -27,7 +27,11 @@ public class DashboardWidgetResolver extends AbstractResolver<DashboardWidgetMod
@Override
public DashboardWidgetModel findDocument(String resourceId) {
- var doc = getNoSqlStore().getDashboardWidgetStorage().getDashboardWidget(resourceId);
+ return getNoSqlStore().getDashboardWidgetStorage().getDashboardWidget(resourceId);
+ }
+
+ @Override
+ public DashboardWidgetModel modifyDocumentForExport(DashboardWidgetModel doc) {
doc.setRev(null);
return doc;
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
index 98a338b8a..14effbb19 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
@@ -28,7 +28,11 @@ public class DataSourceResolver extends AbstractResolver<SpDataStream> {
@Override
public SpDataStream findDocument(String resourceId) {
- var doc = getNoSqlStore().getDataStreamStorage().getElementById(resourceId);
+ return getNoSqlStore().getDataStreamStorage().getElementById(resourceId);
+ }
+
+ @Override
+ public SpDataStream modifyDocumentForExport(SpDataStream doc) {
doc.setRev(null);
return doc;
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewResolver.java
index 47e946d3e..4c81c427a 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewResolver.java
@@ -26,17 +26,27 @@ import org.apache.streampipes.model.dashboard.DashboardModel;
import org.apache.streampipes.model.export.ExportItem;
import java.util.List;
+import java.util.Objects;
import java.util.stream.Collectors;
public class DataViewResolver extends AbstractResolver<DashboardModel> {
@Override
public DashboardModel findDocument(String resourceId) {
- var doc = getNoSqlStore().getDataExplorerDashboardStorage().getDashboard(resourceId);
+ return getNoSqlStore().getDataExplorerDashboardStorage().getDashboard(resourceId);
+ }
+
+ @Override
+ public DashboardModel modifyDocumentForExport(DashboardModel doc) {
doc.setCouchDbRev(null);
return doc;
}
+ @Override
+ protected boolean existsDoc(DashboardModel doc) {
+ return Objects.nonNull(doc) && doc.getCouchDbId() != null;
+ }
+
@Override
public DashboardModel readDocument(String serializedDoc) throws JsonProcessingException {
return SerializationUtils.getSpObjectMapper().readValue(serializedDoc, DashboardModel.class);
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java
index 969312517..0414b37ec 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java
@@ -27,7 +27,11 @@ public class DataViewWidgetResolver extends AbstractResolver<DataExplorerWidgetM
@Override
public DataExplorerWidgetModel findDocument(String resourceId) {
- var doc = getNoSqlStore().getDataExplorerWidgetStorage().getDataExplorerWidget(resourceId);
+ return getNoSqlStore().getDataExplorerWidgetStorage().getDataExplorerWidget(resourceId);
+ }
+
+ @Override
+ public DataExplorerWidgetModel modifyDocumentForExport(DataExplorerWidgetModel doc) {
doc.setRev(null);
return doc;
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
index bd967fc90..e0e2ae8c6 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
@@ -27,7 +27,11 @@ public class FileResolver extends AbstractResolver<FileMetadata> {
@Override
public FileMetadata findDocument(String resourceId) {
- var doc = getNoSqlStore().getFileMetadataStorage().getMetadataById(resourceId);
+ return getNoSqlStore().getFileMetadataStorage().getMetadataById(resourceId);
+ }
+
+ @Override
+ public FileMetadata modifyDocumentForExport(FileMetadata doc) {
doc.setRev(null);
return doc;
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
index ecde1338c..c5a86df51 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
@@ -27,7 +27,11 @@ public class MeasurementResolver extends AbstractResolver<DataLakeMeasure> {
@Override
public DataLakeMeasure findDocument(String resourceId) {
- var doc = getNoSqlStore().getDataLakeStorage().findOne(resourceId);
+ return getNoSqlStore().getDataLakeStorage().findOne(resourceId);
+ }
+
+ @Override
+ public DataLakeMeasure modifyDocumentForExport(DataLakeMeasure doc) {
doc.setRev(null);
return doc;
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
index 173289f0c..2117f549e 100644
--- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
@@ -31,7 +31,11 @@ public class PipelineResolver extends AbstractResolver<Pipeline> {
@Override
public Pipeline findDocument(String resourceId) {
- var doc = getNoSqlStore().getPipelineStorageAPI().getPipeline(resourceId);
+ return getNoSqlStore().getPipelineStorageAPI().getPipeline(resourceId);
+ }
+
+ @Override
+ public Pipeline modifyDocumentForExport(Pipeline doc) {
doc.setRev(null);
doc.setRestartOnSystemReboot(false);
doc.setRunning(false);
diff --git a/ui/src/app/assets/dialog/base-asset-links.directive.ts b/ui/src/app/assets/dialog/base-asset-links.directive.ts
index 79002369b..28df15b43 100644
--- a/ui/src/app/assets/dialog/base-asset-links.directive.ts
+++ b/ui/src/app/assets/dialog/base-asset-links.directive.ts
@@ -89,6 +89,7 @@ export abstract class BaseAssetLinksDirective {
...this.files,
...this.adapters
];
+ this.afterResourcesLoaded();
});
}
diff --git a/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.html b/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.html
index ca528415c..143d09387 100644
--- a/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.html
+++ b/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.html
@@ -79,7 +79,7 @@
</div>
<div *ngIf="selectedLinkType.linkQueryHint === 'adapter'" fxLayout="column" class="link-configuration">
<mat-form-field color="accent" fxFlex="100">
- <mat-label>Data Views</mat-label>
+ <mat-label>Adapter</mat-label>
<mat-select (selectionChange)="changeLabel($event.value.elementId, $event.value.name, $event.value)"
[(value)]="currentResource"
fxFlex
diff --git a/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.ts b/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.ts
index 0a1b8b6f5..417a086b7 100644
--- a/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.ts
+++ b/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.ts
@@ -90,7 +90,7 @@ export class EditAssetLinkDialogComponent extends BaseAssetLinksDirective implem
}
getCurrAssetLinkType(): AssetLinkType {
- return this.assetLinkTypes.find(a => a.linkType === this.assetLink.linkType);
+ return this.assetLinkTypes.find(a => a.linkType === this.clonedAssetLink.linkType);
}
store() {