You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2022/07/31 21:20:32 UTC
[incubator-streampipes] 06/06: [STREAMPIPES-565] Add data export feature
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch STREAMPIPES-545
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 0f37e31f3f8c82feed16e3a02d61cb8a0c1d117b
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Sun Jul 31 23:20:12 2022 +0200
[STREAMPIPES-565] Add data export feature
---
.gitignore | 2 +-
pom.xml | 4 +
.../backend/StreamPipesResourceConfig.java | 1 +
streampipes-data-explorer-commons/pom.xml | 19 ++-
.../dataexplorer/commons/DataExplorerWriter.java | 17 ++
.../dataexplorer/sdk/DataLakeQueryOrdering.java | 18 ++
streampipes-data-export/pom.xml | 44 +++++
.../streampipes/export/AssetLinkCollector.java | 50 ++++++
.../streampipes/export/AssetLinkResolver.java | 83 +++++++++
.../apache/streampipes/export/ExportManager.java | 46 +++++
.../export/constants/ResolvableAssetLinks.java | 26 +--
.../export/generator/ExportPackageGenerator.java | 152 +++++++++++++++++
.../export/generator/ZipFileBuilder.java | 113 ++++++++++++
.../export/resolver/AbstractResolver.java | 53 ++++++
.../export/resolver/AdapterResolver.java | 25 ++-
.../export/resolver/DashboardResolver.java | 44 +++++
.../export/resolver/DashboardWidgetResolver.java | 24 ++-
.../export/resolver/DataSourceResolver.java | 24 ++-
.../export/resolver/DataViewResolver.java | 45 +++++
.../export/resolver/DataViewWidgetResolver.java | 24 ++-
.../export/resolver/MeasurementResolver.java | 24 ++-
.../export/resolver/PipelineResolver.java | 24 ++-
.../export/utils/SerializationUtils.java | 26 ++-
.../sinks/internal/jvm/datalake/DataLakeUtils.java | 3 +-
.../apache/streampipes/model/assets/AssetLink.java | 24 +++
.../model/export/AssetExportConfiguration.java | 120 +++++++++++++
.../model/export/ExportConfiguration.java | 30 ++--
.../AssetLink.java => export/ExportItem.java} | 41 +++--
.../export/StreamPipesApplicationPackage.java | 190 +++++++++++++++++++++
streampipes-rest/pom.xml | 5 +
.../rest/impl/admin/DataExportResource.java | 60 +++++++
.../src/lib/model/assets/asset.model.ts | 1 +
.../src/lib/model/gen/streampipes-model.ts | 102 +++++++++--
.../asset-details-panel.component.ts | 9 +-
.../edit-asset-link-dialog.component.html | 12 ++
.../edit-asset-link-dialog.component.ts | 22 ++-
ui/src/app/configuration/configuration-tabs.ts | 1 +
ui/src/app/configuration/configuration.module.ts | 12 ++
.../export/data-export-import.component.html | 52 ++++++
.../export/data-export-import.component.scss | 22 ---
.../export/data-export-import.component.ts | 81 +++++++++
.../configuration/export/data-export.service.ts | 46 +++++
.../data-export-dialog.component.html | 49 ++++++
.../data-export-dialog.component.scss | 22 +--
.../export-dialog/data-export-dialog.component.ts | 63 +++++++
.../data-export-item.component.html | 26 +++
.../data-export-item.component.scss | 22 ---
.../data-export-item/data-export-item.component.ts | 34 ++--
.../data-import-dialog.component.html | 17 ++
.../data-import-dialog.component.scss | 22 +--
.../import-dialog/data-import-dialog.component.ts} | 17 +-
.../messaging-configuration.component.ts | 2 +-
.../pipeline-element-configuration.component.ts | 2 +-
.../security-configuration.component.ts | 2 +-
54 files changed, 1708 insertions(+), 291 deletions(-)
diff --git a/.gitignore b/.gitignore
index 4b3258fc0..324db8704 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,7 +83,7 @@ npm-debug.log
/test_data/
ui/src/assets/lib/apps/*
-
+ui/.angular
# compiled output
ui/dist
diff --git a/pom.xml b/pom.xml
index 575df68cf..be1159d57 100644
--- a/pom.xml
+++ b/pom.xml
@@ -957,6 +957,7 @@ IoT data streams.
<module>streampipes-resource-management</module>
<module>streampipes-sdk-bundle</module>
<module>streampipes-data-explorer-commons</module>
+ <module>streampipes-data-export</module>
</modules>
<profiles>
@@ -1273,6 +1274,9 @@ IoT data streams.
<!-- Exclude some UI files which we need to check in more detail -->
<exclude>ui/src/assets/img/svg/**</exclude>
+ <!-- Exclude .angular folder -->
+ <exclude>ui/.angular/**</exclude>
+
<!-- Exclude disclaimer and notice files -->
<exclude>DISCLAIMER</exclude>
<exclude>NOTICE-binary</exclude>
diff --git a/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesResourceConfig.java b/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesResourceConfig.java
index 0a6efad7b..4924a5352 100644
--- a/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesResourceConfig.java
+++ b/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesResourceConfig.java
@@ -62,6 +62,7 @@ public class StreamPipesResourceConfig extends ResourceConfig {
register(ContainerProvidedOptions.class);
register(DashboardWidget.class);
register(Dashboard.class);
+ register(DataExportResource.class);
register(DataLakeImageResource.class);
register(DataLakeResourceV3.class);
register(DataLakeMeasureResourceV3.class);
diff --git a/streampipes-data-explorer-commons/pom.xml b/streampipes-data-explorer-commons/pom.xml
index 2d14cd7e1..999370c82 100644
--- a/streampipes-data-explorer-commons/pom.xml
+++ b/streampipes-data-explorer-commons/pom.xml
@@ -1,4 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -46,4 +63,4 @@
<maven.compiler.target>11</maven.compiler.target>
</properties>
-</project>
\ No newline at end of file
+</project>
diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerWriter.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerWriter.java
index afb7387e2..a44b93b79 100644
--- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerWriter.java
+++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerWriter.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.streampipes.dataexplorer.commons;
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryOrdering.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryOrdering.java
index b187cd38a..6952d0dcc 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryOrdering.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryOrdering.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
package org.apache.streampipes.dataexplorer.sdk;
public enum DataLakeQueryOrdering {
diff --git a/streampipes-data-export/pom.xml b/streampipes-data-export/pom.xml
new file mode 100644
index 000000000..216981d8b
--- /dev/null
+++ b/streampipes-data-export/pom.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-parent</artifactId>
+ <groupId>org.apache.streampipes</groupId>
+ <version>0.70.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-data-export</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-model</artifactId>
+ <version>0.70.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-storage-management</artifactId>
+ <version>0.70.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkCollector.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkCollector.java
new file mode 100644
index 000000000..ba3e16df8
--- /dev/null
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkCollector.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import org.apache.streampipes.model.assets.AssetLink;
+import org.apache.streampipes.model.assets.SpAsset;
+import org.apache.streampipes.model.assets.SpAssetModel;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class AssetLinkCollector {
+
+ private SpAssetModel assetModel;
+
+ public AssetLinkCollector(SpAssetModel assetModel) {
+ this.assetModel = assetModel;
+ }
+
+ public Set<AssetLink> collectAssetLinks() {
+ var assetLinks = new HashSet<>(assetModel.getAssetLinks());
+ assetModel.getAssets().forEach(asset -> addLinks(assetLinks, asset));
+
+ return assetLinks;
+ }
+
+ private void addLinks(HashSet<AssetLink> assetLinks,
+ SpAsset asset) {
+ assetLinks.addAll(asset.getAssetLinks());
+ if (asset.getAssets() != null) {
+ asset.getAssets().forEach(a -> addLinks(assetLinks, a));
+ }
+ }
+}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
new file mode 100644
index 000000000..795c88f3e
--- /dev/null
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streampipes.export.constants.ResolvableAssetLinks;
+import org.apache.streampipes.export.resolver.*;
+import org.apache.streampipes.export.utils.SerializationUtils;
+import org.apache.streampipes.model.assets.AssetLink;
+import org.apache.streampipes.model.assets.SpAssetModel;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
+import org.apache.streampipes.storage.management.StorageDispatcher;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class AssetLinkResolver {
+
+ private final String assetId;
+ private final ObjectMapper mapper;
+
+ public AssetLinkResolver(String assetId) {
+ this.assetId = assetId;
+ this.mapper = SerializationUtils.getDefaultObjectMapper();
+ }
+
+ public AssetExportConfiguration resolveResources() {
+
+ try {
+ var asset = getAsset();
+ var assetLinks = new AssetLinkCollector(asset).collectAssetLinks();
+ var exportConfig = new AssetExportConfiguration();
+ exportConfig.setAssetId(this.assetId);
+ exportConfig.setAssetName(asset.getAssetName());
+ exportConfig.setAdapters(new AdapterResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.ADAPTER)));
+ exportConfig.setDashboards(new DashboardResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.DASHBOARD)));
+ exportConfig.setDataViews(new DataViewResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.DATA_VIEW)));
+ exportConfig.setDataSources(new DataSourceResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.DATA_SOURCE)));
+ exportConfig.setPipelines(new PipelineResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.PIPELINE)));
+ exportConfig.setDataLakeMeasures(new MeasurementResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.MEASUREMENT)));
+
+ return exportConfig;
+ } catch (IOException e) {
+ e.printStackTrace();
+ return new AssetExportConfiguration();
+ }
+ }
+
+ private Set<AssetLink> getLinks(Set<AssetLink> assetLinks,
+ String queryHint) {
+ return assetLinks
+ .stream()
+ .filter(link -> link.getQueryHint().equals(queryHint))
+ .collect(Collectors.toSet());
+ }
+
+ private SpAssetModel getAsset() throws IOException {
+ return deserialize(StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage().findOne(this.assetId));
+ }
+
+ private SpAssetModel deserialize(Map<String, Object> asset) {
+ return this.mapper.convertValue(asset, SpAssetModel.class);
+ }
+
+}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
new file mode 100644
index 000000000..1ceaa5405
--- /dev/null
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import org.apache.streampipes.export.generator.ExportPackageGenerator;
+import org.apache.streampipes.model.export.ExportConfiguration;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class ExportManager {
+
+ public static ExportConfiguration getExportPreview(List<String> selectedAssetIds) {
+ var exportConfig = new ExportConfiguration();
+ var assetExportConfigurations = selectedAssetIds
+ .stream()
+ .map(assetId -> new AssetLinkResolver(assetId).resolveResources())
+ .collect(Collectors.toList());
+
+ exportConfig.setAssetExportConfiguration(assetExportConfigurations);
+
+ return exportConfig;
+ }
+
+ public static byte[] getExportPackage(ExportConfiguration exportConfiguration) throws IOException {
+ return new ExportPackageGenerator(exportConfiguration).generateExportPackage();
+ }
+
+}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/constants/ResolvableAssetLinks.java
similarity index 55%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-data-export/src/main/java/org/apache/streampipes/export/constants/ResolvableAssetLinks.java
index 8fa1736b3..d3c9cd721 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/constants/ResolvableAssetLinks.java
@@ -16,24 +16,14 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
+package org.apache.streampipes.export.constants;
-public class DataLakeUtils {
+public class ResolvableAssetLinks {
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
-
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
- }
-
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
- }
+ public static final String DATA_VIEW = "data-view";
+ public static final String DASHBOARD = "dashboard";
+ public static final String MEASUREMENT = "measurement";
+ public static final String ADAPTER = "adapter";
+ public static final String DATA_SOURCE = "data-source";
+ public static final String PIPELINE = "pipeline";
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
new file mode 100644
index 000000000..832ae826b
--- /dev/null
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export.generator;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streampipes.export.resolver.*;
+import org.apache.streampipes.export.utils.SerializationUtils;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
+import org.apache.streampipes.model.export.ExportConfiguration;
+import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.model.export.StreamPipesApplicationPackage;
+import org.apache.streampipes.storage.management.StorageDispatcher;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+public class ExportPackageGenerator {
+
+ private final ExportConfiguration exportConfiguration;
+ private ObjectMapper defaultMapper;
+ private ObjectMapper spMapper;
+
+ public ExportPackageGenerator(ExportConfiguration exportConfiguration) {
+ this.exportConfiguration = exportConfiguration;
+ this.defaultMapper = SerializationUtils.getDefaultObjectMapper();
+ this.spMapper = SerializationUtils.getSpObjectMapper();
+ }
+
+ public byte[] generateExportPackage() throws IOException {
+ ZipFileBuilder builder = ZipFileBuilder.create();
+ var manifest = new StreamPipesApplicationPackage();
+
+ addAssets(builder, exportConfiguration
+ .getAssetExportConfiguration()
+ .stream()
+ .map(AssetExportConfiguration::getAssetId)
+ .collect(Collectors.toList()));
+
+ this.exportConfiguration.getAssetExportConfiguration().forEach(config -> {
+
+ config.getAdapters().forEach(item -> addDoc(builder,
+ item,
+ new AdapterResolver(),
+ manifest::addAdapter));
+
+ config.getDashboards().forEach(item -> {
+ var resolver = new DashboardResolver();
+ addDoc(builder,
+ item,
+ resolver,
+ manifest::addDashboard);
+
+ var widgets = resolver.getWidgets(item.getResourceId());
+ var widgetResolver = new DashboardWidgetResolver();
+ widgets.forEach(widgetId -> addDoc(builder, widgetId, widgetResolver, manifest::addDashboardWidget));
+ });
+
+ config.getDataSources().forEach(item -> addDoc(builder,
+ item,
+ new DataSourceResolver(),
+ manifest::addDataSource));
+
+ config.getDataLakeMeasures().forEach(item -> addDoc(builder,
+ item,
+ new MeasurementResolver(),
+ manifest::addDataLakeMeasure));
+
+ config.getPipelines().forEach(item -> addDoc(builder,
+ item,
+ new PipelineResolver(),
+ manifest::addPipeline));
+
+ config.getDataViews().forEach(item -> {
+ var resolver = new DataViewResolver();
+ addDoc(builder,
+ item,
+ resolver,
+ manifest::addDataView);
+
+ var widgets = resolver.getWidgets(item.getResourceId());
+ var widgetResolver = new DataViewWidgetResolver();
+ widgets.forEach(widgetId -> addDoc(builder, widgetId, widgetResolver, manifest::addDataViewWidget));
+ });
+ });
+
+ builder.addManifest(defaultMapper.writeValueAsString(manifest));
+
+
+ return builder.buildZip();
+ }
+
+ private void addDoc(ZipFileBuilder builder,
+ String resourceId,
+ AbstractResolver<?> resolver,
+ Consumer<String> function) {
+ addDoc(builder, new ExportItem(resourceId, "", true), resolver, function);
+ }
+
+ private void addDoc(ZipFileBuilder builder,
+ ExportItem exportItem,
+ AbstractResolver<?> resolver,
+ Consumer<String> function) {
+ try {
+ var resourceId = exportItem.getResourceId();
+ var sanitizedResourceId = sanitize(resourceId);
+ builder.addText(sanitizedResourceId, resolver.getSerializedDocument(resourceId));
+ function.accept(sanitizedResourceId);
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private String sanitize(String resourceId) {
+ return resourceId.replaceAll(":", "").replaceAll("\\.", "");
+ }
+
+ private void addAssets(ZipFileBuilder builder,
+ List<String> assetIds) {
+ assetIds.forEach(assetId -> {
+ try {
+ var asset = getAsset(assetId);
+ builder.addText(String.valueOf(asset.get("_id")), this.defaultMapper.writeValueAsString(asset));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ });
+ }
+
+ private Map<String, Object> getAsset(String assetId) throws IOException {
+ return StorageDispatcher.INSTANCE.getNoSqlStore().getGenericStorage().findOne(assetId);
+ }
+}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ZipFileBuilder.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ZipFileBuilder.java
new file mode 100644
index 000000000..ea93d03e6
--- /dev/null
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ZipFileBuilder.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export.generator;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+public class ZipFileBuilder {
+
+ private final Map<String, byte[]> binaryEntries;
+ private final Map<String, String> textEntries;
+ private final Map<String, File> fileEntries;
+ private String manifest;
+
+ public static ZipFileBuilder create() {
+ return new ZipFileBuilder();
+ }
+
+ private ZipFileBuilder() {
+ this.binaryEntries = new HashMap<>();
+ this.fileEntries = new HashMap<>();
+ this.textEntries = new HashMap<>();
+ }
+
+ public ZipFileBuilder addText(String filename,
+ String content) {
+ this.textEntries.put(filename, content);
+
+ return this;
+ }
+
+ public ZipFileBuilder addBinary(String filename,
+ byte[] content) {
+ this.binaryEntries.put(filename, content);
+
+ return this;
+ }
+
+ public ZipFileBuilder addFile(String filename,
+ File file) {
+ this.fileEntries.put(filename, file);
+
+ return this;
+ }
+
+ public ZipFileBuilder addManifest(String manifest) {
+ this.manifest = manifest;
+
+ return this;
+ }
+
+ public byte[] buildZip() throws IOException {
+ return makeZip();
+ }
+
+ private byte[] makeZip() throws IOException {
+ byte[] buffer = new byte[1024];
+
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ ZipOutputStream out = new ZipOutputStream(outputStream);
+
+ for (String documentKey : this.textEntries.keySet()) {
+ byte[] document = asBytes(this.textEntries.get(documentKey));
+ addZipEntry(documentKey + ".json", document, out, buffer);
+ }
+
+ addZipEntry("manifest.json", asBytes(manifest), out, buffer);
+ out.closeEntry();
+ out.close();
+ return outputStream.toByteArray();
+ }
+
+ private byte[] asBytes(String document) {
+ return document.getBytes(StandardCharsets.UTF_8);
+ }
+
+ private void addZipEntry(String filename,
+ byte[] document,
+ ZipOutputStream out,
+ byte[] buffer) throws IOException {
+ ZipEntry ze = new ZipEntry(filename);
+ out.putNextEntry(ze);
+
+ try (InputStream in = new ByteArrayInputStream(document)) {
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
new file mode 100644
index 000000000..be8695691
--- /dev/null
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export.resolver;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.streampipes.export.utils.SerializationUtils;
+import org.apache.streampipes.model.assets.AssetLink;
+import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.storage.api.INoSqlStorage;
+import org.apache.streampipes.storage.management.StorageDispatcher;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public abstract class AbstractResolver<T> {
+
+
+ public Set<ExportItem> resolve(Set<AssetLink> assetLinks) {
+ return assetLinks
+ .stream()
+ .map(link -> findDocument(link.getResourceId()))
+ .map(this::convert)
+ .collect(Collectors.toSet());
+ }
+
+ public String getSerializedDocument(String resourceId) throws JsonProcessingException {
+ return SerializationUtils.getSpObjectMapper().writeValueAsString(findDocument(resourceId));
+ }
+
+ protected INoSqlStorage getNoSqlStore() {
+ return StorageDispatcher.INSTANCE.getNoSqlStore();
+ }
+
+ public abstract T findDocument(String resourceId);
+
+ public abstract ExportItem convert(T document);
+}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
index 8fa1736b3..45ae530c4 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
@@ -16,24 +16,21 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
-public class DataLakeUtils {
+package org.apache.streampipes.export.resolver;
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
+import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.model.export.ExportItem;
+
+public class AdapterResolver extends AbstractResolver<AdapterDescription> {
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ @Override
+ public AdapterDescription findDocument(String resourceId) {
+ return getNoSqlStore().getAdapterInstanceStorage().getAdapter(resourceId);
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ @Override
+ public ExportItem convert(AdapterDescription document) {
+ return new ExportItem(document.getElementId(), document.getName(), true);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
new file mode 100644
index 000000000..3c5c1ffa6
--- /dev/null
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export.resolver;
+
+import org.apache.streampipes.model.dashboard.DashboardItem;
+import org.apache.streampipes.model.dashboard.DashboardModel;
+import org.apache.streampipes.model.export.ExportItem;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class DashboardResolver extends AbstractResolver<DashboardModel> {
+
+ @Override
+ public DashboardModel findDocument(String resourceId) {
+ return getNoSqlStore().getDashboardStorage().getDashboard(resourceId);
+ }
+
+ @Override
+ public ExportItem convert(DashboardModel document) {
+ return new ExportItem(document.getCouchDbId(), document.getName(), true);
+ }
+
+ public List<String> getWidgets(String resourceId) {
+ var document = findDocument(resourceId);
+ return document.getWidgets().stream().map(DashboardItem::getId).collect(Collectors.toList());
+ }
+}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java
index 8fa1736b3..7cf8c7381 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java
@@ -16,24 +16,20 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
+package org.apache.streampipes.export.resolver;
-public class DataLakeUtils {
+import org.apache.streampipes.model.dashboard.DashboardWidgetModel;
+import org.apache.streampipes.model.export.ExportItem;
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
+public class DashboardWidgetResolver extends AbstractResolver<DashboardWidgetModel> {
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ @Override
+ public DashboardWidgetModel findDocument(String resourceId) {
+ return getNoSqlStore().getDashboardWidgetStorage().getDashboardWidget(resourceId);
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ @Override
+ public ExportItem convert(DashboardWidgetModel document) {
+ return new ExportItem(document.getId(), document.getVisualizationName(), true);
}
}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
index 8fa1736b3..6f8083d96 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
@@ -16,24 +16,20 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
+package org.apache.streampipes.export.resolver;
-public class DataLakeUtils {
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.export.ExportItem;
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
+public class DataSourceResolver extends AbstractResolver<SpDataStream> {
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ @Override
+ public SpDataStream findDocument(String resourceId) {
+ return getNoSqlStore().getDataStreamStorage().getElementById(resourceId);
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ @Override
+ public ExportItem convert(SpDataStream document) {
+ return new ExportItem(document.getElementId(), document.getName(), true);
}
}
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewResolver.java
new file mode 100644
index 000000000..d5b501502
--- /dev/null
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewResolver.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+
+package org.apache.streampipes.export.resolver;
+
+import org.apache.streampipes.model.dashboard.DashboardItem;
+import org.apache.streampipes.model.dashboard.DashboardModel;
+import org.apache.streampipes.model.export.ExportItem;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class DataViewResolver extends AbstractResolver<DashboardModel> {
+
+ @Override
+ public DashboardModel findDocument(String resourceId) {
+ return getNoSqlStore().getDataExplorerDashboardStorage().getDashboard(resourceId);
+ }
+
+ @Override
+ public ExportItem convert(DashboardModel document) {
+ return new ExportItem(document.getCouchDbId(), document.getName(), true);
+ }
+
+ public List<String> getWidgets(String resourceId) {
+ var document = findDocument(resourceId);
+ return document.getWidgets().stream().map(DashboardItem::getId).collect(Collectors.toList());
+ }
+}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java
index 8fa1736b3..bdeb55031 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java
@@ -16,24 +16,20 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
+package org.apache.streampipes.export.resolver;
-public class DataLakeUtils {
+import org.apache.streampipes.model.datalake.DataExplorerWidgetModel;
+import org.apache.streampipes.model.export.ExportItem;
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
+public class DataViewWidgetResolver extends AbstractResolver<DataExplorerWidgetModel> {
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ @Override
+ public DataExplorerWidgetModel findDocument(String resourceId) {
+ return getNoSqlStore().getDataExplorerWidgetStorage().getDataExplorerWidget(resourceId);
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ @Override
+ public ExportItem convert(DataExplorerWidgetModel document) {
+ return new ExportItem(document.getId(), document.getWidgetId(), true);
}
}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
index 8fa1736b3..da155d86c 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
@@ -16,24 +16,20 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
+package org.apache.streampipes.export.resolver;
-public class DataLakeUtils {
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.export.ExportItem;
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
+public class MeasurementResolver extends AbstractResolver<DataLakeMeasure> {
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ @Override
+ public DataLakeMeasure findDocument(String resourceId) {
+ return getNoSqlStore().getDataLakeStorage().findOne(resourceId);
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ @Override
+ public ExportItem convert(DataLakeMeasure document) {
+ return new ExportItem(document.getElementId(), document.getMeasureName(), true);
}
}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
index 8fa1736b3..93a77acf8 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
@@ -16,24 +16,20 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
+package org.apache.streampipes.export.resolver;
-public class DataLakeUtils {
+import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.model.pipeline.Pipeline;
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
+public class PipelineResolver extends AbstractResolver<Pipeline> {
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ @Override
+ public Pipeline findDocument(String resourceId) {
+ return getNoSqlStore().getPipelineStorageAPI().getPipeline(resourceId);
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ @Override
+ public ExportItem convert(Pipeline document) {
+ return new ExportItem(document.getPipelineId(), document.getName(), true);
}
}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/SerializationUtils.java
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/SerializationUtils.java
index 8fa1736b3..44eae525e 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/SerializationUtils.java
@@ -16,24 +16,22 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
+package org.apache.streampipes.export.utils;
-public class DataLakeUtils {
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streampipes.serializers.json.JacksonSerializer;
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
+public class SerializationUtils {
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ public static ObjectMapper getSpObjectMapper() {
+ return JacksonSerializer.getObjectMapper();
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ public static ObjectMapper getDefaultObjectMapper() {
+ var mapper = new ObjectMapper();
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ return mapper;
}
}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
index 8fa1736b3..e74605421 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
@@ -21,7 +21,8 @@ package org.apache.streampipes.sinks.internal.jvm.datalake;
public class DataLakeUtils {
public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
+ //return s.toLowerCase().replaceAll(" ", "_");
+ return s.replaceAll(" ", "_");
}
private static String renameReservedKeywords(String runtimeName) {
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLink.java b/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLink.java
index b04dee4bc..74d15f002 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLink.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLink.java
@@ -18,11 +18,14 @@
package org.apache.streampipes.model.assets;
+import java.util.Objects;
+
public class AssetLink {
private String resourceId;
private String linkType;
private String linkLabel;
+ private String queryHint;
private boolean editingDisabled;
public AssetLink() {
@@ -59,4 +62,25 @@ public class AssetLink {
public void setEditingDisabled(boolean editingDisabled) {
this.editingDisabled = editingDisabled;
}
+
+ public String getQueryHint() {
+ return queryHint;
+ }
+
+ public void setQueryHint(String queryHint) {
+ this.queryHint = queryHint;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ AssetLink assetLink = (AssetLink) o;
+ return resourceId.equals(assetLink.resourceId) && queryHint.equals(assetLink.queryHint);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(resourceId, queryHint);
+ }
}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/export/AssetExportConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/export/AssetExportConfiguration.java
new file mode 100644
index 000000000..9e17d42be
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/export/AssetExportConfiguration.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.export;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class AssetExportConfiguration {
+
+ private String assetId;
+ private String assetName;
+
+ private Set<ExportItem> adapters;
+ private Set<ExportItem> dashboards;
+ private Set<ExportItem> dataViews;
+ private Set<ExportItem> dataLakeMeasures;
+ private Set<ExportItem> dataSources;
+ private Set<ExportItem> pipelines;
+ private Set<ExportItem> files;
+
+ public AssetExportConfiguration() {
+ this.adapters = new HashSet<>();
+ this.dashboards = new HashSet<>();
+ this.dataViews = new HashSet<>();
+ this.dataLakeMeasures = new HashSet<>();
+ this.dataSources = new HashSet<>();
+ this.pipelines = new HashSet<>();
+ this.files = new HashSet<>();
+ }
+
+ public Set<ExportItem> getAdapters() {
+ return adapters;
+ }
+
+ public void setAdapters(Set<ExportItem> adapters) {
+ this.adapters = adapters;
+ }
+
+ public Set<ExportItem> getDashboards() {
+ return dashboards;
+ }
+
+ public void setDashboards(Set<ExportItem> dashboards) {
+ this.dashboards = dashboards;
+ }
+
+ public Set<ExportItem> getDataViews() {
+ return dataViews;
+ }
+
+ public void setDataViews(Set<ExportItem> dataViews) {
+ this.dataViews = dataViews;
+ }
+
+ public Set<ExportItem> getDataLakeMeasures() {
+ return dataLakeMeasures;
+ }
+
+ public void setDataLakeMeasures(Set<ExportItem> dataLakeMeasures) {
+ this.dataLakeMeasures = dataLakeMeasures;
+ }
+
+ public Set<ExportItem> getDataSources() {
+ return dataSources;
+ }
+
+ public void setDataSources(Set<ExportItem> dataSources) {
+ this.dataSources = dataSources;
+ }
+
+ public String getAssetId() {
+ return assetId;
+ }
+
+ public void setAssetId(String assetId) {
+ this.assetId = assetId;
+ }
+
+ public Set<ExportItem> getPipelines() {
+ return pipelines;
+ }
+
+ public void setPipelines(Set<ExportItem> pipelines) {
+ this.pipelines = pipelines;
+ }
+
+ public Set<ExportItem> getFiles() {
+ return files;
+ }
+
+ public void setFiles(Set<ExportItem> files) {
+ this.files = files;
+ }
+
+ public String getAssetName() {
+ return assetName;
+ }
+
+ public void setAssetName(String assetName) {
+ this.assetName = assetName;
+ }
+
+
+}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportConfiguration.java
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportConfiguration.java
index 8fa1736b3..2a0052ba1 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportConfiguration.java
@@ -16,24 +16,28 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
-public class DataLakeUtils {
+package org.apache.streampipes.model.export;
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
+import org.apache.streampipes.model.shared.annotation.TsModel;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@TsModel
+public class ExportConfiguration {
+
+ private List<AssetExportConfiguration> assetExportConfiguration;
+
+ public ExportConfiguration() {
+ this.assetExportConfiguration = new ArrayList<>();
}
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ public List<AssetExportConfiguration> getAssetExportConfiguration() {
+ return assetExportConfiguration;
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ public void setAssetExportConfiguration(List<AssetExportConfiguration> assetExportConfiguration) {
+ this.assetExportConfiguration = assetExportConfiguration;
}
}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLink.java b/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
similarity index 62%
copy from streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLink.java
copy to streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
index b04dee4bc..cb62711a0 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLink.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
@@ -16,16 +16,21 @@
*
*/
-package org.apache.streampipes.model.assets;
+package org.apache.streampipes.model.export;
-public class AssetLink {
+public class ExportItem {
private String resourceId;
- private String linkType;
- private String linkLabel;
- private boolean editingDisabled;
+ private String label;
+ private boolean selected;
- public AssetLink() {
+ public ExportItem() {
+ }
+
+ public ExportItem(String resourceId, String label, boolean selected) {
+ this.resourceId = resourceId;
+ this.label = label;
+ this.selected = selected;
}
public String getResourceId() {
@@ -36,27 +41,19 @@ public class AssetLink {
this.resourceId = resourceId;
}
- public String getLinkType() {
- return linkType;
- }
-
- public void setLinkType(String linkType) {
- this.linkType = linkType;
- }
-
- public String getLinkLabel() {
- return linkLabel;
+ public String getLabel() {
+ return label;
}
- public void setLinkLabel(String linkLabel) {
- this.linkLabel = linkLabel;
+ public void setLabel(String label) {
+ this.label = label;
}
- public boolean isEditingDisabled() {
- return editingDisabled;
+ public boolean isSelected() {
+ return selected;
}
- public void setEditingDisabled(boolean editingDisabled) {
- this.editingDisabled = editingDisabled;
+ public void setSelected(boolean selected) {
+ this.selected = selected;
}
}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/export/StreamPipesApplicationPackage.java b/streampipes-model/src/main/java/org/apache/streampipes/model/export/StreamPipesApplicationPackage.java
new file mode 100644
index 000000000..c0b3f9e1b
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/export/StreamPipesApplicationPackage.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.export;
+
+import org.apache.streampipes.model.shared.annotation.TsModel;
+
+import java.util.HashSet;
+import java.util.Set;
+
+@TsModel
+public class StreamPipesApplicationPackage {
+
+ private Set<String> requiredProcessorAppIds;
+ private Set<String> requiredDataSinkAppIds;
+ private Set<String> requiredAdapterAppIds;
+
+ private Set<String> adapters;
+ private Set<String> dashboards;
+ private Set<String> dashboardWidgets;
+ private Set<String> dataViews;
+ private Set<String> dataViewWidgets;
+ private Set<String> dataLakeMeasures;
+ private Set<String> dataSources;
+ private Set<String> pipelines;
+ private Set<String> files;
+
+ public StreamPipesApplicationPackage() {
+ this.requiredProcessorAppIds = new HashSet<>();
+ this.requiredDataSinkAppIds = new HashSet<>();
+ this.requiredAdapterAppIds = new HashSet<>();
+
+ this.adapters = new HashSet<>();
+ this.dashboards = new HashSet<>();
+ this.dashboardWidgets = new HashSet<>();
+ this.dataViews = new HashSet<>();
+ this.dataViewWidgets = new HashSet<>();
+ this.dataLakeMeasures = new HashSet<>();
+ this.dataSources = new HashSet<>();
+ this.pipelines = new HashSet<>();
+ this.files = new HashSet<>();
+ }
+
+ public Set<String> getRequiredProcessorAppIds() {
+ return requiredProcessorAppIds;
+ }
+
+ public void setRequiredProcessorAppIds(Set<String> requiredProcessorAppIds) {
+ this.requiredProcessorAppIds = requiredProcessorAppIds;
+ }
+
+ public Set<String> getRequiredDataSinkAppIds() {
+ return requiredDataSinkAppIds;
+ }
+
+ public void setRequiredDataSinkAppIds(Set<String> requiredDataSinkAppIds) {
+ this.requiredDataSinkAppIds = requiredDataSinkAppIds;
+ }
+
+ public Set<String> getRequiredAdapterAppIds() {
+ return requiredAdapterAppIds;
+ }
+
+ public void setRequiredAdapterAppIds(Set<String> requiredAdapterAppIds) {
+ this.requiredAdapterAppIds = requiredAdapterAppIds;
+ }
+
+ public Set<String> getAdapters() {
+ return adapters;
+ }
+
+ public void setAdapters(Set<String> adapters) {
+ this.adapters = adapters;
+ }
+
+ public void addAdapter(String adapter) {
+ this.adapters.add(adapter);
+ }
+
+ public Set<String> getDashboards() {
+ return dashboards;
+ }
+
+ public void setDashboards(Set<String> dashboards) {
+ this.dashboards = dashboards;
+ }
+
+ public void addDashboard(String dashboard) {
+ this.dashboards.add(dashboard);
+ }
+
+ public Set<String> getDashboardWidgets() {
+ return dashboardWidgets;
+ }
+
+ public void setDashboardWidgets(Set<String> dashboardWidgets) {
+ this.dashboardWidgets = dashboardWidgets;
+ }
+
+ public void addDashboardWidget(String dashboardWidget) {
+ this.dashboardWidgets.add(dashboardWidget);
+ }
+
+ public Set<String> getDataViews() {
+ return dataViews;
+ }
+
+ public void setDataViews(Set<String> dataViews) {
+ this.dataViews = dataViews;
+ }
+
+ public void addDataView(String dataView) {
+ this.dataViews.add(dataView);
+ }
+
+ public Set<String> getDataViewWidgets() {
+ return dataViewWidgets;
+ }
+
+ public void setDataViewWidgets(Set<String> dataViewWidgets) {
+ this.dataViewWidgets = dataViewWidgets;
+ }
+
+ public void addDataViewWidget(String dataViewWidget) {
+ this.dataViewWidgets.add(dataViewWidget);
+ }
+
+ public Set<String> getDataLakeMeasures() {
+ return dataLakeMeasures;
+ }
+
+ public void setDataLakeMeasures(Set<String> dataLakeMeasures) {
+ this.dataLakeMeasures = dataLakeMeasures;
+ }
+
+ public void addDataLakeMeasure(String dataLakeMeasure) {
+ this.dataLakeMeasures.add(dataLakeMeasure);
+ }
+
+ public Set<String> getDataSources() {
+ return dataSources;
+ }
+
+ public void setDataSources(Set<String> dataSources) {
+ this.dataSources = dataSources;
+ }
+
+ public void addDataSource(String dataSource) {
+ this.dataSources.add(dataSource);
+ }
+
+ public Set<String> getPipelines() {
+ return pipelines;
+ }
+
+ public void setPipelines(Set<String> pipelines) {
+ this.pipelines = pipelines;
+ }
+
+ public void addPipeline(String pipeline) {
+ this.pipelines.add(pipeline);
+ }
+
+ public Set<String> getFiles() {
+ return files;
+ }
+
+ public void setFiles(Set<String> files) {
+ this.files = files;
+ }
+
+ public void addFile(String file) {
+ this.files.add(file);
+ }
+}
diff --git a/streampipes-rest/pom.xml b/streampipes-rest/pom.xml
index 8cedd5b4d..8f2db93f4 100644
--- a/streampipes-rest/pom.xml
+++ b/streampipes-rest/pom.xml
@@ -50,6 +50,11 @@
<artifactId>streampipes-data-explorer</artifactId>
<version>0.70.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-data-export</artifactId>
+ <version>0.70.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-measurement-units</artifactId>
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/DataExportResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/DataExportResource.java
new file mode 100644
index 000000000..fff79451b
--- /dev/null
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/DataExportResource.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.admin;
+
+import org.apache.streampipes.export.ExportManager;
+import org.apache.streampipes.model.export.ExportConfiguration;
+import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
+import org.apache.streampipes.rest.security.AuthConstants;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.List;
+
+@Path("/v2/export")
+@Component
+@PreAuthorize(AuthConstants.IS_ADMIN_ROLE)
+public class DataExportResource extends AbstractAuthGuardedRestResource {
+
+ @Path("/preview")
+ @POST
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getExportPreview(List<String> selectedAssetIds) {
+ var exportConfig = ExportManager.getExportPreview(selectedAssetIds);
+ return ok(exportConfig);
+ }
+
+ @Path("/download")
+ @POST
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ public Response getExportPreview(ExportConfiguration exportConfiguration) throws IOException {
+ var applicationPackage = ExportManager.getExportPackage(exportConfiguration);
+ return ok(applicationPackage);
+ }
+
+}
diff --git a/ui/projects/streampipes/platform-services/src/lib/model/assets/asset.model.ts b/ui/projects/streampipes/platform-services/src/lib/model/assets/asset.model.ts
index 511062885..626e627dd 100644
--- a/ui/projects/streampipes/platform-services/src/lib/model/assets/asset.model.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/model/assets/asset.model.ts
@@ -37,6 +37,7 @@ export interface AssetLink {
resourceId: string;
linkType: 'data-view' | 'dashboard' | 'adapter' | 'source' | string;
linkLabel: string;
+ queryHint: string;
editingDisabled: boolean;
navigationActive: boolean;
}
diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
index 393aa4c56..9a1a1d9b2 100644
--- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
@@ -18,7 +18,7 @@
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
-// Generated using typescript-generator version 2.27.744 on 2022-04-25 13:56:39.
+// Generated using typescript-generator version 2.27.744 on 2022-07-31 20:23:49.
export class AbstractStreamPipesEntity {
"@class": "org.apache.streampipes.model.base.AbstractStreamPipesEntity" | "org.apache.streampipes.model.base.NamedStreamPipesEntity" | "org.apache.streampipes.model.connect.adapter.AdapterDescription" | "org.apache.streampipes.model.connect.adapter.AdapterSetDescription" | "org.apache.streampipes.model.connect.adapter.GenericAdapterSetDescription" | "org.apache.streampipes.model.connect.adapter.SpecificAdapterSetDescription" | "org.apache.streampipes.model.connect.adapter.AdapterStre [...]
@@ -151,8 +151,8 @@ export class NamedStreamPipesEntity extends AbstractStreamPipesEntity {
instance.applicationLinks = __getCopyArrayFn(ApplicationLink.fromData)(data.applicationLinks);
instance.internallyManaged = data.internallyManaged;
instance.connectedTo = __getCopyArrayFn(__identity<string>())(data.connectedTo);
- instance.uri = data.uri;
instance.dom = data.dom;
+ instance.uri = data.uri;
instance._rev = data._rev;
return instance;
}
@@ -169,11 +169,11 @@ export class AdapterDescription extends NamedStreamPipesEntity {
eventGrounding: EventGrounding;
icon: string;
rules: TransformationRuleDescriptionUnion[];
- schemaRules: any[];
+ schemaRules: TransformationRuleDescriptionUnion[];
selectedEndpointUrl: string;
- streamRules: any[];
+ streamRules: TransformationRuleDescriptionUnion[];
userName: string;
- valueRules: any[];
+ valueRules: TransformationRuleDescriptionUnion[];
static fromData(data: AdapterDescription, target?: AdapterDescription): AdapterDescription {
if (!data) {
@@ -192,9 +192,9 @@ export class AdapterDescription extends NamedStreamPipesEntity {
instance.selectedEndpointUrl = data.selectedEndpointUrl;
instance.correspondingServiceGroup = data.correspondingServiceGroup;
instance.correspondingDataStreamElementId = data.correspondingDataStreamElementId;
- instance.valueRules = __getCopyArrayFn(__identity<any>())(data.valueRules);
- instance.streamRules = __getCopyArrayFn(__identity<any>())(data.streamRules);
- instance.schemaRules = __getCopyArrayFn(__identity<any>())(data.schemaRules);
+ instance.streamRules = __getCopyArrayFn(TransformationRuleDescription.fromDataUnion)(data.streamRules);
+ instance.valueRules = __getCopyArrayFn(TransformationRuleDescription.fromDataUnion)(data.valueRules);
+ instance.schemaRules = __getCopyArrayFn(TransformationRuleDescription.fromDataUnion)(data.schemaRules);
return instance;
}
@@ -588,6 +588,35 @@ export class ApplicationLink extends UnnamedStreamPipesEntity {
}
}
+export class AssetExportConfiguration {
+ adapters: ExportItem[];
+ assetId: string;
+ assetName: string;
+ dashboards: ExportItem[];
+ dataLakeMeasures: ExportItem[];
+ dataSources: ExportItem[];
+ dataViews: ExportItem[];
+ files: ExportItem[];
+ pipelines: ExportItem[];
+
+ static fromData(data: AssetExportConfiguration, target?: AssetExportConfiguration): AssetExportConfiguration {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new AssetExportConfiguration();
+ instance.assetId = data.assetId;
+ instance.assetName = data.assetName;
+ instance.adapters = __getCopyArrayFn(ExportItem.fromData)(data.adapters);
+ instance.dashboards = __getCopyArrayFn(ExportItem.fromData)(data.dashboards);
+ instance.dataViews = __getCopyArrayFn(ExportItem.fromData)(data.dataViews);
+ instance.dataLakeMeasures = __getCopyArrayFn(ExportItem.fromData)(data.dataLakeMeasures);
+ instance.dataSources = __getCopyArrayFn(ExportItem.fromData)(data.dataSources);
+ instance.pipelines = __getCopyArrayFn(ExportItem.fromData)(data.pipelines);
+ instance.files = __getCopyArrayFn(ExportItem.fromData)(data.files);
+ return instance;
+ }
+}
+
export class BoundPipelineElement extends UnnamedStreamPipesEntity {
"@class": "org.apache.streampipes.model.template.BoundPipelineElement";
connectedTo: BoundPipelineElement[];
@@ -1525,6 +1554,36 @@ export class EventStreamQualityRequirement extends UnnamedStreamPipesEntity {
}
}
+export class ExportConfiguration {
+ assetExportConfiguration: AssetExportConfiguration[];
+
+ static fromData(data: ExportConfiguration, target?: ExportConfiguration): ExportConfiguration {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new ExportConfiguration();
+ instance.assetExportConfiguration = __getCopyArrayFn(AssetExportConfiguration.fromData)(data.assetExportConfiguration);
+ return instance;
+ }
+}
+
+export class ExportItem {
+ label: string;
+ resourceId: string;
+ selected: boolean;
+
+ static fromData(data: ExportItem, target?: ExportItem): ExportItem {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new ExportItem();
+ instance.resourceId = data.resourceId;
+ instance.label = data.label;
+ instance.selected = data.selected;
+ return instance;
+ }
+}
+
export class FileMetadata {
createdAt: number;
createdByUser: string;
@@ -1668,9 +1727,9 @@ export class GenericAdapterSetDescription extends AdapterSetDescription implemen
}
const instance = target || new GenericAdapterSetDescription();
super.fromData(data, instance);
- instance.formatDescription = FormatDescription.fromData(data.formatDescription);
- instance.protocolDescription = ProtocolDescription.fromData(data.protocolDescription);
instance.eventSchema = EventSchema.fromData(data.eventSchema);
+ instance.protocolDescription = ProtocolDescription.fromData(data.protocolDescription);
+ instance.formatDescription = FormatDescription.fromData(data.formatDescription);
return instance;
}
}
@@ -1687,9 +1746,9 @@ export class GenericAdapterStreamDescription extends AdapterStreamDescription im
}
const instance = target || new GenericAdapterStreamDescription();
super.fromData(data, instance);
- instance.formatDescription = FormatDescription.fromData(data.formatDescription);
- instance.protocolDescription = ProtocolDescription.fromData(data.protocolDescription);
instance.eventSchema = EventSchema.fromData(data.eventSchema);
+ instance.protocolDescription = ProtocolDescription.fromData(data.protocolDescription);
+ instance.formatDescription = FormatDescription.fromData(data.formatDescription);
return instance;
}
}
@@ -3007,6 +3066,25 @@ export class StaticPropertyGroup extends StaticProperty {
}
}
+export class StreamPipesApplicationPackage {
+ assetExportConfigurations: AssetExportConfiguration[];
+ requiredAdapterAppIds: string[];
+ requiredDataSinkAppIds: string[];
+ requiredProcessorAppIds: string[];
+
+ static fromData(data: StreamPipesApplicationPackage, target?: StreamPipesApplicationPackage): StreamPipesApplicationPackage {
+ if (!data) {
+ return data;
+ }
+ const instance = target || new StreamPipesApplicationPackage();
+ instance.requiredProcessorAppIds = __getCopyArrayFn(__identity<string>())(data.requiredProcessorAppIds);
+ instance.requiredDataSinkAppIds = __getCopyArrayFn(__identity<string>())(data.requiredDataSinkAppIds);
+ instance.requiredAdapterAppIds = __getCopyArrayFn(__identity<string>())(data.requiredAdapterAppIds);
+ instance.assetExportConfigurations = __getCopyArrayFn(AssetExportConfiguration.fromData)(data.assetExportConfigurations);
+ return instance;
+ }
+}
+
export class SuccessMessage extends Message {
static fromData(data: SuccessMessage, target?: SuccessMessage): SuccessMessage {
diff --git a/ui/src/app/assets/components/asset-details/asset-details-panel/asset-details-panel.component.ts b/ui/src/app/assets/components/asset-details/asset-details-panel/asset-details-panel.component.ts
index f349c3e8d..6d8e3d66b 100644
--- a/ui/src/app/assets/components/asset-details/asset-details-panel/asset-details-panel.component.ts
+++ b/ui/src/app/assets/components/asset-details/asset-details-panel/asset-details-panel.component.ts
@@ -78,7 +78,14 @@ export class SpAssetDetailsPanelComponent implements OnInit {
}
openCreateAssetLinkDialog(): void {
- const assetLink: AssetLink = {linkLabel: '', linkType: 'data-view', editingDisabled: false, resourceId: '', navigationActive: true};
+ const assetLink: AssetLink = {
+ linkLabel: '',
+ linkType: 'data-view',
+ editingDisabled: false,
+ resourceId: '',
+ navigationActive: true,
+ queryHint: 'data-view'
+ };
this.openEditAssetLinkDialog(assetLink, -1, true);
}
diff --git a/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.html b/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.html
index b723edd0e..480c4b6da 100644
--- a/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.html
+++ b/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.html
@@ -77,6 +77,18 @@
</mat-select>
</mat-form-field>
</div>
+ <div *ngIf="selectedLinkType.linkQueryHint === 'adapter'" fxLayout="column" class="link-configuration">
+ <mat-form-field color="accent" fxFlex="100">
+ <mat-label>Data Views</mat-label>
+ <mat-select (selectionChange)="changeLabel($event.value.elementId, $event.value.name, $event.value)"
+ [(value)]="currentResource"
+ fxFlex
+ required>
+ <mat-option *ngFor="let adapter of adapters"
+ [value]="adapter">{{adapter.name}}</mat-option>
+ </mat-select>
+ </mat-form-field>
+ </div>
<div *ngIf="selectedLinkType.linkQueryHint === 'measurement'" fxLayout="column" class="link-configuration">
<mat-form-field color="accent" fxFlex="100">
<mat-label>Data Lake Storage</mat-label>
diff --git a/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.ts b/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.ts
index ee42b9f4f..a989a7f64 100644
--- a/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.ts
+++ b/ui/src/app/assets/dialog/edit-asset-link/edit-asset-link-dialog.component.ts
@@ -19,6 +19,9 @@
import { Component, Input, OnInit } from '@angular/core';
import { DialogRef } from '@streampipes/shared-ui';
import {
+ AdapterDescriptionUnion,
+ SpDataStream,
+ AdapterService,
AssetLink,
AssetLinkType,
Dashboard,
@@ -27,16 +30,12 @@ import {
DataViewDataExplorerService,
GenericStorageService,
Pipeline,
- PipelineService
+ PipelineService,
+ PipelineElementService
} from '@streampipes/platform-services';
import { FormGroup } from '@angular/forms';
import { zip } from 'rxjs';
import { MatSelectChange } from '@angular/material/select';
-import {
- AdapterDescriptionUnion,
- SpDataStream
-} from '../../../../../projects/streampipes/platform-services/src/lib/model/gen/streampipes-model';
-import { PipelineElementService } from '../../../../../projects/streampipes/platform-services/src/lib/apis/pipeline-element.service';
@Component({
selector: 'sp-edit-asset-link-dialog-component',
@@ -77,7 +76,8 @@ export class EditAssetLinkDialogComponent implements OnInit {
private dataViewService: DataViewDataExplorerService,
private dashboardService: DashboardService,
private dataLakeService: DatalakeRestService,
- private pipelineElementService: PipelineElementService) {
+ private pipelineElementService: PipelineElementService,
+ private adapterService: AdapterService) {
}
ngOnInit(): void {
@@ -105,19 +105,22 @@ export class EditAssetLinkDialogComponent implements OnInit {
this.dataViewService.getDataViews(),
this.dashboardService.getDashboards(),
this.pipelineElementService.getDataStreams(),
- this.dataLakeService.getAllMeasurementSeries()).subscribe(response => {
+ this.dataLakeService.getAllMeasurementSeries(),
+ this.adapterService.getAdapters()).subscribe(response => {
this.pipelines = response[0];
this.dataViews = response[1];
this.dashboards = response[2];
this.dataSources = response[3];
this.dataLakeMeasures = response[4];
+ this.adapters = response[5];
this.allResources = [
...this.pipelines,
...this.dataViews,
...this.dashboards,
...this.dataSources,
- ...this.dataLakeMeasures
+ ...this.dataLakeMeasures,
+ ...this.adapters
];
if (!this.createMode) {
this.currentResource = this.allResources.find(r => r._id === this.clonedAssetLink.resourceId ||
@@ -131,6 +134,7 @@ export class EditAssetLinkDialogComponent implements OnInit {
const linkType = this.assetLinkTypes.find(a => a.linkType === this.selectedLinkType.linkType);
this.clonedAssetLink.editingDisabled = false;
this.clonedAssetLink.linkType = linkType.linkType;
+ this.clonedAssetLink.queryHint = linkType.linkQueryHint;
this.clonedAssetLink.navigationActive = linkType.navigationActive;
}
diff --git a/ui/src/app/configuration/configuration-tabs.ts b/ui/src/app/configuration/configuration-tabs.ts
index dba3af875..367d7a927 100644
--- a/ui/src/app/configuration/configuration-tabs.ts
+++ b/ui/src/app/configuration/configuration-tabs.ts
@@ -25,6 +25,7 @@ export class SpConfigurationTabs {
{itemId: 'general', itemTitle: 'General', itemLink: ['configuration', 'general']},
{itemId: 'datalake', itemTitle: 'Data Lake', itemLink: ['configuration', 'datalake']},
{itemId: 'email', itemTitle: 'Mail', itemLink: ['configuration', 'email']},
+ {itemId: 'export', itemTitle: 'Export/Import', itemLink: ['configuration', 'export']},
{itemId: 'messaging', itemTitle: 'Messaging', itemLink: ['configuration', 'messaging']},
{itemId: 'pipelineelement', itemTitle: 'Pipeline Element Configuration', itemLink: ['configuration', 'pipelineelement']},
{itemId: 'security', itemTitle: 'Security', itemLink: ['configuration', 'security']}
diff --git a/ui/src/app/configuration/configuration.module.ts b/ui/src/app/configuration/configuration.module.ts
index 9ca80b6fc..1aa5060b0 100644
--- a/ui/src/app/configuration/configuration.module.ts
+++ b/ui/src/app/configuration/configuration.module.ts
@@ -54,6 +54,10 @@ import { GeneralConfigurationComponent } from './general-configuration/general-c
import { SecurityAuthenticationConfigurationComponent } from './security-configuration/authentication-configuration/authentication-configuration.component';
import { RouterModule } from '@angular/router';
import { SharedUiModule } from '@streampipes/shared-ui';
+import { SpDataExportImportComponent } from './export/data-export-import.component';
+import { SpDataExportDialogComponent } from './export/export-dialog/data-export-dialog.component';
+import { SpDataImportDialogComponent } from './export/import-dialog/data-import-dialog.component';
+import { SpDataExportItemComponent } from './export/export-dialog/data-export-item/data-export-item.component';
@NgModule({
imports: [
@@ -94,6 +98,10 @@ import { SharedUiModule } from '@streampipes/shared-ui';
path: 'email',
component: EmailConfigurationComponent
},
+ {
+ path: 'export',
+ component: SpDataExportImportComponent
+ },
{
path: 'messaging',
component: MessagingConfigurationComponent
@@ -131,6 +139,10 @@ import { SharedUiModule } from '@streampipes/shared-ui';
SecurityServiceConfigComponent,
MessagingConfigurationComponent,
DatalakeConfigurationComponent,
+ SpDataExportImportComponent,
+ SpDataExportDialogComponent,
+ SpDataExportItemComponent,
+ SpDataImportDialogComponent
],
providers: [ConfigurationService],
})
diff --git a/ui/src/app/configuration/export/data-export-import.component.html b/ui/src/app/configuration/export/data-export-import.component.html
new file mode 100644
index 000000000..82a86a161
--- /dev/null
+++ b/ui/src/app/configuration/export/data-export-import.component.html
@@ -0,0 +1,52 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<sp-basic-nav-tabs [spNavigationItems]="tabs" [activeLink]="'export'">
+ <div fxLayout="column" class="page-container-padding">
+ <div fxFlex="100" fxLayout="column" fxLayoutAlign="start start">
+ <sp-split-section title="Export" subtitle="Export application data">
+ <div class="subsection-title">Export assets and all linked resources</div>
+ <div fxLayout="column" *ngFor="let asset of assets">
+ <mat-checkbox (change)="handleSelectionChange($event, asset._id)"><h4>{{asset.assetName}}</h4></mat-checkbox>
+ </div>
+ <div class="mt-10">
+ <button mat-button
+ mat-raised-button
+ color="accent"
+ [disabled]="selectedAssets.length === 0"
+ (click)="openExportDialog()">
+ <i class="material-icons">file_download</i><span> Start export process</span>
+ </button>
+ </div>
+ </sp-split-section>
+ </div>
+ <mat-divider></mat-divider>
+ <div fxFlex="100" fxLayout="column" fxLayoutAlign="start start">
+ <sp-split-section title="Import" subtitle="Import application data">
+ <div class="subsection-title">Import from application package</div>
+
+ <div class="mt-10">
+ <button mat-button mat-raised-button color="accent">
+ <i class="material-icons">save</i><span> Save</span>
+ </button>
+ </div>
+ </sp-split-section>
+ </div>
+ </div>
+</sp-basic-nav-tabs>
+
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/ui/src/app/configuration/export/data-export-import.component.scss
similarity index 55%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to ui/src/app/configuration/export/data-export-import.component.scss
index 8fa1736b3..13cbc4aac 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/ui/src/app/configuration/export/data-export-import.component.scss
@@ -15,25 +15,3 @@
* limitations under the License.
*
*/
-
-package org.apache.streampipes.sinks.internal.jvm.datalake;
-
-public class DataLakeUtils {
-
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
-
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
- }
-
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
- }
-}
diff --git a/ui/src/app/configuration/export/data-export-import.component.ts b/ui/src/app/configuration/export/data-export-import.component.ts
new file mode 100644
index 000000000..32b007e2b
--- /dev/null
+++ b/ui/src/app/configuration/export/data-export-import.component.ts
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, OnInit } from '@angular/core';
+import { DialogService, PanelType, SpBreadcrumbService } from '@streampipes/shared-ui';
+import { SpConfigurationRoutes } from '../configuration.routes';
+import { SpConfigurationTabs } from '../configuration-tabs';
+import { AssetManagementService, SpAsset } from '@streampipes/platform-services';
+import { MatCheckboxChange } from '@angular/material/checkbox';
+import { SpDataExportDialogComponent } from './export-dialog/data-export-dialog.component';
+
+@Component({
+ selector: 'sp-data-export-import',
+ templateUrl: './data-export-import.component.html',
+ styleUrls: ['./data-export-import.component.scss']
+})
+export class SpDataExportImportComponent implements OnInit {
+
+ tabs = SpConfigurationTabs.getTabs();
+
+ assets: SpAsset[];
+ selectedAssets: string[] = [];
+
+ constructor(private breadcrumbService: SpBreadcrumbService,
+ private assetManagementService: AssetManagementService,
+ private dialogService: DialogService) {
+
+ }
+
+ ngOnInit(): void {
+ this.breadcrumbService
+ .updateBreadcrumb([SpConfigurationRoutes.BASE, {label: SpConfigurationTabs.getTabs()[3].itemTitle}]);
+ this.loadAssets();
+ }
+
+ loadAssets(): void {
+ this.assetManagementService
+ .getAllAssets()
+ .subscribe(assets => this.assets = assets.sort((a, b) => a.assetName.localeCompare(b.assetName)));
+ }
+
+ handleSelectionChange(event: MatCheckboxChange,
+ assetId: string) {
+ if (event.checked) {
+ this.selectedAssets.push(assetId);
+ } else {
+ this.selectedAssets.splice(this.selectedAssets.indexOf(assetId), 1);
+ }
+ }
+
+ openExportDialog() {
+ const dialogRef = this.dialogService.open(SpDataExportDialogComponent, {
+ panelType: PanelType.SLIDE_IN_PANEL,
+ title: 'Export resources',
+ width: '50vw',
+ data: {
+ 'selectedAssets': this.selectedAssets,
+ }
+ });
+
+ dialogRef.afterClosed().subscribe(() => {
+
+ });
+ }
+
+}
diff --git a/ui/src/app/configuration/export/data-export.service.ts b/ui/src/app/configuration/export/data-export.service.ts
new file mode 100644
index 000000000..14994fdb4
--- /dev/null
+++ b/ui/src/app/configuration/export/data-export.service.ts
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Injectable } from '@angular/core';
+import { PlatformServicesCommons } from '@streampipes/platform-services';
+import { HttpClient } from '@angular/common/http';
+import { Observable } from 'rxjs';
+import { ExportConfiguration } from '../../../../projects/streampipes/platform-services/src/lib/model/gen/streampipes-model';
+import { map } from 'rxjs/operators';
+
+@Injectable({providedIn: 'root'})
+export class DataExportService {
+
+ constructor(private platformServicesCommons: PlatformServicesCommons,
+ private http: HttpClient) {
+ }
+
+ getExportPreview(assetIds: string[]): Observable<ExportConfiguration> {
+ return this.http.post(this.basePath + '/preview', assetIds)
+ .pipe(map(res => res as ExportConfiguration));
+ }
+
+ triggerExport(exportConfig: ExportConfiguration): Observable<Blob> {
+ return this.http.post(this.basePath + '/download', exportConfig, {responseType: 'blob'});
+ }
+
+ private get basePath(): string {
+ return this.platformServicesCommons.apiBasePath + '/export';
+ }
+
+}
diff --git a/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.html b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.html
new file mode 100644
index 000000000..bec004e3f
--- /dev/null
+++ b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.html
@@ -0,0 +1,49 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<div class="sp-dialog-container">
+ <div class="sp-dialog-content">
+ <div fxFlex="100" fxLayout="column" class="p-15" *ngIf="preview">
+ <div *ngFor="let config of preview.assetExportConfiguration">
+ <h4>Exported items {{config.assetName}}</h4>
+ <sp-data-export-item [exportItems]="config.adapters" sectionTitle="Adapters"></sp-data-export-item>
+ <sp-data-export-item [exportItems]="config.dashboards" sectionTitle="Dashboards"></sp-data-export-item>
+ <sp-data-export-item [exportItems]="config.dataViews" sectionTitle="Data Views"></sp-data-export-item>
+ <sp-data-export-item [exportItems]="config.dataSources" sectionTitle="Data Sources"></sp-data-export-item>
+ <sp-data-export-item [exportItems]="config.dataLakeMeasures" sectionTitle="Data Lake Storage"></sp-data-export-item>
+ <sp-data-export-item [exportItems]="config.pipelines" sectionTitle="Pipelines"></sp-data-export-item>
+ </div>
+ </div>
+ </div>
+ <mat-divider></mat-divider>
+ <div class="sp-dialog-actions">
+ <div fxLayout="column">
+ <div fxLayout="row">
+ <button mat-button
+ mat-raised-button
+ color="accent"
+ style="margin-right: 10px;" (click)="generateDownloadPackage()">
+ <i class="material-icons">file_download</i><span> Download export</span>
+ </button>
+ <button mat-button mat-raised-button class="mat-basic" (click)="close()">
+ Cancel
+ </button>
+ </div>
+ </div>
+ </div>
+</div>
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.scss
similarity index 55%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to ui/src/app/configuration/export/export-dialog/data-export-dialog.component.scss
index 8fa1736b3..fddade7bf 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.scss
@@ -16,24 +16,4 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
-
-public class DataLakeUtils {
-
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
-
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
- }
-
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
- }
-}
+@import '../../../../scss/sp/sp-dialog.scss';
diff --git a/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.ts b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.ts
new file mode 100644
index 000000000..33af53433
--- /dev/null
+++ b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.ts
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, Input, OnInit } from '@angular/core';
+import { DialogRef } from '@streampipes/shared-ui';
+import { DataExportService } from '../data-export.service';
+import { ExportConfiguration } from '@streampipes/platform-services';
+
+@Component({
+ selector: 'sp-data-export-dialog',
+ templateUrl: './data-export-dialog.component.html',
+ styleUrls: ['./data-export-dialog.component.scss'],
+})
+export class SpDataExportDialogComponent implements OnInit {
+
+ @Input()
+ selectedAssets: string[];
+
+ preview: ExportConfiguration;
+
+ constructor(private dialogRef: DialogRef<SpDataExportDialogComponent>,
+ private dataExportService: DataExportService) {
+
+ }
+
+ ngOnInit(): void {
+ this.dataExportService.getExportPreview(this.selectedAssets).subscribe(preview => {
+ this.preview = preview;
+ });
+ }
+
+ close(): void {
+ this.dialogRef.close();
+ }
+
+ generateDownloadPackage(): void {
+ this.dataExportService.triggerExport(this.preview).subscribe(result => {
+ this.downloadFile(result);
+ });
+ }
+
+ downloadFile(data: any) {
+ const blob = new Blob([data], { type: 'application/zip' });
+ const url = window.URL.createObjectURL(blob);
+ window.open(url);
+ }
+
+}
diff --git a/ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.html b/ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.html
new file mode 100644
index 000000000..d9879aa15
--- /dev/null
+++ b/ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.html
@@ -0,0 +1,26 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<div fxLayout="column" fxFlex="100">
+ <h5>{{sectionTitle}}</h5>
+ <mat-checkbox *ngFor="let exportItem of exportItems; let i = index"
+ [checked]="exportItem.selected"
+ (change)="changeItem($event, i)">
+ {{exportItem.label}}
+ </mat-checkbox>
+</div>
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.scss
similarity index 55%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.scss
index 8fa1736b3..13cbc4aac 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.scss
@@ -15,25 +15,3 @@
* limitations under the License.
*
*/
-
-package org.apache.streampipes.sinks.internal.jvm.datalake;
-
-public class DataLakeUtils {
-
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
-
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
- }
-
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
- }
-}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.ts
similarity index 56%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.ts
index 8fa1736b3..1d6e68374 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.ts
@@ -16,24 +16,30 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
+import { Component, Input, OnInit } from '@angular/core';
+import { ExportItem } from '@streampipes/platform-services';
+import { MatCheckboxChange } from '@angular/material/checkbox';
-public class DataLakeUtils {
+@Component({
+ selector: 'sp-data-export-item',
+ templateUrl: './data-export-item.component.html',
+ styleUrls: ['./data-export-item.component.scss'],
+})
+export class SpDataExportItemComponent implements OnInit {
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
+ @Input()
+ exportItems: ExportItem[];
+
+ @Input()
+ sectionTitle: string;
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
+ ngOnInit(): void {
}
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
+ changeItem(event: MatCheckboxChange,
+ index: number) {
+ this.exportItems[index].selected = event.checked;
}
+
+
}
diff --git a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
new file mode 100644
index 000000000..fb99b649e
--- /dev/null
+++ b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
@@ -0,0 +1,17 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.scss
similarity index 55%
copy from streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
copy to ui/src/app/configuration/export/import-dialog/data-import-dialog.component.scss
index 8fa1736b3..fddade7bf 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeUtils.java
+++ b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.scss
@@ -16,24 +16,4 @@
*
*/
-package org.apache.streampipes.sinks.internal.jvm.datalake;
-
-public class DataLakeUtils {
-
- public static String prepareString(String s) {
- return s.toLowerCase().replaceAll(" ", "_");
- }
-
- private static String renameReservedKeywords(String runtimeName) {
- if (InfluxDbReservedKeywords.keywordList.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) {
- return runtimeName + "_";
- } else {
- return runtimeName;
- }
- }
-
- public static String sanitizePropertyRuntimeName(String runtimeName) {
- String sanitizedRuntimeName = prepareString(runtimeName);
- return renameReservedKeywords(sanitizedRuntimeName);
- }
-}
+@import '../../../../scss/sp/sp-dialog.scss';
diff --git a/ui/src/app/configuration/security-configuration/security-configuration.component.ts b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
similarity index 58%
copy from ui/src/app/configuration/security-configuration/security-configuration.component.ts
copy to ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
index ee896a53e..5ef2a7fb6 100644
--- a/ui/src/app/configuration/security-configuration/security-configuration.component.ts
+++ b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
@@ -17,23 +17,16 @@
*/
import { Component, OnInit } from '@angular/core';
-import { SpConfigurationTabs } from '../configuration-tabs';
-import { SpBreadcrumbService } from '@streampipes/shared-ui';
-import { SpConfigurationRoutes } from '../configuration.routes';
@Component({
- selector: 'sp-security-configuration',
- templateUrl: './security-configuration.component.html',
- styleUrls: ['./security-configuration.component.scss']
+ selector: 'sp-data-import-dialog',
+ templateUrl: './data-import-dialog.component.html',
+ styleUrls: ['./data-import-dialog.component.scss'],
})
-export class SecurityConfigurationComponent implements OnInit {
-
- tabs = SpConfigurationTabs.getTabs();
-
- constructor(private breadcrumbService: SpBreadcrumbService) {}
+export class SpDataImportDialogComponent implements OnInit {
ngOnInit(): void {
- this.breadcrumbService.updateBreadcrumb([SpConfigurationRoutes.BASE, {label: SpConfigurationTabs.getTabs()[5].itemTitle}]);
}
+
}
diff --git a/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.ts b/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.ts
index 0b68ae0c3..4d972b321 100644
--- a/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.ts
+++ b/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.ts
@@ -42,7 +42,7 @@ export class MessagingConfigurationComponent implements OnInit {
}
ngOnInit() {
- this.breadcrumbService.updateBreadcrumb([SpConfigurationRoutes.BASE, {label: SpConfigurationTabs.getTabs()[3].itemTitle}]);
+ this.breadcrumbService.updateBreadcrumb([SpConfigurationRoutes.BASE, {label: SpConfigurationTabs.getTabs()[4].itemTitle}]);
this.getMessagingSettings();
}
diff --git a/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.ts b/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.ts
index 7fb8d260a..0d28979a4 100644
--- a/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.ts
+++ b/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.ts
@@ -60,7 +60,7 @@ export class PipelineElementConfigurationComponent implements OnInit {
}
ngOnInit() {
- this.breadcrumbService.updateBreadcrumb([SpConfigurationRoutes.BASE, {label: SpConfigurationTabs.getTabs()[4].itemTitle}]);
+ this.breadcrumbService.updateBreadcrumb([SpConfigurationRoutes.BASE, {label: SpConfigurationTabs.getTabs()[5].itemTitle}]);
}
getConsulServices(): void {
diff --git a/ui/src/app/configuration/security-configuration/security-configuration.component.ts b/ui/src/app/configuration/security-configuration/security-configuration.component.ts
index ee896a53e..ca30cbab9 100644
--- a/ui/src/app/configuration/security-configuration/security-configuration.component.ts
+++ b/ui/src/app/configuration/security-configuration/security-configuration.component.ts
@@ -33,7 +33,7 @@ export class SecurityConfigurationComponent implements OnInit {
constructor(private breadcrumbService: SpBreadcrumbService) {}
ngOnInit(): void {
- this.breadcrumbService.updateBreadcrumb([SpConfigurationRoutes.BASE, {label: SpConfigurationTabs.getTabs()[5].itemTitle}]);
+ this.breadcrumbService.updateBreadcrumb([SpConfigurationRoutes.BASE, {label: SpConfigurationTabs.getTabs()[6].itemTitle}]);
}
}