You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2022/08/01 14:02:43 UTC
[incubator-streampipes] branch STREAMPIPES-563 updated (dd37cfaf5 -> 1622d8167)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a change to branch STREAMPIPES-563
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
from dd37cfaf5 [STREAMPIPES-563] Move data lake logic into package streampipes-data-explorer-commons
add 3a30261a0 [hotfix] Write tags in data explorer
add 85eec4db2 [hotfix] Extend data explorer API, add menu layout to base css
add eae8d4e8b [hotfix] Extend data explorer query builder
add bfa035132 [STREAMPIPES-545] Support data source links
add d6581497f [STREAMPIPES-545] Move adapter API to platform-services module
add 0f37e31f3 [STREAMPIPES-565] Add data export feature
add 59129d3cd [STREAMPIPES-565] Add import feature to configuration
new 1622d8167 Merge branch 'STREAMPIPES-545' into STREAMPIPES-563
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 2 +-
pom.xml | 4 +
.../backend/StreamPipesResourceConfig.java | 2 +
.../streampipes/commons/zip/ZipFileExtractor.java | 34 +++-
streampipes-data-explorer-commons/pom.xml | 19 +-
.../dataexplorer/commons/DataExplorerWriter.java | 2 +-
.../commons/couchdb/CouchDbConfigurations.java | 1 -
.../dataexplorer/sdk/DataLakeQueryBuilder.java | 8 +-
.../dataexplorer/sdk/DataLakeQueryOrdering.java | 18 ++
streampipes-data-export/pom.xml | 49 +++++
.../streampipes/export/AssetLinkCollector.java | 50 +++++
.../streampipes/export/AssetLinkResolver.java | 83 +++++++++
.../apache/streampipes/export/ExportManager.java | 46 +++++
.../apache/streampipes/export/ImportManager.java | 39 ++++
.../export/constants/ExportConstants.java | 20 +-
.../export/constants/ResolvableAssetLinks.java | 25 +--
.../export/dataimport/ImportGenerator.java | 106 +++++++++++
.../export/dataimport/PerformImportGenerator.java | 142 ++++++++++++++
.../export/dataimport/PreviewImportGenerator.java | 102 +++++++++++
.../export/generator/ExportPackageGenerator.java | 155 ++++++++++++++++
.../export/generator/ZipFileBuilder.java | 115 ++++++++++++
.../streampipes/export/model/PermissionInfo.java | 36 ++--
.../export/resolver/AbstractResolver.java | 65 +++++++
.../export/resolver/AdapterResolver.java | 59 ++++++
.../export/resolver/DashboardResolver.java | 63 +++++++
.../export/resolver/DashboardWidgetResolver.java | 54 ++++++
.../export/resolver/DataSourceResolver.java | 54 ++++++
.../export/resolver/DataViewResolver.java | 64 +++++++
.../export/resolver/DataViewWidgetResolver.java | 54 ++++++
.../export/resolver/MeasurementResolver.java | 54 ++++++
.../export/resolver/PipelineResolver.java | 56 ++++++
.../export/utils/SerializationUtils.java | 27 +--
.../apache/streampipes/model/assets/AssetLink.java | 24 +++
.../model/datalake/DataLakeMeasure.java | 14 ++
.../model/export/AssetExportConfiguration.java | 160 ++++++++++++++++
.../model/export/ExportConfiguration.java | 31 ++--
.../AssetLink.java => export/ExportItem.java} | 41 ++---
.../export/StreamPipesApplicationPackage.java | 204 +++++++++++++++++++++
.../setup/tasks/CreateAssetLinkTypeTask.java | 4 +-
streampipes-rest/pom.xml | 5 +
.../rest/impl/admin/DataExportResource.java | 60 ++++++
.../rest/impl/admin/DataImportResource.java | 63 +++++++
.../src/lib/apis/adapter.service.ts} | 39 +---
.../lib/apis/data-view-data-explorer.service.ts | 6 +
.../src/lib/model/assets/asset.model.ts | 1 +
.../src/lib/model/gen/streampipes-model.ts | 122 ++++++++++--
.../platform-services/src/public-api.ts | 1 +
ui/src/app/assets/assets.module.ts | 2 +
.../asset-details-panel.component.ts | 9 +-
.../asset-overview/asset-overview.component.ts | 38 +++-
.../create-asset-dialog.component.html | 50 +++++
.../create-asset-dialog.component.scss | 19 +-
.../create-asset/create-asset-dialog.component.ts | 59 ++++++
.../edit-asset-link-dialog.component.html | 24 +++
.../edit-asset-link-dialog.component.ts | 32 +++-
ui/src/app/configuration/configuration-tabs.ts | 1 +
ui/src/app/configuration/configuration.module.ts | 12 ++
.../datalake-configuration.component.html | 13 +-
.../export/data-export-import.component.html | 52 ++++++
.../export/data-export-import.component.scss} | 0
.../export/data-export-import.component.ts | 96 ++++++++++
.../configuration/export/data-export.service.ts | 79 ++++++++
.../data-export-dialog.component.html | 49 +++++
.../data-export-dialog.component.scss} | 0
.../export-dialog/data-export-dialog.component.ts | 64 +++++++
.../data-export-item.component.html | 26 +++
.../data-export-item.component.scss} | 0
.../data-export-item/data-export-item.component.ts | 35 ++--
.../data-import-dialog.component.html | 93 ++++++++++
.../data-import-dialog.component.scss} | 0
.../import-dialog/data-import-dialog.component.ts | 94 ++++++++++
.../messaging-configuration.component.ts | 2 +-
.../pipeline-element-configuration.component.ts | 2 +-
.../security-configuration.component.ts | 2 +-
.../adapter-description.component.ts | 5 +-
.../data-marketplace/data-marketplace.component.ts | 5 +-
.../existing-adapters.component.ts | 5 +-
.../filter-toolbar/filter-toolbar.component.ts | 4 +-
.../new-adapter/new-adapter.component.ts | 6 +-
ui/src/app/connect/connect.module.ts | 2 -
.../delete-adapter-dialog.component.ts | 5 +-
ui/src/app/connect/services/connect.service.ts | 21 +++
ui/src/scss/_variables.scss | 3 +-
ui/src/scss/sp/layout.scss | 3 +
84 files changed, 3083 insertions(+), 212 deletions(-)
create mode 100644 streampipes-data-export/pom.xml
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkCollector.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/ImportManager.java
copy streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java => streampipes-data-export/src/main/java/org/apache/streampipes/export/constants/ExportConstants.java (58%)
copy streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java => streampipes-data-export/src/main/java/org/apache/streampipes/export/constants/ResolvableAssetLinks.java (58%)
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/ImportGenerator.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ZipFileBuilder.java
copy streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java => streampipes-data-export/src/main/java/org/apache/streampipes/export/model/PermissionInfo.java (57%)
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
create mode 100644 streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
copy streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java => streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/SerializationUtils.java (58%)
create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/export/AssetExportConfiguration.java
copy streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java => streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportConfiguration.java (57%)
copy streampipes-model/src/main/java/org/apache/streampipes/model/{assets/AssetLink.java => export/ExportItem.java} (62%)
create mode 100644 streampipes-model/src/main/java/org/apache/streampipes/model/export/StreamPipesApplicationPackage.java
create mode 100644 streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/DataExportResource.java
create mode 100644 streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/DataImportResource.java
rename ui/{src/app/connect/services/data-marketplace.service.ts => projects/streampipes/platform-services/src/lib/apis/adapter.service.ts} (68%)
create mode 100644 ui/src/app/assets/dialog/create-asset/create-asset-dialog.component.html
copy streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java => ui/src/app/assets/dialog/create-asset/create-asset-dialog.component.scss (58%)
create mode 100644 ui/src/app/assets/dialog/create-asset/create-asset-dialog.component.ts
create mode 100644 ui/src/app/configuration/export/data-export-import.component.html
copy ui/src/app/{apidocs/apidocs.component.scss => configuration/export/data-export-import.component.scss} (100%)
create mode 100644 ui/src/app/configuration/export/data-export-import.component.ts
create mode 100644 ui/src/app/configuration/export/data-export.service.ts
create mode 100644 ui/src/app/configuration/export/export-dialog/data-export-dialog.component.html
copy ui/src/app/{connect/dialog/adapter-template/adapter-template-dialog.component.scss => configuration/export/export-dialog/data-export-dialog.component.scss} (100%)
create mode 100644 ui/src/app/configuration/export/export-dialog/data-export-dialog.component.ts
create mode 100644 ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.html
copy ui/src/app/{apidocs/apidocs.component.scss => configuration/export/export-dialog/data-export-item/data-export-item.component.scss} (100%)
copy streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java => ui/src/app/configuration/export/export-dialog/data-export-item/data-export-item.component.ts (56%)
create mode 100644 ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
copy ui/src/app/{connect/dialog/adapter-template/adapter-template-dialog.component.scss => configuration/export/import-dialog/data-import-dialog.component.scss} (100%)
create mode 100644 ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
[incubator-streampipes] 01/01: Merge branch 'STREAMPIPES-545' into STREAMPIPES-563
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch STREAMPIPES-563
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 1622d8167e0b421812136769773ad3a01fb078f9
Merge: dd37cfaf5 59129d3cd
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Mon Aug 1 16:02:36 2022 +0200
Merge branch 'STREAMPIPES-545' into STREAMPIPES-563
.gitignore | 2 +-
pom.xml | 4 +
.../backend/StreamPipesResourceConfig.java | 2 +
.../streampipes/commons/zip/ZipFileExtractor.java | 34 +++-
streampipes-data-explorer-commons/pom.xml | 19 +-
.../dataexplorer/commons/DataExplorerWriter.java | 2 +-
.../commons/couchdb/CouchDbConfigurations.java | 1 -
.../dataexplorer/sdk/DataLakeQueryBuilder.java | 8 +-
.../dataexplorer/sdk/DataLakeQueryOrdering.java | 18 ++
streampipes-data-export/pom.xml | 49 +++++
.../streampipes/export/AssetLinkCollector.java | 50 +++++
.../streampipes/export/AssetLinkResolver.java | 83 +++++++++
.../apache/streampipes/export/ExportManager.java | 46 +++++
.../apache/streampipes/export/ImportManager.java | 39 ++++
.../export/constants/ExportConstants.java | 20 +-
.../export/constants/ResolvableAssetLinks.java | 25 +--
.../export/dataimport/ImportGenerator.java | 106 +++++++++++
.../export/dataimport/PerformImportGenerator.java | 142 ++++++++++++++
.../export/dataimport/PreviewImportGenerator.java | 102 +++++++++++
.../export/generator/ExportPackageGenerator.java | 155 ++++++++++++++++
.../export/generator/ZipFileBuilder.java | 115 ++++++++++++
.../streampipes/export/model/PermissionInfo.java | 36 ++--
.../export/resolver/AbstractResolver.java | 65 +++++++
.../export/resolver/AdapterResolver.java | 59 ++++++
.../export/resolver/DashboardResolver.java | 63 +++++++
.../export/resolver/DashboardWidgetResolver.java | 54 ++++++
.../export/resolver/DataSourceResolver.java | 54 ++++++
.../export/resolver/DataViewResolver.java | 64 +++++++
.../export/resolver/DataViewWidgetResolver.java | 54 ++++++
.../export/resolver/MeasurementResolver.java | 54 ++++++
.../export/resolver/PipelineResolver.java | 56 ++++++
.../export/utils/SerializationUtils.java | 27 +--
.../apache/streampipes/model/assets/AssetLink.java | 24 +++
.../model/datalake/DataLakeMeasure.java | 14 ++
.../model/export/AssetExportConfiguration.java | 160 ++++++++++++++++
.../model/export/ExportConfiguration.java | 31 ++--
.../AssetLink.java => export/ExportItem.java} | 41 ++---
.../export/StreamPipesApplicationPackage.java | 204 +++++++++++++++++++++
.../setup/tasks/CreateAssetLinkTypeTask.java | 4 +-
streampipes-rest/pom.xml | 5 +
.../rest/impl/admin/DataExportResource.java | 60 ++++++
.../rest/impl/admin/DataImportResource.java | 63 +++++++
.../src/lib/apis/adapter.service.ts} | 39 +---
.../lib/apis/data-view-data-explorer.service.ts | 6 +
.../src/lib/model/assets/asset.model.ts | 1 +
.../src/lib/model/gen/streampipes-model.ts | 122 ++++++++++--
.../platform-services/src/public-api.ts | 1 +
ui/src/app/assets/assets.module.ts | 2 +
.../asset-details-panel.component.ts | 9 +-
.../asset-overview/asset-overview.component.ts | 38 +++-
.../create-asset-dialog.component.html | 50 +++++
.../create-asset-dialog.component.scss | 19 +-
.../create-asset/create-asset-dialog.component.ts | 59 ++++++
.../edit-asset-link-dialog.component.html | 24 +++
.../edit-asset-link-dialog.component.ts | 32 +++-
ui/src/app/configuration/configuration-tabs.ts | 1 +
ui/src/app/configuration/configuration.module.ts | 12 ++
.../datalake-configuration.component.html | 13 +-
.../export/data-export-import.component.html | 52 ++++++
.../export/data-export-import.component.scss | 19 --
.../export/data-export-import.component.ts | 96 ++++++++++
.../configuration/export/data-export.service.ts | 79 ++++++++
.../data-export-dialog.component.html | 49 +++++
.../data-export-dialog.component.scss | 19 +-
.../export-dialog/data-export-dialog.component.ts | 64 +++++++
.../data-export-item.component.html | 26 +++
.../data-export-item.component.scss | 19 --
.../data-export-item/data-export-item.component.ts | 35 ++--
.../data-import-dialog.component.html | 93 ++++++++++
.../data-import-dialog.component.scss | 19 +-
.../import-dialog/data-import-dialog.component.ts | 94 ++++++++++
.../messaging-configuration.component.ts | 2 +-
.../pipeline-element-configuration.component.ts | 2 +-
.../security-configuration.component.ts | 2 +-
.../adapter-description.component.ts | 5 +-
.../data-marketplace/data-marketplace.component.ts | 5 +-
.../existing-adapters.component.ts | 5 +-
.../filter-toolbar/filter-toolbar.component.ts | 4 +-
.../new-adapter/new-adapter.component.ts | 6 +-
ui/src/app/connect/connect.module.ts | 2 -
.../delete-adapter-dialog.component.ts | 5 +-
ui/src/app/connect/services/connect.service.ts | 21 +++
ui/src/scss/_variables.scss | 3 +-
ui/src/scss/sp/layout.scss | 3 +
84 files changed, 3085 insertions(+), 286 deletions(-)
diff --cc streampipes-data-explorer-commons/pom.xml
index 5da55d089,999370c82..976048553
--- a/streampipes-data-explorer-commons/pom.xml
+++ b/streampipes-data-explorer-commons/pom.xml
@@@ -45,6 -58,9 +62,6 @@@
</dependency>
</dependencies>
- <properties>
- <maven.compiler.source>11</maven.compiler.source>
- <maven.compiler.target>11</maven.compiler.target>
- </properties>
+
- </project>
+ </project>
diff --cc streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java
index 8dc7d7370,cf64457e4..adf9d8983
--- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java
+++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/couchdb/CouchDbConfigurations.java
@@@ -15,22 -15,12 +15,21 @@@
* limitations under the License.
*
*/
+package org.apache.streampipes.dataexplorer.commons.couchdb;
-package org.apache.streampipes.sinks.internal.jvm.config;
+import org.apache.streampipes.svcdiscovery.api.model.ConfigItem;
-public class ConfigKeys {
+import java.util.Arrays;
+import java.util.List;
-
- public final static String COUCHDB_HOST = "SP_COUCHDB_HOST";
- public final static String COUCHDB_PORT = "SP_COUCHDB_PORT";
- public final static String COUCHDB_PROTOCOL = "SP_COUCHDB_PROTOCOL";
-}
+public class CouchDbConfigurations {
+
+ public static List<ConfigItem> getDefaults() {
+ return Arrays.asList(
+ ConfigItem.from(CouchDbEnvKeys.COUCHDB_HOST, "couchdb", "Hostname for CouchDB to store image blobs"),
+ ConfigItem.from(CouchDbEnvKeys.COUCHDB_PORT, 5984, ""),
+ ConfigItem.from(CouchDbEnvKeys.COUCHDB_PROTOCOL, "http", "")
+ );
+ }
+
+}
diff --cc streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
index f1bdc8f82,b8b259a62..5b90fd955
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
@@@ -25,10 -27,10 +27,14 @@@ import org.apache.streampipes.model.sha
@TsModel
public class DataLakeMeasure extends UnnamedStreamPipesEntity {
+ public final static String CURRENT_SCHEMA_VERSION = "1.1";
++
+ @JsonProperty("_rev")
+ private @SerializedName("_rev") String rev;
+
private String measureName;
+
+ private String timestampField;
private EventSchema eventSchema;
private String pipelineId;
private String pipelineName;
@@@ -98,19 -92,11 +104,27 @@@
this.pipelineIsRunning = pipelineIsRunning;
}
+ public String getSchemaVersion() {
+ return schemaVersion;
+ }
+
+ public void setSchemaVersion(String schemaVersion) {
+ this.schemaVersion = schemaVersion;
+ }
+
+ public String getTimestampField() {
+ return timestampField;
+ }
+
+ public void setTimestampField(String timestampField) {
+ this.timestampField = timestampField;
+ }
++
+ public String getRev() {
+ return rev;
+ }
+
+ public void setRev(String rev) {
+ this.rev = rev;
+ }
}