You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/12/05 09:57:17 UTC
(streampipes) branch 2252-enhance-data-lake-sink-schema-management-options updated: refactor(#2252): Address PR review comments
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/2252-enhance-data-lake-sink-schema-management-options by this push:
new b2b0253a4 refactor(#2252): Address PR review comments
b2b0253a4 is described below
commit b2b0253a4c5193bba0d749436398c099d1fbbe00
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Tue Dec 5 10:56:57 2023 +0100
refactor(#2252): Address PR review comments
---
.../dataexplorer/DataExplorerSchemaManagement.java | 39 +++++++++----------
.../apache/streampipes/rest/ResetManagement.java | 5 +--
ui/package-lock.json | 44 ++++++++++++++++++++++
ui/package.json | 6 ++-
.../template/PipelineInvocationBuilder.ts | 2 -
5 files changed, 70 insertions(+), 26 deletions(-)
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
index 53c1b68b2..22a931a70 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
@@ -61,7 +61,7 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
/**
* For new measurements an entry is generated in the database. For existing measurements the schema is updated
- * according to the provided update strategy.
+ * according to the update strategy defined by the measurement.
*/
@Override
public DataLakeMeasure createOrUpdateMeasurement(DataLakeMeasure measure) {
@@ -200,24 +200,25 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
DataLakeMeasure measure1,
DataLakeMeasure measure2
) {
- return new ArrayList<>(
- Stream
- .concat(
- measure1
- .getEventSchema()
- .getEventProperties()
- .stream(),
- measure2
- .getEventSchema()
- .getEventProperties()
- .stream()
- )
- .collect(Collectors.toMap(
- EventProperty::getRuntimeName,
- Function.identity(),
- (eventProperty, eventProperty2) -> eventProperty
- ))
- .values());
+ // Combine the event properties from both measures into a single Stream
+ var allMeasurementProperties = Stream.concat(
+ measure1.getEventSchema()
+ .getEventProperties()
+ .stream(),
+ measure2.getEventSchema()
+ .getEventProperties()
+ .stream()
+ );
+ // Filter event properties by removing duplicate runtime names
+ // If there are duplicate keys, choose the first occurrence
+ var unifiedEventProperties = allMeasurementProperties
+ .collect(Collectors.toMap(
+ EventProperty::getRuntimeName,
+ Function.identity(),
+ (eventProperty, eventProperty2) -> eventProperty
+ ))
+ .values();
+ return new ArrayList<>(unifiedEventProperties);
}
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
index eab28167a..ff6066159 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
@@ -22,7 +22,6 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException;
import org.apache.streampipes.connect.management.management.AdapterMasterManagement;
import org.apache.streampipes.dataexplorer.DataExplorerQueryManagement;
import org.apache.streampipes.dataexplorer.DataExplorerSchemaManagement;
-import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
import org.apache.streampipes.manager.file.FileManager;
import org.apache.streampipes.manager.pipeline.PipelineCacheManager;
import org.apache.streampipes.manager.pipeline.PipelineCanvasMetadataCacheManager;
@@ -98,8 +97,8 @@ public class ResetManagement {
var dataLakeStorage = StorageDispatcher.INSTANCE
.getNoSqlStore()
.getDataLakeStorage();
- IDataExplorerSchemaManagement dataLakeMeasureManagement = new DataExplorerSchemaManagement(dataLakeStorage);
- DataExplorerQueryManagement dataExplorerQueryManagement =
+ var dataLakeMeasureManagement = new DataExplorerSchemaManagement(dataLakeStorage);
+ var dataExplorerQueryManagement =
new DataExplorerQueryManagement(dataLakeMeasureManagement);
List<DataLakeMeasure> allMeasurements = dataLakeMeasureManagement.getAllMeasurements();
allMeasurements.forEach(measurement -> {
diff --git a/ui/package-lock.json b/ui/package-lock.json
index f25d67249..6c5df3d80 100644
--- a/ui/package-lock.json
+++ b/ui/package-lock.json
@@ -30,6 +30,8 @@
"@ngbracket/ngx-layout": "^15.0.1-beta.42-aa93e27",
"@ngx-loading-bar/core": "6.0.2",
"@ngx-loading-bar/http-client": "6.0.2",
+ "@ngx-matomo/router": "^4.1.0",
+ "@ngx-matomo/tracker": "^4.1.0",
"@panzoom/panzoom": "^4.5.1",
"@swimlane/ngx-charts": "^20.4.1",
"angular-datatables": "^15.0.0",
@@ -6054,6 +6056,32 @@
"@ngx-loading-bar/core": "6.0.2"
}
},
+ "node_modules/@ngx-matomo/router": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ngx-matomo/router/-/router-4.1.0.tgz",
+ "integrity": "sha512-DffJewCaXlweyNFf5IIohkE6bqLi5Ik1V60/NR8eEuUqYSCzAB9rZQDDJd2ngwvQwGo2gH6FW111oGJVq9TBig==",
+ "dependencies": {
+ "tslib": "^2.0.0"
+ },
+ "peerDependencies": {
+ "@angular/common": "^15.0.0 || ^16.0.0",
+ "@angular/core": "^15.0.0 || ^16.0.0",
+ "@angular/router": "^15.0.0 || ^16.0.0",
+ "@ngx-matomo/tracker": "4.1.0"
+ }
+ },
+ "node_modules/@ngx-matomo/tracker": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ngx-matomo/tracker/-/tracker-4.1.0.tgz",
+ "integrity": "sha512-/YA4xsZw3HvPNERxxHUvwu4rjCUlyHDpvTiRg+FGfcLuOknmb6dN3MtA1X89yUq7UatnaOzvu8peresd2Z9/lw==",
+ "dependencies": {
+ "tslib": "^2.0.0"
+ },
+ "peerDependencies": {
+ "@angular/common": "^15.0.0 || ^16.0.0",
+ "@angular/core": "^15.0.0 || ^16.0.0"
+ }
+ },
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -32699,6 +32727,22 @@
"tslib": "^2.0.0"
}
},
+ "@ngx-matomo/router": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ngx-matomo/router/-/router-4.1.0.tgz",
+ "integrity": "sha512-DffJewCaXlweyNFf5IIohkE6bqLi5Ik1V60/NR8eEuUqYSCzAB9rZQDDJd2ngwvQwGo2gH6FW111oGJVq9TBig==",
+ "requires": {
+ "tslib": "^2.0.0"
+ }
+ },
+ "@ngx-matomo/tracker": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ngx-matomo/tracker/-/tracker-4.1.0.tgz",
+ "integrity": "sha512-/YA4xsZw3HvPNERxxHUvwu4rjCUlyHDpvTiRg+FGfcLuOknmb6dN3MtA1X89yUq7UatnaOzvu8peresd2Z9/lw==",
+ "requires": {
+ "tslib": "^2.0.0"
+ }
+ },
"@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
diff --git a/ui/package.json b/ui/package.json
index 295be0b14..c8ab1971a 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -37,7 +37,6 @@
"@angular/common": "^15.2.9",
"@angular/compiler": "^15.2.9",
"@angular/core": "^15.2.9",
- "@ngbracket/ngx-layout": "^15.0.1-beta.42-aa93e27",
"@angular/forms": "^15.2.9",
"@angular/material": "^15.2.9",
"@angular/platform-browser": "^15.2.9",
@@ -50,8 +49,11 @@
"@danielmoncada/angular-datetime-picker": "^15.0.2",
"@fortawesome/fontawesome-free": "6.4.2",
"@jsplumb/browser-ui": "^6.2.10",
+ "@ngbracket/ngx-layout": "^15.0.1-beta.42-aa93e27",
"@ngx-loading-bar/core": "6.0.2",
"@ngx-loading-bar/http-client": "6.0.2",
+ "@ngx-matomo/router": "^4.1.0",
+ "@ngx-matomo/tracker": "^4.1.0",
"@panzoom/panzoom": "^4.5.1",
"@swimlane/ngx-charts": "^20.4.1",
"angular-datatables": "^15.0.0",
@@ -65,8 +67,8 @@
"datatables.net": "^1.13.1",
"datatables.net-dt": "^1.13.1",
"echarts": "^5.4.1",
- "echarts-wordcloud": "^2.1.0",
"echarts-simple-transform": "^1.0.0",
+ "echarts-wordcloud": "^2.1.0",
"file-saver": "2.0.5",
"jquery": "^3.7.0",
"jquery-ui-dist": "1.13.2",
diff --git a/ui/src/app/core-services/template/PipelineInvocationBuilder.ts b/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
index b8a2f7d3d..9192a735f 100644
--- a/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
+++ b/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
@@ -20,8 +20,6 @@ import {
MappingPropertyUnary,
PipelineTemplateInvocation,
} from '@streampipes/platform-services';
-import { OneOfStaticProperty } from '../../../../projects/streampipes/platform-services/src/lib/model/gen/streampipes-model';
-import { optionsProviders } from '@danielmoncada/angular-datetime-picker';
export class PipelineInvocationBuilder {
private pipelineTemplateInvocation: PipelineTemplateInvocation;