You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/12/05 09:57:17 UTC

(streampipes) branch 2252-enhance-data-lake-sink-schema-management-options updated: refactor(#2252): Address PR review comments

This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git


The following commit(s) were added to refs/heads/2252-enhance-data-lake-sink-schema-management-options by this push:
     new b2b0253a4 refactor(#2252): Address PR review comments
b2b0253a4 is described below

commit b2b0253a4c5193bba0d749436398c099d1fbbe00
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Tue Dec 5 10:56:57 2023 +0100

    refactor(#2252): Address PR review comments
---
 .../dataexplorer/DataExplorerSchemaManagement.java | 39 +++++++++----------
 .../apache/streampipes/rest/ResetManagement.java   |  5 +--
 ui/package-lock.json                               | 44 ++++++++++++++++++++++
 ui/package.json                                    |  6 ++-
 .../template/PipelineInvocationBuilder.ts          |  2 -
 5 files changed, 70 insertions(+), 26 deletions(-)

diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
index 53c1b68b2..22a931a70 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
@@ -61,7 +61,7 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
 
   /**
    * For new measurements an entry is generated in the database. For existing measurements the schema is updated
-   * according to the provided update strategy.
+   * according to the update strategy defined by the measurement.
    */
   @Override
   public DataLakeMeasure createOrUpdateMeasurement(DataLakeMeasure measure) {
@@ -200,24 +200,25 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
       DataLakeMeasure measure1,
       DataLakeMeasure measure2
   ) {
-    return new ArrayList<>(
-        Stream
-            .concat(
-                measure1
-                    .getEventSchema()
-                    .getEventProperties()
-                    .stream(),
-                measure2
-                    .getEventSchema()
-                    .getEventProperties()
-                    .stream()
-            )
-            .collect(Collectors.toMap(
-                EventProperty::getRuntimeName,
-                Function.identity(),
-                (eventProperty, eventProperty2) -> eventProperty
-            ))
-            .values());
+    // Combine the event properties from both measures into a single Stream
+    var allMeasurementProperties = Stream.concat(
+        measure1.getEventSchema()
+                .getEventProperties()
+                .stream(),
+        measure2.getEventSchema()
+                .getEventProperties()
+                .stream()
+    );
 
+    // Filter event properties by removing duplicate runtime names
+    // If there are duplicate keys, choose the first occurrence
+    var unifiedEventProperties = allMeasurementProperties
+        .collect(Collectors.toMap(
+            EventProperty::getRuntimeName,
+            Function.identity(),
+            (eventProperty, eventProperty2) -> eventProperty
+        ))
+        .values();
+    return new ArrayList<>(unifiedEventProperties);
   }
 }
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
index eab28167a..ff6066159 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
@@ -22,7 +22,6 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException;
 import org.apache.streampipes.connect.management.management.AdapterMasterManagement;
 import org.apache.streampipes.dataexplorer.DataExplorerQueryManagement;
 import org.apache.streampipes.dataexplorer.DataExplorerSchemaManagement;
-import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
 import org.apache.streampipes.manager.file.FileManager;
 import org.apache.streampipes.manager.pipeline.PipelineCacheManager;
 import org.apache.streampipes.manager.pipeline.PipelineCanvasMetadataCacheManager;
@@ -98,8 +97,8 @@ public class ResetManagement {
     var dataLakeStorage = StorageDispatcher.INSTANCE
         .getNoSqlStore()
         .getDataLakeStorage();
-    IDataExplorerSchemaManagement dataLakeMeasureManagement = new DataExplorerSchemaManagement(dataLakeStorage);
-    DataExplorerQueryManagement dataExplorerQueryManagement =
+    var dataLakeMeasureManagement = new DataExplorerSchemaManagement(dataLakeStorage);
+    var dataExplorerQueryManagement =
         new DataExplorerQueryManagement(dataLakeMeasureManagement);
     List<DataLakeMeasure> allMeasurements = dataLakeMeasureManagement.getAllMeasurements();
     allMeasurements.forEach(measurement -> {
diff --git a/ui/package-lock.json b/ui/package-lock.json
index f25d67249..6c5df3d80 100644
--- a/ui/package-lock.json
+++ b/ui/package-lock.json
@@ -30,6 +30,8 @@
         "@ngbracket/ngx-layout": "^15.0.1-beta.42-aa93e27",
         "@ngx-loading-bar/core": "6.0.2",
         "@ngx-loading-bar/http-client": "6.0.2",
+        "@ngx-matomo/router": "^4.1.0",
+        "@ngx-matomo/tracker": "^4.1.0",
         "@panzoom/panzoom": "^4.5.1",
         "@swimlane/ngx-charts": "^20.4.1",
         "angular-datatables": "^15.0.0",
@@ -6054,6 +6056,32 @@
         "@ngx-loading-bar/core": "6.0.2"
       }
     },
+    "node_modules/@ngx-matomo/router": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ngx-matomo/router/-/router-4.1.0.tgz",
+      "integrity": "sha512-DffJewCaXlweyNFf5IIohkE6bqLi5Ik1V60/NR8eEuUqYSCzAB9rZQDDJd2ngwvQwGo2gH6FW111oGJVq9TBig==",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "peerDependencies": {
+        "@angular/common": "^15.0.0 || ^16.0.0",
+        "@angular/core": "^15.0.0 || ^16.0.0",
+        "@angular/router": "^15.0.0 || ^16.0.0",
+        "@ngx-matomo/tracker": "4.1.0"
+      }
+    },
+    "node_modules/@ngx-matomo/tracker": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ngx-matomo/tracker/-/tracker-4.1.0.tgz",
+      "integrity": "sha512-/YA4xsZw3HvPNERxxHUvwu4rjCUlyHDpvTiRg+FGfcLuOknmb6dN3MtA1X89yUq7UatnaOzvu8peresd2Z9/lw==",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "peerDependencies": {
+        "@angular/common": "^15.0.0 || ^16.0.0",
+        "@angular/core": "^15.0.0 || ^16.0.0"
+      }
+    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -32699,6 +32727,22 @@
         "tslib": "^2.0.0"
       }
     },
+    "@ngx-matomo/router": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ngx-matomo/router/-/router-4.1.0.tgz",
+      "integrity": "sha512-DffJewCaXlweyNFf5IIohkE6bqLi5Ik1V60/NR8eEuUqYSCzAB9rZQDDJd2ngwvQwGo2gH6FW111oGJVq9TBig==",
+      "requires": {
+        "tslib": "^2.0.0"
+      }
+    },
+    "@ngx-matomo/tracker": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@ngx-matomo/tracker/-/tracker-4.1.0.tgz",
+      "integrity": "sha512-/YA4xsZw3HvPNERxxHUvwu4rjCUlyHDpvTiRg+FGfcLuOknmb6dN3MtA1X89yUq7UatnaOzvu8peresd2Z9/lw==",
+      "requires": {
+        "tslib": "^2.0.0"
+      }
+    },
     "@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
diff --git a/ui/package.json b/ui/package.json
index 295be0b14..c8ab1971a 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -37,7 +37,6 @@
     "@angular/common": "^15.2.9",
     "@angular/compiler": "^15.2.9",
     "@angular/core": "^15.2.9",
-    "@ngbracket/ngx-layout": "^15.0.1-beta.42-aa93e27",
     "@angular/forms": "^15.2.9",
     "@angular/material": "^15.2.9",
     "@angular/platform-browser": "^15.2.9",
@@ -50,8 +49,11 @@
     "@danielmoncada/angular-datetime-picker": "^15.0.2",
     "@fortawesome/fontawesome-free": "6.4.2",
     "@jsplumb/browser-ui": "^6.2.10",
+    "@ngbracket/ngx-layout": "^15.0.1-beta.42-aa93e27",
     "@ngx-loading-bar/core": "6.0.2",
     "@ngx-loading-bar/http-client": "6.0.2",
+    "@ngx-matomo/router": "^4.1.0",
+    "@ngx-matomo/tracker": "^4.1.0",
     "@panzoom/panzoom": "^4.5.1",
     "@swimlane/ngx-charts": "^20.4.1",
     "angular-datatables": "^15.0.0",
@@ -65,8 +67,8 @@
     "datatables.net": "^1.13.1",
     "datatables.net-dt": "^1.13.1",
     "echarts": "^5.4.1",
-    "echarts-wordcloud": "^2.1.0",
     "echarts-simple-transform": "^1.0.0",
+    "echarts-wordcloud": "^2.1.0",
     "file-saver": "2.0.5",
     "jquery": "^3.7.0",
     "jquery-ui-dist": "1.13.2",
diff --git a/ui/src/app/core-services/template/PipelineInvocationBuilder.ts b/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
index b8a2f7d3d..9192a735f 100644
--- a/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
+++ b/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
@@ -20,8 +20,6 @@ import {
     MappingPropertyUnary,
     PipelineTemplateInvocation,
 } from '@streampipes/platform-services';
-import { OneOfStaticProperty } from '../../../../projects/streampipes/platform-services/src/lib/model/gen/streampipes-model';
-import { optionsProviders } from '@danielmoncada/angular-datetime-picker';
 
 export class PipelineInvocationBuilder {
     private pipelineTemplateInvocation: PipelineTemplateInvocation;