You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/12/04 16:43:52 UTC

(streampipes) branch 2252-enhance-data-lake-sink-schema-management-options updated (fdb62d87b -> 9afaf4a58)

This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a change to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git


    from fdb62d87b refactor(#2246): Add migration script for data lake sink
     new 7009dde09 refactor(#2246): Register migration
     new af931d989 refactor(#2246): Add REST endpoint string
     new 9afaf4a58 refactor(#2246): Fix update bug

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../streampipes/dataexplorer/DataExplorerSchemaManagement.java |  1 +
 .../internal/jvm/InternalSinksExtensionModuleExports.java      |  3 ++-
 .../streampipes/sinks/internal/jvm/datalake/DataLakeSink.java  | 10 +++++++++-
 .../rest/extensions/migration/DataSinkMigrationResource.java   |  2 ++
 4 files changed, 14 insertions(+), 2 deletions(-)


(streampipes) 01/03: refactor(#2246): Register migration

Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git

commit 7009dde09a6103e72176828ce60f6853b5d9df59
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Dec 4 16:49:19 2023 +0100

    refactor(#2246): Register migration
---
 .../sinks/internal/jvm/InternalSinksExtensionModuleExports.java        | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
index ccf27a5da..a32aaa7cf 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
@@ -23,6 +23,7 @@ import org.apache.streampipes.extensions.api.declarer.IExtensionModuleExport;
 import org.apache.streampipes.extensions.api.migration.IModelMigrator;
 import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
 import org.apache.streampipes.sinks.internal.jvm.datalake.DataLakeSink;
+import org.apache.streampipes.sinks.internal.jvm.migration.DataLakeSinkMigrationV1;
 import org.apache.streampipes.sinks.internal.jvm.notification.NotificationProducer;
 
 import java.util.Collections;
@@ -44,6 +45,6 @@ public class InternalSinksExtensionModuleExports implements IExtensionModuleExpo
 
   @Override
   public List<IModelMigrator<?, ?>> migrators() {
-    return Collections.emptyList();
+    return List.of(new DataLakeSinkMigrationV1());
   }
 }


(streampipes) 03/03: refactor(#2246): Fix update bug

Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git

commit 9afaf4a589db2ecb85e01cac6978c17424b08647
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Dec 4 17:43:42 2023 +0100

    refactor(#2246): Fix update bug
---
 .../streampipes/dataexplorer/DataExplorerSchemaManagement.java |  1 +
 .../streampipes/sinks/internal/jvm/datalake/DataLakeSink.java  | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
index 858cd1f8c..53c1b68b2 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
@@ -86,6 +86,7 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
       DataLakeMeasure measure,
       DataLakeMeasure existingMeasure
   ) {
+    measure.setElementId(existingMeasure.getElementId());
     if (DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA.equals(measure.getSchemaUpdateStrategy())) {
       // For the update schema strategy the old schema is overwritten with the new one
       updateMeasurement(measure);
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
index b95b183dc..56aa41676 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.dataexplorer.commons.TimeSeriesStore;
 import org.apache.streampipes.extensions.api.pe.context.EventSinkRuntimeContext;
 import org.apache.streampipes.model.DataSinkType;
 import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.DataLakeMeasureSchemaUpdateStrategy;
 import org.apache.streampipes.model.graph.DataSinkDescription;
 import org.apache.streampipes.model.runtime.Event;
 import org.apache.streampipes.model.schema.PropertyScope;
@@ -80,10 +81,17 @@ public class DataLakeSink extends StreamPipesDataSink {
                                 .get(0)
                                 .getEventSchema();
 
-    String schemaUpdateOption = extractor.selectedSingleValue(SCHEMA_UPDATE_KEY, String.class);
 
     var measure = new DataLakeMeasure(measureName, timestampField, eventSchema);
 
+    var schemaUpdateOptionString = extractor.selectedSingleValue(SCHEMA_UPDATE_KEY, String.class);
+
+    if (schemaUpdateOptionString.equals(DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA.toString())) {
+      measure.setSchemaUpdateStrategy(DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA);
+    } else {
+      measure.setSchemaUpdateStrategy(DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA);
+    }
+
     this.timeSeriesStore = new TimeSeriesStore(
         Environments.getEnvironment(),
         runtimeContext.getStreamPipesClient(),


(streampipes) 02/03: refactor(#2246): Add REST endpoint string

Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git

commit af931d98913495c991c3efd3f1f1ca5b761d8c9c
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Dec 4 17:05:45 2023 +0100

    refactor(#2246): Add REST endpoint string
---
 .../rest/extensions/migration/DataSinkMigrationResource.java            | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
index ceea800b1..33ca0bede 100644
--- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
+++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
@@ -30,6 +30,7 @@ import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.ExampleObject;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import jakarta.ws.rs.Path;
 import org.apache.http.HttpStatus;
 
 import jakarta.ws.rs.Consumes;
@@ -37,6 +38,7 @@ import jakarta.ws.rs.POST;
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
 
+@Path("api/v1/migrations/sink")
 public class DataSinkMigrationResource extends MigrateExtensionsResource<
         DataSinkInvocation,
         IDataSinkParameterExtractor,