You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/12/04 16:43:52 UTC
(streampipes) branch 2252-enhance-data-lake-sink-schema-management-options updated (fdb62d87b -> 9afaf4a58)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a change to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git
from fdb62d87b refactor(#2246): Add migration script for data lake sink
new 7009dde09 refactor(#2246): Register migration
new af931d989 refactor(#2246): Add REST endpoint string
new 9afaf4a58 refactor(#2246): Fix update bug
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../streampipes/dataexplorer/DataExplorerSchemaManagement.java | 1 +
.../internal/jvm/InternalSinksExtensionModuleExports.java | 3 ++-
.../streampipes/sinks/internal/jvm/datalake/DataLakeSink.java | 10 +++++++++-
.../rest/extensions/migration/DataSinkMigrationResource.java | 2 ++
4 files changed, 14 insertions(+), 2 deletions(-)
(streampipes) 01/03: refactor(#2246): Register migration
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 7009dde09a6103e72176828ce60f6853b5d9df59
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Dec 4 16:49:19 2023 +0100
refactor(#2246): Register migration
---
.../sinks/internal/jvm/InternalSinksExtensionModuleExports.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
index ccf27a5da..a32aaa7cf 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
@@ -23,6 +23,7 @@ import org.apache.streampipes.extensions.api.declarer.IExtensionModuleExport;
import org.apache.streampipes.extensions.api.migration.IModelMigrator;
import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
import org.apache.streampipes.sinks.internal.jvm.datalake.DataLakeSink;
+import org.apache.streampipes.sinks.internal.jvm.migration.DataLakeSinkMigrationV1;
import org.apache.streampipes.sinks.internal.jvm.notification.NotificationProducer;
import java.util.Collections;
@@ -44,6 +45,6 @@ public class InternalSinksExtensionModuleExports implements IExtensionModuleExpo
@Override
public List<IModelMigrator<?, ?>> migrators() {
- return Collections.emptyList();
+ return List.of(new DataLakeSinkMigrationV1());
}
}
(streampipes) 03/03: refactor(#2246): Fix update bug
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 9afaf4a589db2ecb85e01cac6978c17424b08647
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Dec 4 17:43:42 2023 +0100
refactor(#2246): Fix update bug
---
.../streampipes/dataexplorer/DataExplorerSchemaManagement.java | 1 +
.../streampipes/sinks/internal/jvm/datalake/DataLakeSink.java | 10 +++++++++-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
index 858cd1f8c..53c1b68b2 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
@@ -86,6 +86,7 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
DataLakeMeasure measure,
DataLakeMeasure existingMeasure
) {
+ measure.setElementId(existingMeasure.getElementId());
if (DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA.equals(measure.getSchemaUpdateStrategy())) {
// For the update schema strategy the old schema is overwritten with the new one
updateMeasurement(measure);
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
index b95b183dc..56aa41676 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.dataexplorer.commons.TimeSeriesStore;
import org.apache.streampipes.extensions.api.pe.context.EventSinkRuntimeContext;
import org.apache.streampipes.model.DataSinkType;
import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.DataLakeMeasureSchemaUpdateStrategy;
import org.apache.streampipes.model.graph.DataSinkDescription;
import org.apache.streampipes.model.runtime.Event;
import org.apache.streampipes.model.schema.PropertyScope;
@@ -80,10 +81,17 @@ public class DataLakeSink extends StreamPipesDataSink {
.get(0)
.getEventSchema();
- String schemaUpdateOption = extractor.selectedSingleValue(SCHEMA_UPDATE_KEY, String.class);
var measure = new DataLakeMeasure(measureName, timestampField, eventSchema);
+ var schemaUpdateOptionString = extractor.selectedSingleValue(SCHEMA_UPDATE_KEY, String.class);
+
+ if (schemaUpdateOptionString.equals(DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA.toString())) {
+ measure.setSchemaUpdateStrategy(DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA);
+ } else {
+ measure.setSchemaUpdateStrategy(DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA);
+ }
+
this.timeSeriesStore = new TimeSeriesStore(
Environments.getEnvironment(),
runtimeContext.getStreamPipesClient(),
(streampipes) 02/03: refactor(#2246): Add REST endpoint string
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch 2252-enhance-data-lake-sink-schema-management-options
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit af931d98913495c991c3efd3f1f1ca5b761d8c9c
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Dec 4 17:05:45 2023 +0100
refactor(#2246): Add REST endpoint string
---
.../rest/extensions/migration/DataSinkMigrationResource.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
index ceea800b1..33ca0bede 100644
--- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
+++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
@@ -30,6 +30,7 @@ import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import jakarta.ws.rs.Path;
import org.apache.http.HttpStatus;
import jakarta.ws.rs.Consumes;
@@ -37,6 +38,7 @@ import jakarta.ws.rs.POST;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
+@Path("api/v1/migrations/sink")
public class DataSinkMigrationResource extends MigrateExtensionsResource<
DataSinkInvocation,
IDataSinkParameterExtractor,