You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/12/06 11:16:53 UTC

(streampipes) branch dev updated: feat(#2252): enhance data lake sink schema management options (#2263)

This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git


The following commit(s) were added to refs/heads/dev by this push:
     new 306d30dcd feat(#2252): enhance data lake sink schema management options (#2263)
306d30dcd is described below

commit 306d30dcde22ea8a262ae8aef5dbe1e96448ac88
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Wed Dec 6 12:16:47 2023 +0100

    feat(#2252): enhance data lake sink schema management options (#2263)
---
 .../dataexplorer/commons/TimeSeriesStore.java      |   6 -
 .../commons/influx/InfluxClientProvider.java       |   2 +
 streampipes-data-explorer/pom.xml                  |  12 +-
 .../dataexplorer/DataExplorerSchemaManagement.java | 203 +++++++++++++-------
 .../api/IDataExplorerSchemaManagement.java         |   2 +-
 .../dataexplorer/query/AutoAggregationHandler.java |  35 +++-
 .../DataExplorerSchemaManagementTest.java          | 204 +++++++++++++++++++++
 .../connectors/influx/sink/InfluxDbClient.java     |  17 +-
 .../streampipes-sinks-internal-jvm/pom.xml         |  11 ++
 .../jvm/InternalSinksExtensionModuleExports.java   |   3 +-
 .../sinks/internal/jvm/datalake/DataLakeSink.java  |  56 ++++--
 .../jvm/migration/DataLakeSinkMigrationV1.java     |  79 ++++++++
 .../documentation.md                               |  22 ++-
 .../strings.en                                     |   3 +
 .../jvm/migration/DataLakeSinkMigrationV1Test.java |  62 +++++++
 .../model/datalake/DataLakeMeasure.java            |  10 +
 .../DataLakeMeasureSchemaUpdateStrategy.java       |  34 ++--
 .../streampipes/ps/DataLakeMeasureResourceV4.java  |  14 +-
 .../apache/streampipes/ps/DataLakeResourceV4.java  |  16 +-
 .../migration/DataSinkMigrationResource.java       |   4 +
 .../apache/streampipes/rest/ResetManagement.java   |   8 +-
 .../src/lib/model/gen/streampipes-model-client.ts  |   2 +-
 .../src/lib/model/gen/streampipes-model.ts         |  14 +-
 .../adapter-started-dialog.component.ts            |   4 +
 .../template/PipelineInvocationBuilder.ts          |  20 ++
 .../save-pipeline/save-pipeline.component.ts       |   1 -
 ui/src/app/editor/services/jsplumb.service.ts      |   1 -
 27 files changed, 683 insertions(+), 162 deletions(-)

diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java
index 31ce5cbf3..4bc662869 100644
--- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java
+++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java
@@ -47,7 +47,6 @@ public class TimeSeriesStore {
     DataExplorerUtils.sanitizeAndRegisterAtDataLake(client, measure);
 
     if (enableImageStore) {
-      // TODO check if event properties are replaces correctly
       this.imageStore = new ImageStore(measure, environment);
     }
 
@@ -67,11 +66,6 @@ public class TimeSeriesStore {
     return true;
   }
 
-
-  public boolean alterRetentionTime(DataLakeMeasure dataLakeMeasure) {
-    return true;
-  }
-
   public void close() throws SpRuntimeException {
     if (imageStore != null) {
       try {
diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java
index 9a9e7a127..a95ecfac7 100644
--- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java
+++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java
@@ -137,8 +137,10 @@ public class InfluxClientProvider {
     return false;
   }
 
+
   private static Environment getEnvironment() {
     return Environments.getEnvironment();
   }
 
+
 }
diff --git a/streampipes-data-explorer/pom.xml b/streampipes-data-explorer/pom.xml
index c68a436d7..2b18b0790 100644
--- a/streampipes-data-explorer/pom.xml
+++ b/streampipes-data-explorer/pom.xml
@@ -48,13 +48,23 @@
             <artifactId>streampipes-storage-management</artifactId>
             <version>0.95.0-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.streampipes</groupId>
+            <artifactId>streampipes-test-utils</artifactId>
+            <version>0.95.0-SNAPSHOT</version>
+            <scope>test</scope>
+        </dependency>
 
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
-
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.influxdb</groupId>
             <artifactId>influxdb-java</artifactId>
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
index 8c4dc4a34..7db5bad83 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java
@@ -21,24 +21,34 @@ package org.apache.streampipes.dataexplorer;
 import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
 import org.apache.streampipes.dataexplorer.utils.DataExplorerUtils;
 import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.DataLakeMeasureSchemaUpdateStrategy;
 import org.apache.streampipes.model.schema.EventProperty;
-import org.apache.streampipes.model.schema.EventPropertyList;
-import org.apache.streampipes.model.schema.EventPropertyNested;
-import org.apache.streampipes.model.schema.EventPropertyPrimitive;
 import org.apache.streampipes.storage.api.IDataLakeStorage;
 import org.apache.streampipes.storage.couchdb.utils.Utils;
-import org.apache.streampipes.storage.management.StorageDispatcher;
 
 import com.google.gson.JsonObject;
 import org.lightcouch.CouchDbClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class DataExplorerSchemaManagement implements IDataExplorerSchemaManagement {
 
+  private static final Logger LOG = LoggerFactory.getLogger(DataExplorerSchemaManagement.class);
+
+  IDataLakeStorage dataLakeStorage;
+
+  public DataExplorerSchemaManagement(IDataLakeStorage dataLakeStorage) {
+    this.dataLakeStorage = dataLakeStorage;
+  }
+
   @Override
   public List<DataLakeMeasure> getAllMeasurements() {
     return DataExplorerUtils.getInfos();
@@ -46,39 +56,68 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
 
   @Override
   public DataLakeMeasure getById(String elementId) {
-    return getDataLakeStorage().findOne(elementId);
+    return dataLakeStorage.findOne(elementId);
   }
 
+  /**
+   * For new measurements an entry is generated in the database. For existing measurements the schema is updated
+   * according to the update strategy defined by the measurement.
+   */
   @Override
-  public DataLakeMeasure createMeasurement(DataLakeMeasure measure) {
-    List<DataLakeMeasure> dataLakeMeasureList = getDataLakeStorage().getAllDataLakeMeasures();
-    Optional<DataLakeMeasure> optional =
-        dataLakeMeasureList.stream().filter(entry -> entry.getMeasureName().equals(measure.getMeasureName()))
-            .findFirst();
-
-    if (optional.isPresent()) {
-      DataLakeMeasure oldEntry = optional.get();
-      if (!compareEventProperties(oldEntry.getEventSchema().getEventProperties(),
-          measure.getEventSchema().getEventProperties())) {
-        oldEntry.setEventSchema(measure.getEventSchema());
-        oldEntry.setTimestampField(measure.getTimestampField());
-        oldEntry.setPipelineName(measure.getPipelineName());
-        getDataLakeStorage().updateDataLakeMeasure(oldEntry);
-        return oldEntry;
-      }
+  public DataLakeMeasure createOrUpdateMeasurement(DataLakeMeasure measure) {
+
+    setDefaultUpdateStrategyIfNoneProvided(measure);
+
+    var existingMeasure = getExistingMeasureByName(measure.getMeasureName());
+
+    if (existingMeasure.isEmpty()) {
+      setSchemaVersionAndStoreMeasurement(measure);
     } else {
-      measure.setSchemaVersion(DataLakeMeasure.CURRENT_SCHEMA_VERSION);
-      getDataLakeStorage().storeDataLakeMeasure(measure);
-      return measure;
+      handleExistingMeasurement(measure, existingMeasure.get());
     }
 
     return measure;
   }
 
+  /**
+   * Destiguishes between the update straregy for existing measurments
+   */
+  private void handleExistingMeasurement(
+      DataLakeMeasure measure,
+      DataLakeMeasure existingMeasure
+  ) {
+    measure.setElementId(existingMeasure.getElementId());
+    if (DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA.equals(measure.getSchemaUpdateStrategy())) {
+      // For the update schema strategy the old schema is overwritten with the new one
+      updateMeasurement(measure);
+    } else {
+      // For the extent existing schema strategy the old schema is merged with the new one
+      unifyEventSchemaAndUpdateMeasure(measure, existingMeasure);
+    }
+  }
+
+
+  /**
+   * Returns the existing measure that has the provided measure name
+   */
+  private Optional<DataLakeMeasure> getExistingMeasureByName(String measureName) {
+    return dataLakeStorage.getAllDataLakeMeasures()
+                          .stream()
+                          .filter(m -> m.getMeasureName()
+                                        .equals(measureName))
+                          .findFirst();
+  }
+
+  private static void setDefaultUpdateStrategyIfNoneProvided(DataLakeMeasure measure) {
+    if (measure.getSchemaUpdateStrategy() == null) {
+      measure.setSchemaUpdateStrategy(DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA);
+    }
+  }
+
   @Override
   public void deleteMeasurement(String elementId) {
-    if (getDataLakeStorage().findOne(elementId) != null) {
-      getDataLakeStorage().deleteDataLakeMeasure(elementId);
+    if (dataLakeStorage.findOne(elementId) != null) {
+      dataLakeStorage.deleteDataLakeMeasure(elementId);
     } else {
       throw new IllegalArgumentException("Could not find measure with this ID");
     }
@@ -88,12 +127,23 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
   public boolean deleteMeasurementByName(String measureName) {
     boolean isSuccess = false;
     CouchDbClient couchDbClient = Utils.getCouchDbDataLakeClient();
-    List<JsonObject> docs = couchDbClient.view("_all_docs").includeDocs(true).query(JsonObject.class);
+    List<JsonObject> docs = couchDbClient.view("_all_docs")
+                                         .includeDocs(true)
+                                         .query(JsonObject.class);
 
     for (JsonObject document : docs) {
-      if (document.get("measureName").toString().replace("\"", "").equals(measureName)) {
-        couchDbClient.remove(document.get("_id").toString().replace("\"", ""),
-            document.get("_rev").toString().replace("\"", ""));
+      if (document.get("measureName")
+                  .toString()
+                  .replace("\"", "")
+                  .equals(measureName)) {
+        couchDbClient.remove(
+            document.get("_id")
+                    .toString()
+                    .replace("\"", ""),
+            document.get("_rev")
+                    .toString()
+                    .replace("\"", "")
+        );
         isSuccess = true;
         break;
       }
@@ -102,58 +152,73 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManageme
     try {
       couchDbClient.close();
     } catch (IOException e) {
-      e.printStackTrace();
+      LOG.error("Could not close CouchDB client", e);
     }
     return isSuccess;
   }
 
   @Override
   public void updateMeasurement(DataLakeMeasure measure) {
-    var existingMeasure = getDataLakeStorage().findOne(measure.getElementId());
+    var existingMeasure = dataLakeStorage.findOne(measure.getElementId());
     if (existingMeasure != null) {
       measure.setRev(existingMeasure.getRev());
-      getDataLakeStorage().updateDataLakeMeasure(measure);
+      dataLakeStorage.updateDataLakeMeasure(measure);
     } else {
-      getDataLakeStorage().storeDataLakeMeasure(measure);
+      dataLakeStorage.storeDataLakeMeasure(measure);
     }
   }
 
-  private IDataLakeStorage getDataLakeStorage() {
-    return StorageDispatcher.INSTANCE.getNoSqlStore().getDataLakeStorage();
+  private void setSchemaVersionAndStoreMeasurement(DataLakeMeasure measure) {
+    measure.setSchemaVersion(DataLakeMeasure.CURRENT_SCHEMA_VERSION);
+    dataLakeStorage.storeDataLakeMeasure(measure);
   }
 
-  private boolean compareEventProperties(List<EventProperty> prop1, List<EventProperty> prop2) {
-    if (prop1.size() != prop2.size()) {
-      return false;
-    }
-
-    return prop1.stream().allMatch(prop -> {
-
-      for (EventProperty property : prop2) {
-        if (prop.getRuntimeName().equals(property.getRuntimeName())) {
-
-          //primitive
-          if (prop instanceof EventPropertyPrimitive && property instanceof EventPropertyPrimitive) {
-            if (((EventPropertyPrimitive) prop)
-                .getRuntimeType()
-                .equals(((EventPropertyPrimitive) property).getRuntimeType())) {
-              return true;
-            }
-
-            //list
-          } else if (prop instanceof EventPropertyList && property instanceof EventPropertyList) {
-            return compareEventProperties(Collections.singletonList(((EventPropertyList) prop).getEventProperty()),
-                Collections.singletonList(((EventPropertyList) property).getEventProperty()));
-
-            //nested
-          } else if (prop instanceof EventPropertyNested && property instanceof EventPropertyNested) {
-            return compareEventProperties(((EventPropertyNested) prop).getEventProperties(),
-                ((EventPropertyNested) property).getEventProperties());
-          }
-        }
-      }
-      return false;
+  /**
+   * First the event schemas of the measurements are merged and then the measure is updated in the database
+   */
+  private void unifyEventSchemaAndUpdateMeasure(
+      DataLakeMeasure measure,
+      DataLakeMeasure existingMeasure
+  ) {
+    var properties = getUnifiedEventProperties(
+        existingMeasure,
+        measure
+    );
+
+    measure
+        .getEventSchema()
+        .setEventProperties(properties);
+
+    updateMeasurement(measure);
+  }
 
-    });
+  /**
+   * Returns the union of the unique event properties of the two measures.
+   * They are unique by runtime name.
+   */
+  private List<EventProperty> getUnifiedEventProperties(
+      DataLakeMeasure measure1,
+      DataLakeMeasure measure2
+  ) {
+// Combine the event properties from both measures into a single Stream
+    var allMeasurementProperties = Stream.concat(
+        measure1.getEventSchema()
+                .getEventProperties()
+                .stream(),
+        measure2.getEventSchema()
+                .getEventProperties()
+                .stream()
+    );
+
+    // Filter event properties by removing duplicate runtime names
+    // If there are duplicate keys, choose the first occurrence
+    var unifiedEventProperties = allMeasurementProperties
+        .collect(Collectors.toMap(
+            EventProperty::getRuntimeName,
+            Function.identity(),
+            (eventProperty, eventProperty2) -> eventProperty
+        ))
+        .values();
+    return new ArrayList<>(unifiedEventProperties);
   }
 }
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java
index d8dc29648..a210b515b 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java
@@ -28,7 +28,7 @@ public interface IDataExplorerSchemaManagement {
 
   DataLakeMeasure getById(String elementId);
 
-  DataLakeMeasure createMeasurement(DataLakeMeasure measure);
+  DataLakeMeasure createOrUpdateMeasurement(DataLakeMeasure measure);
 
   void deleteMeasurement(String elementId);
 
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java
index 1bff44a1d..68bf23171 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams;
 import org.apache.streampipes.dataexplorer.param.model.SelectColumn;
 import org.apache.streampipes.dataexplorer.querybuilder.DataLakeQueryOrdering;
 import org.apache.streampipes.model.datalake.SpQueryResult;
+import org.apache.streampipes.storage.management.StorageDispatcher;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,7 +63,10 @@ public class AutoAggregationHandler {
   }
 
   private IDataExplorerQueryManagement getDataLakeQueryManagement() {
-    return new DataExplorerQueryManagement(new DataExplorerSchemaManagement());
+    var dataLakeStorage = StorageDispatcher.INSTANCE
+        .getNoSqlStore()
+        .getDataLakeStorage();
+    return new DataExplorerQueryManagement(new DataExplorerSchemaManagement(dataLakeStorage));
   }
 
   public ProvidedRestQueryParams makeAutoAggregationQueryParams() throws IllegalArgumentException {
@@ -106,7 +110,13 @@ public class AutoAggregationHandler {
 
     SpQueryResult result = dataLakeQueryManagement.getData(countParams, true);
 
-    return result.getTotal() > 0 ? ((Double) result.getAllDataSeries().get(0).getRows().get(0).get(1)).intValue() : 0;
+    return result.getTotal() > 0 ? (
+        (Double) result.getAllDataSeries()
+                       .get(0)
+                       .getRows()
+                       .get(0)
+                       .get(1)
+    ).intValue() : 0;
   }
 
   private SpQueryResult fireQuery(ProvidedRestQueryParams params) {
@@ -116,7 +126,8 @@ public class AutoAggregationHandler {
   private int getAggregationValue(SpQueryResult newest, SpQueryResult oldest) throws ParseException {
     long timerange = extractTimestamp(newest) - extractTimestamp(oldest);
     double v = timerange / MAX_RETURN_LIMIT;
-    return Double.valueOf(v).intValue();
+    return Double.valueOf(v)
+                 .intValue();
   }
 
   private SpQueryResult getSingleRecord(DataLakeQueryOrdering order) throws ParseException {
@@ -131,8 +142,12 @@ public class AutoAggregationHandler {
 
   private String transformColumns(String rawQuery) {
     List<SelectColumn> columns =
-        Arrays.stream(rawQuery.split(COMMA)).map(SelectColumn::fromApiQueryString).collect(Collectors.toList());
-    return columns.stream().map(SelectColumn::getOriginalField).collect(Collectors.joining(COMMA));
+        Arrays.stream(rawQuery.split(COMMA))
+              .map(SelectColumn::fromApiQueryString)
+              .collect(Collectors.toList());
+    return columns.stream()
+                  .map(SelectColumn::getOriginalField)
+                  .collect(Collectors.joining(COMMA));
   }
 
   private String getSampleField(SpQueryResult result) {
@@ -145,8 +160,14 @@ public class AutoAggregationHandler {
   }
 
   private long extractTimestamp(SpQueryResult result) throws ParseException {
-    int timestampIndex = result.getHeaders().indexOf(TIMESTAMP_FIELD);
-    return tryParseDate(result.getAllDataSeries().get(0).getRows().get(0).get(timestampIndex).toString()).getTime();
+    int timestampIndex = result.getHeaders()
+                               .indexOf(TIMESTAMP_FIELD);
+    return tryParseDate(result.getAllDataSeries()
+                              .get(0)
+                              .getRows()
+                              .get(0)
+                              .get(timestampIndex)
+                              .toString()).getTime();
   }
 
   private Date tryParseDate(String v) throws ParseException {
diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java
new file mode 100644
index 000000000..baa6053b9
--- /dev/null
+++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.dataexplorer;
+
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.DataLakeMeasureSchemaUpdateStrategy;
+import org.apache.streampipes.model.schema.EventProperty;
+import org.apache.streampipes.storage.api.IDataLakeStorage;
+import org.apache.streampipes.test.generator.EventPropertyPrimitiveTestBuilder;
+import org.apache.streampipes.test.generator.EventSchemaTestBuilder;
+import org.apache.streampipes.vocabulary.XSD;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.net.URI;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class DataExplorerSchemaManagementTest {
+  public static final String NEW_PROPERTY = "newProperty";
+  public static final String OLD_PROPERTY = "oldProperty";
+
+  private IDataLakeStorage dataLakeStorageMock;
+
+  @Before
+  public void setUp() {
+    dataLakeStorageMock = mock(IDataLakeStorage.class);
+  }
+
+  @Test
+  public void createMeasurementThatNotExisted() {
+    when(dataLakeStorageMock.getAllDataLakeMeasures()).thenReturn(List.of());
+    when(dataLakeStorageMock.storeDataLakeMeasure(any())).thenReturn(true);
+    var schemaManagement = new DataExplorerSchemaManagement(dataLakeStorageMock);
+
+    var oldMeasure = getSampleMeasure(
+        DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA,
+        List.of()
+    );
+    var resultingMeasure = schemaManagement.createOrUpdateMeasurement(oldMeasure);
+
+    assertEquals(oldMeasure.getMeasureName(), resultingMeasure.getMeasureName());
+    verify(dataLakeStorageMock, Mockito.times(1))
+        .storeDataLakeMeasure(any());
+  }
+
+
+  @Test
+  public void createMeasurementWithUpdateStrategy() {
+
+    var oldMeasure = getSampleMeasure(
+        DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA,
+        List.of(
+            getEventProperty(OLD_PROPERTY, XSD.STRING)
+        )
+    );
+
+    when(dataLakeStorageMock.getAllDataLakeMeasures()).thenReturn(List.of(oldMeasure));
+    when(dataLakeStorageMock.findOne(any())).thenReturn(oldMeasure);
+    var schemaManagement = new DataExplorerSchemaManagement(dataLakeStorageMock);
+
+    var newMeasure = getNewMeasure(DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA);
+
+    var resultMeasure = schemaManagement.createOrUpdateMeasurement(newMeasure);
+
+    assertEquals(newMeasure.getMeasureName(), resultMeasure.getMeasureName());
+    verify(dataLakeStorageMock, Mockito.times(1))
+        .updateDataLakeMeasure(any());
+    assertFalse(containsPropertyWithName(resultMeasure, OLD_PROPERTY));
+    assertTrue(containsPropertyWithName(resultMeasure, NEW_PROPERTY));
+
+  }
+
+
+  @Test
+  public void createMeasurementWithExtendSchemaStrategy() {
+
+    var oldMeasure = getSampleMeasure(
+        DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA,
+        List.of(
+            getEventProperty(OLD_PROPERTY, XSD.STRING)
+        )
+    );
+    when(dataLakeStorageMock.getAllDataLakeMeasures()).thenReturn(List.of(oldMeasure));
+    when(dataLakeStorageMock.findOne(any())).thenReturn(oldMeasure);
+    var schemaManagement = new DataExplorerSchemaManagement(dataLakeStorageMock);
+    var newMeasure = getNewMeasure(DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA);
+
+    var resultMeasure = schemaManagement.createOrUpdateMeasurement(newMeasure);
+
+    assertEquals(newMeasure.getMeasureName(), resultMeasure.getMeasureName());
+    verify(dataLakeStorageMock, Mockito.times(1)).updateDataLakeMeasure(any());
+    assertTrue(containsPropertyWithName(resultMeasure, OLD_PROPERTY));
+    assertTrue(containsPropertyWithName(resultMeasure, NEW_PROPERTY));
+  }
+
+
+  @Test
+  public void createMeasurementWithExtendSchemaStrategyAndDifferentPropertyTypes() {
+    var oldMeasure = getSampleMeasure(
+        DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA,
+        List.of(
+            getEventProperty(OLD_PROPERTY, XSD.STRING),
+            getEventProperty(NEW_PROPERTY, XSD.INTEGER)
+        )
+    );
+
+    when(dataLakeStorageMock.getAllDataLakeMeasures()).thenReturn(List.of(oldMeasure));
+    when(dataLakeStorageMock.findOne(any())).thenReturn(oldMeasure);
+
+    var schemaManagement = new DataExplorerSchemaManagement(dataLakeStorageMock);
+
+    var newMeasure = getNewMeasure(DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA);
+
+    var resultMeasure = schemaManagement.createOrUpdateMeasurement(newMeasure);
+    assertEquals(newMeasure.getMeasureName(), resultMeasure.getMeasureName());
+    verify(dataLakeStorageMock, Mockito.times(1)).updateDataLakeMeasure(any());
+    assertEquals(
+        2,
+        resultMeasure.getEventSchema()
+                     .getEventProperties()
+                     .size()
+    );
+    assertTrue(containsPropertyWithName(resultMeasure, OLD_PROPERTY));
+    assertTrue(containsPropertyWithName(resultMeasure, NEW_PROPERTY));
+  }
+
+  private EventProperty getEventProperty(
+      String runtimeName,
+      URI runtimeType
+  ) {
+    return EventPropertyPrimitiveTestBuilder
+        .create()
+        .withRuntimeName(runtimeName)
+        .withRuntimeType(runtimeType)
+        .build();
+  }
+
+  private DataLakeMeasure getNewMeasure(DataLakeMeasureSchemaUpdateStrategy updateStrategy) {
+    return getSampleMeasure(
+        updateStrategy,
+        List.of(getEventProperty(NEW_PROPERTY, XSD.STRING))
+    );
+  }
+
+  private DataLakeMeasure getSampleMeasure(
+      DataLakeMeasureSchemaUpdateStrategy updateStrategy,
+      List<EventProperty> eventProperties
+  ) {
+    var measure = new DataLakeMeasure();
+    measure.setMeasureName("testMeasure");
+    measure.setSchemaUpdateStrategy(updateStrategy);
+
+    measure.setEventSchema(
+        EventSchemaTestBuilder
+            .create()
+            .withEventProperties(
+                eventProperties
+            )
+            .build()
+    );
+
+    return measure;
+  }
+
+  private boolean containsPropertyWithName(
+      DataLakeMeasure measure,
+      String runtimeName
+  ) {
+    return measure
+        .getEventSchema()
+        .getEventProperties()
+        .stream()
+        .anyMatch(
+            eventProperty -> eventProperty.getRuntimeName()
+                                          .equals(runtimeName)
+        );
+  }
+}
\ No newline at end of file
diff --git a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/sink/InfluxDbClient.java b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/sink/InfluxDbClient.java
index 63b8ce2f6..1be930666 100644
--- a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/sink/InfluxDbClient.java
+++ b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/sink/InfluxDbClient.java
@@ -26,7 +26,6 @@ import org.apache.streampipes.model.runtime.Event;
 
 import org.influxdb.BatchOptions;
 import org.influxdb.dto.Point;
-import org.influxdb.dto.Query;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -72,7 +71,7 @@ public class InfluxDbClient extends SharedInfluxClient {
     // Checking whether the database exists
     if (!influxClientProvider.databaseExists(influxDb, databaseName)) {
       LOG.info("Database '" + databaseName + "' not found. Gets created ...");
-      createDatabase(databaseName);
+      influxClientProvider.createDatabase(influxDb, databaseName);
     }
 
     // setting up the database
@@ -80,20 +79,6 @@ public class InfluxDbClient extends SharedInfluxClient {
     influxDb.enableBatch(BatchOptions.DEFAULTS.actions(batchSize).flushDuration(flushDuration));
   }
 
-
-  /**
-   * Creates a new database with the given name
-   *
-   * @param dbName The name of the database which should be created
-   */
-  private void createDatabase(String dbName) throws SpRuntimeException {
-    if (!dbName.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) {
-      throw new SpRuntimeException(
-          "Databasename '" + dbName + "' not allowed. Allowed names: ^[a-zA-Z_][a-zA-Z0-9_]*$");
-    }
-    influxDb.query(new Query("CREATE DATABASE \"" + dbName + "\"", ""));
-  }
-
   /**
    * Saves an event to the connnected InfluxDB database
    *
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/pom.xml b/streampipes-extensions/streampipes-sinks-internal-jvm/pom.xml
index f41f811d8..34680e814 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/pom.xml
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/pom.xml
@@ -76,6 +76,17 @@
             <artifactId>commons-pool2</artifactId>
         </dependency>
 
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
index ccf27a5da..a32aaa7cf 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/InternalSinksExtensionModuleExports.java
@@ -23,6 +23,7 @@ import org.apache.streampipes.extensions.api.declarer.IExtensionModuleExport;
 import org.apache.streampipes.extensions.api.migration.IModelMigrator;
 import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
 import org.apache.streampipes.sinks.internal.jvm.datalake.DataLakeSink;
+import org.apache.streampipes.sinks.internal.jvm.migration.DataLakeSinkMigrationV1;
 import org.apache.streampipes.sinks.internal.jvm.notification.NotificationProducer;
 
 import java.util.Collections;
@@ -44,6 +45,6 @@ public class InternalSinksExtensionModuleExports implements IExtensionModuleExpo
 
   @Override
   public List<IModelMigrator<?, ?>> migrators() {
-    return Collections.emptyList();
+    return List.of(new DataLakeSinkMigrationV1());
   }
 }
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
index 82d8bf2b5..cbf151133 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java
@@ -24,24 +24,29 @@ import org.apache.streampipes.dataexplorer.commons.TimeSeriesStore;
 import org.apache.streampipes.extensions.api.pe.context.EventSinkRuntimeContext;
 import org.apache.streampipes.model.DataSinkType;
 import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.DataLakeMeasureSchemaUpdateStrategy;
 import org.apache.streampipes.model.graph.DataSinkDescription;
 import org.apache.streampipes.model.runtime.Event;
-import org.apache.streampipes.model.schema.EventSchema;
 import org.apache.streampipes.model.schema.PropertyScope;
 import org.apache.streampipes.sdk.builder.DataSinkBuilder;
 import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder;
 import org.apache.streampipes.sdk.helpers.EpRequirements;
 import org.apache.streampipes.sdk.helpers.Labels;
 import org.apache.streampipes.sdk.helpers.Locales;
+import org.apache.streampipes.sdk.helpers.Options;
 import org.apache.streampipes.sdk.utils.Assets;
 import org.apache.streampipes.wrapper.params.compat.SinkParams;
 import org.apache.streampipes.wrapper.standalone.StreamPipesDataSink;
 
-
 public class DataLakeSink extends StreamPipesDataSink {
 
   private static final String DATABASE_MEASUREMENT_KEY = "db_measurement";
   private static final String TIMESTAMP_MAPPING_KEY = "timestamp_mapping";
+  public static final String SCHEMA_UPDATE_KEY = "schema_update";
+
+  public static final String SCHEMA_UPDATE_OPTION = "Update schema";
+
+  public static final String EXTEND_EXISTING_SCHEMA_OPTION = "Extend existing schema";
 
   private TimeSeriesStore timeSeriesStore;
 
@@ -49,32 +54,53 @@ public class DataLakeSink extends StreamPipesDataSink {
   @Override
   public DataSinkDescription declareModel() {
     return DataSinkBuilder
-        .create("org.apache.streampipes.sinks.internal.jvm.datalake", 0)
+        .create("org.apache.streampipes.sinks.internal.jvm.datalake", 1)
         .withLocales(Locales.EN)
         .withAssets(Assets.DOCUMENTATION, Assets.ICON)
         .category(DataSinkType.INTERNAL)
-        .requiredStream(StreamRequirementsBuilder.create()
-            .requiredPropertyWithUnaryMapping(
-                EpRequirements.timestampReq(),
-                Labels.withId(TIMESTAMP_MAPPING_KEY),
-                PropertyScope.NONE)
-            .build())
+        .requiredStream(StreamRequirementsBuilder
+                            .create()
+                            .requiredPropertyWithUnaryMapping(
+                                EpRequirements.timestampReq(),
+                                Labels.withId(TIMESTAMP_MAPPING_KEY),
+                                PropertyScope.NONE
+                            )
+                            .build())
         .requiredTextParameter(Labels.withId(DATABASE_MEASUREMENT_KEY))
+        .requiredSingleValueSelection(
+            Labels.withId(SCHEMA_UPDATE_KEY),
+            Options.from(SCHEMA_UPDATE_OPTION, EXTEND_EXISTING_SCHEMA_OPTION)
+        )
+
         .build();
   }
 
   @Override
   public void onInvocation(SinkParams parameters, EventSinkRuntimeContext runtimeContext) throws SpRuntimeException {
-    String timestampField = parameters.extractor().mappingPropertyValue(TIMESTAMP_MAPPING_KEY);
-    String measureName = parameters.extractor().singleValueParameter(DATABASE_MEASUREMENT_KEY, String.class);
-    EventSchema eventSchema = parameters.getInputSchemaInfos().get(0).getEventSchema();
+    var extractor = parameters.extractor();
+    var timestampField = extractor.mappingPropertyValue(TIMESTAMP_MAPPING_KEY);
+    var measureName = extractor.singleValueParameter(DATABASE_MEASUREMENT_KEY, String.class);
+    var eventSchema = parameters.getInputSchemaInfos()
+                                .get(0)
+                                .getEventSchema();
+
+
+    var measure = new DataLakeMeasure(measureName, timestampField, eventSchema);
+
+    var schemaUpdateOptionString = extractor.selectedSingleValue(SCHEMA_UPDATE_KEY, String.class);
 
-    DataLakeMeasure measure = new DataLakeMeasure(measureName, timestampField, eventSchema);
+    if (schemaUpdateOptionString.equals(EXTEND_EXISTING_SCHEMA_OPTION)) {
+      measure.setSchemaUpdateStrategy(DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA);
+    } else {
+      measure.setSchemaUpdateStrategy(DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA);
+    }
 
-    this.timeSeriesStore = new TimeSeriesStore(Environments.getEnvironment(),
+    this.timeSeriesStore = new TimeSeriesStore(
+        Environments.getEnvironment(),
         runtimeContext.getStreamPipesClient(),
         measure,
-        true);
+        true
+    );
 
   }
 
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/migration/DataLakeSinkMigrationV1.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/migration/DataLakeSinkMigrationV1.java
new file mode 100644
index 000000000..b63c35999
--- /dev/null
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/migration/DataLakeSinkMigrationV1.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.sinks.internal.jvm.migration;
+
+import org.apache.streampipes.extensions.api.extractor.IDataSinkParameterExtractor;
+import org.apache.streampipes.extensions.api.migration.IDataSinkMigrator;
+import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix;
+import org.apache.streampipes.model.graph.DataSinkInvocation;
+import org.apache.streampipes.model.migration.MigrationResult;
+import org.apache.streampipes.model.migration.ModelMigratorConfig;
+import org.apache.streampipes.model.staticproperty.OneOfStaticProperty;
+import org.apache.streampipes.sdk.helpers.Labels;
+import org.apache.streampipes.sdk.helpers.Options;
+import org.apache.streampipes.sinks.internal.jvm.datalake.DataLakeSink;
+
+public class DataLakeSinkMigrationV1 implements IDataSinkMigrator {
+
+  @Override
+  public ModelMigratorConfig config() {
+    return new ModelMigratorConfig(
+        "org.apache.streampipes.sinks.internal.jvm.datalake",
+        SpServiceTagPrefix.DATA_SINK,
+        0,
+        1
+    );
+  }
+
+  /**
+   * Adds the static property for schema update to the sink and selects the option to update the
+   * schema as a default
+   */
+  @Override
+  public MigrationResult<DataSinkInvocation> migrate(
+      DataSinkInvocation element,
+      IDataSinkParameterExtractor extractor
+  ) throws RuntimeException {
+    var oneOfStaticProperty = createDefaultSchemaUpdateStrategy();
+
+    element.getStaticProperties()
+           .add(oneOfStaticProperty);
+
+    return MigrationResult.success(element);
+  }
+
+  private static OneOfStaticProperty createDefaultSchemaUpdateStrategy() {
+    var label = Labels.from(
+        DataLakeSink.SCHEMA_UPDATE_KEY,
+        DataLakeSink.SCHEMA_UPDATE_OPTION,
+        "Update existing schemas with the new one or extend the existing schema with new properties"
+    );
+    var schemaUpdateStaticProperty = new OneOfStaticProperty(
+        label.getInternalId(),
+        label.getLabel(),
+        label.getDescription()
+    );
+
+    var options = Options.from(DataLakeSink.SCHEMA_UPDATE_OPTION, DataLakeSink.EXTEND_EXISTING_SCHEMA_OPTION);
+    options.get(0)
+           .setSelected(true);
+    schemaUpdateStaticProperty.setOptions(options);
+    return schemaUpdateStaticProperty;
+  }
+}
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/resources/org.apache.streampipes.sinks.internal.jvm.datalake/documentation.md b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/resources/org.apache.streampipes.sinks.internal.jvm.datalake/documentation.md
index b9661f99b..2d58b7e3d 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/resources/org.apache.streampipes.sinks.internal.jvm.datalake/documentation.md
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/resources/org.apache.streampipes.sinks.internal.jvm.datalake/documentation.md
@@ -27,7 +27,8 @@
 ## Description
 
 Stores events in the internal data lake so that data can be visualized in the live dashboard or in the data explorer.
-Simply create a pipeline with a data lake sink, switch to one of the data exploration tool and start exploring your data!
+Simply create a pipeline with a data lake sink, switch to one of the data exploration tool and start exploring your
+data!
 
 ***
 
@@ -40,7 +41,22 @@ This sink requires an event that provides a timestamp value (a field that is mar
 
 ## Configuration
 
-### Index
+### Identifier
 
-The name of the storage group of this event. 
+The name of the measurement (table) where the events are stored.
 
+### Schema Update Options
+
+The Schema Update Options dictate the behavior when encountering a measurement (table) with the same identifier.
+
+#### Option 1: Update Schema
+
+- **Description:** Overrides the existing schema.
+- **Effect on Data:** The data remains in the data lake, but accessing old data is restricted to file export.
+- **Impact on Features:** Other StreamPipes features, such as the Data Explorer, will only display the new event schema.
+
+#### Option 2: Extend Existing Schema
+
+- **Description:** Keeps old event fields in the event schema.
+- **Strategy:** This follows an append-only strategy, allowing continued work with historic data.
+- **Consideration:** Old properties may exist for which no new data is generated.
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/resources/org.apache.streampipes.sinks.internal.jvm.datalake/strings.en b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/resources/org.apache.streampipes.sinks.internal.jvm.datalake/strings.en
index b79cc538b..0b604bdfd 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/resources/org.apache.streampipes.sinks.internal.jvm.datalake/strings.en
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/resources/org.apache.streampipes.sinks.internal.jvm.datalake/strings.en
@@ -23,3 +23,6 @@ db_measurement.description=The name of the identifier under which the data is to
 
 timestamp_mapping.title=Timestamp Field
 timestamp_mapping.description=The value which contains a timestamp
+
+schema_update.title=Schema Update
+schema_update.description=Update existing schemas with the new one or extend the existing schema with new properties
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/test/java/org/apache/streampipes/sinks/internal/jvm/migration/DataLakeSinkMigrationV1Test.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/test/java/org/apache/streampipes/sinks/internal/jvm/migration/DataLakeSinkMigrationV1Test.java
new file mode 100644
index 000000000..a162ded7d
--- /dev/null
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/test/java/org/apache/streampipes/sinks/internal/jvm/migration/DataLakeSinkMigrationV1Test.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.sinks.internal.jvm.migration;
+
+import org.apache.streampipes.model.graph.DataSinkInvocation;
+import org.apache.streampipes.model.migration.MigrationResult;
+import org.apache.streampipes.model.staticproperty.OneOfStaticProperty;
+import org.apache.streampipes.sdk.extractor.DataSinkParameterExtractor;
+import org.apache.streampipes.sinks.internal.jvm.datalake.DataLakeSink;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+public class DataLakeSinkMigrationV1Test {
+
+  @Test
+  public void migrate() {
+    var dataLakeSinkMigrationV1 = new DataLakeSinkMigrationV1();
+
+    var extractor = mock(DataSinkParameterExtractor.class);
+    var invocation = new DataSinkInvocation();
+    invocation.setStaticProperties(new ArrayList<>());
+
+    var actual = dataLakeSinkMigrationV1.migrate(invocation, extractor);
+
+    assertTrue(actual.success());
+    assertEquals(actual.element()
+                       .getStaticProperties()
+                       .size(), 1);
+    var schemaUpdateStaticProperty = getOneOfStaticProperty(actual);
+    assertEquals(schemaUpdateStaticProperty.getInternalName(), DataLakeSink.SCHEMA_UPDATE_KEY);
+    assertEquals(schemaUpdateStaticProperty.getOptions().get(0).isSelected(), true);
+    assertEquals(schemaUpdateStaticProperty.getOptions().get(1).isSelected(), false);
+  }
+
+  private static OneOfStaticProperty getOneOfStaticProperty(MigrationResult<DataSinkInvocation> actual) {
+    return (OneOfStaticProperty) actual.element()
+                                                                 .getStaticProperties()
+                                                                 .get(0);
+  }
+}
\ No newline at end of file
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
index 44076e605..2d9441b28 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
@@ -51,6 +51,8 @@ public class DataLakeMeasure {
 
   private String schemaVersion;
 
+  private DataLakeMeasureSchemaUpdateStrategy schemaUpdateStrategy = DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA;
+
   public DataLakeMeasure() {
     super();
   }
@@ -129,6 +131,14 @@ public class DataLakeMeasure {
     this.timestampField = timestampField;
   }
 
+  public DataLakeMeasureSchemaUpdateStrategy getSchemaUpdateStrategy() {
+    return schemaUpdateStrategy;
+  }
+
+  public void setSchemaUpdateStrategy(DataLakeMeasureSchemaUpdateStrategy schemaUpdateStrategy) {
+    this.schemaUpdateStrategy = schemaUpdateStrategy;
+  }
+
   /**
    * This can be used to get the name of the timestamp property without the stream prefix
    *
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasureSchemaUpdateStrategy.java
similarity index 61%
copy from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java
copy to streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasureSchemaUpdateStrategy.java
index d8dc29648..acd5bc232 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasureSchemaUpdateStrategy.java
@@ -16,23 +16,19 @@
  *
  */
 
-package org.apache.streampipes.dataexplorer.api;
+package org.apache.streampipes.model.datalake;
 
-import org.apache.streampipes.model.datalake.DataLakeMeasure;
-
-import java.util.List;
-
-public interface IDataExplorerSchemaManagement {
-
-  List<DataLakeMeasure> getAllMeasurements();
-
-  DataLakeMeasure getById(String elementId);
-
-  DataLakeMeasure createMeasurement(DataLakeMeasure measure);
-
-  void deleteMeasurement(String elementId);
-
-  boolean deleteMeasurementByName(String measureName);
-
-  void updateMeasurement(DataLakeMeasure measure);
-}
+/**
+ * This enum contains the different options that are available for the update strategy of the data lake measure
+ */
+public enum DataLakeMeasureSchemaUpdateStrategy {
+  /**
+   * This strategy will update the schema of the data lake measure when it changes
+   */
+  UPDATE_SCHEMA,
+  /**
+   * This strategy will extend the schema of the data lake measure when it changes and keep the old fields
+   */
+  EXTEND_EXISTING_SCHEMA
+
+}
\ No newline at end of file
diff --git a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV4.java b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV4.java
index c40f2e56b..7ed976cbc 100644
--- a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV4.java
+++ b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV4.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.dataexplorer.influx.DataLakeMeasurementCount;
 import org.apache.streampipes.model.datalake.DataLakeMeasure;
 import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
 import org.apache.streampipes.rest.shared.annotation.JacksonSerialized;
+import org.apache.streampipes.storage.management.StorageDispatcher;
 
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.DELETE;
@@ -46,7 +47,10 @@ public class DataLakeMeasureResourceV4 extends AbstractAuthGuardedRestResource {
   private final IDataExplorerSchemaManagement dataLakeMeasureManagement;
 
   public DataLakeMeasureResourceV4() {
-    this.dataLakeMeasureManagement = new DataExplorerSchemaManagement();
+    var dataLakeStorage = StorageDispatcher.INSTANCE
+        .getNoSqlStore()
+        .getDataLakeStorage();
+    this.dataLakeMeasureManagement = new DataExplorerSchemaManagement(dataLakeStorage);
   }
 
   @POST
@@ -54,7 +58,7 @@ public class DataLakeMeasureResourceV4 extends AbstractAuthGuardedRestResource {
   @Produces(MediaType.APPLICATION_JSON)
   @Consumes(MediaType.APPLICATION_JSON)
   public Response addDataLake(DataLakeMeasure dataLakeMeasure) {
-    DataLakeMeasure result = this.dataLakeMeasureManagement.createMeasurement(dataLakeMeasure);
+    DataLakeMeasure result = this.dataLakeMeasureManagement.createOrUpdateMeasurement(dataLakeMeasure);
     return ok(result);
   }
 
@@ -85,8 +89,10 @@ public class DataLakeMeasureResourceV4 extends AbstractAuthGuardedRestResource {
   @Produces(MediaType.APPLICATION_JSON)
   @Consumes(MediaType.APPLICATION_JSON)
   @Path("{id}")
-  public Response updateDataLakeMeasure(@PathParam("id") String elementId,
-                                        DataLakeMeasure measure) {
+  public Response updateDataLakeMeasure(
+      @PathParam("id") String elementId,
+      DataLakeMeasure measure
+  ) {
     if (elementId.equals(measure.getElementId())) {
       try {
         this.dataLakeMeasureManagement.updateMeasurement(measure);
diff --git a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
index a0d798c37..f2ba8b46c 100644
--- a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
+++ b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
@@ -27,6 +27,7 @@ import org.apache.streampipes.model.datalake.DataSeries;
 import org.apache.streampipes.model.datalake.SpQueryResult;
 import org.apache.streampipes.model.monitoring.SpLogMessage;
 import org.apache.streampipes.rest.core.base.impl.AbstractRestResource;
+import org.apache.streampipes.storage.management.StorageDispatcher;
 
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -35,8 +36,6 @@ import io.swagger.v3.oas.annotations.media.ArraySchema;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.DELETE;
@@ -80,22 +79,25 @@ import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams
 @Path("v4/datalake")
 public class DataLakeResourceV4 extends AbstractRestResource {
 
-  private static final Logger logger = LoggerFactory.getLogger(DataLakeResourceV4.class);
-
   private DataExplorerQueryManagement dataLakeManagement;
   private final DataExplorerSchemaManagement dataExplorerSchemaManagement;
 
   public DataLakeResourceV4() {
-    this.dataExplorerSchemaManagement = new DataExplorerSchemaManagement();
+    var dataLakeStorage = StorageDispatcher.INSTANCE
+        .getNoSqlStore()
+        .getDataLakeStorage();
+    this.dataExplorerSchemaManagement = new DataExplorerSchemaManagement(dataLakeStorage);
     this.dataLakeManagement = new DataExplorerQueryManagement(dataExplorerSchemaManagement);
   }
 
   public DataLakeResourceV4(DataExplorerQueryManagement dataLakeManagement) {
+    var dataLakeStorage = StorageDispatcher.INSTANCE
+        .getNoSqlStore()
+        .getDataLakeStorage();
     this.dataLakeManagement = dataLakeManagement;
-    this.dataExplorerSchemaManagement = new DataExplorerSchemaManagement();
+    this.dataExplorerSchemaManagement = new DataExplorerSchemaManagement(dataLakeStorage);
   }
 
-
   @DELETE
   @Path("/measurements/{measurementID}")
   @Operation(summary = "Remove data from a single measurement series with given id", tags = {"Data Lake"},
diff --git a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
index ceea800b1..397fce7b4 100644
--- a/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
+++ b/streampipes-rest-extensions/src/main/java/org/apache/streampipes/rest/extensions/migration/DataSinkMigrationResource.java
@@ -25,6 +25,7 @@ import org.apache.streampipes.model.graph.DataSinkInvocation;
 import org.apache.streampipes.rest.shared.annotation.JacksonSerialized;
 import org.apache.streampipes.sdk.extractor.DataSinkParameterExtractor;
 
+
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Content;
@@ -34,9 +35,12 @@ import org.apache.http.HttpStatus;
 
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
 
+
+@Path("api/v1/migrations/sink")
 public class DataSinkMigrationResource extends MigrateExtensionsResource<
         DataSinkInvocation,
         IDataSinkParameterExtractor,
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
index 6638ccc3e..ff6066159 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java
@@ -22,7 +22,6 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException;
 import org.apache.streampipes.connect.management.management.AdapterMasterManagement;
 import org.apache.streampipes.dataexplorer.DataExplorerQueryManagement;
 import org.apache.streampipes.dataexplorer.DataExplorerSchemaManagement;
-import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
 import org.apache.streampipes.manager.file.FileManager;
 import org.apache.streampipes.manager.pipeline.PipelineCacheManager;
 import org.apache.streampipes.manager.pipeline.PipelineCanvasMetadataCacheManager;
@@ -95,8 +94,11 @@ public class ResetManagement {
     });
 
     // Remove all data in data lake
-    IDataExplorerSchemaManagement dataLakeMeasureManagement = new DataExplorerSchemaManagement();
-    DataExplorerQueryManagement dataExplorerQueryManagement =
+    var dataLakeStorage = StorageDispatcher.INSTANCE
+        .getNoSqlStore()
+        .getDataLakeStorage();
+    var dataLakeMeasureManagement = new DataExplorerSchemaManagement(dataLakeStorage);
+    var dataExplorerQueryManagement =
         new DataExplorerQueryManagement(dataLakeMeasureManagement);
     List<DataLakeMeasure> allMeasurements = dataLakeMeasureManagement.getAllMeasurements();
     allMeasurements.forEach(measurement -> {
diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
index fe8e13571..2947b5510 100644
--- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
@@ -19,7 +19,7 @@
 /* tslint:disable */
 /* eslint-disable */
 // @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2023-10-27 10:44:54.
+// Generated using typescript-generator version 3.2.1263 on 2023-12-04 13:14:26.
 
 export class ExtensionsServiceEndpointItem {
     appId: string;
diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
index 85f9a230b..f1613ff34 100644
--- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
@@ -16,11 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 /* tslint:disable */
 /* eslint-disable */
 // @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2023-10-30 22:49:29.
+// Generated using typescript-generator version 3.2.1263 on 2023-12-04 13:14:24.
 
 export class NamedStreamPipesEntity {
     '@class':
@@ -45,10 +44,6 @@ export class NamedStreamPipesEntity {
     'includesLocales': boolean;
     'internallyManaged': boolean;
     'name': string;
-    /**
-     * @deprecated
-     */
-    'uri': string;
 
     static 'fromData'(
         data: NamedStreamPipesEntity,
@@ -78,7 +73,6 @@ export class NamedStreamPipesEntity {
         instance.includesLocales = data.includesLocales;
         instance.internallyManaged = data.internallyManaged;
         instance.name = data.name;
-        instance.uri = data.uri;
         return instance;
     }
 }
@@ -1157,6 +1151,7 @@ export class DataLakeMeasure {
     'pipelineId': string;
     'pipelineIsRunning': boolean;
     'pipelineName': string;
+    'schemaUpdateStrategy': DataLakeMeasureSchemaUpdateStrategy;
     'schemaVersion': string;
     'timestampField': string;
 
@@ -1176,6 +1171,7 @@ export class DataLakeMeasure {
         instance.pipelineId = data.pipelineId;
         instance.pipelineIsRunning = data.pipelineIsRunning;
         instance.pipelineName = data.pipelineName;
+        instance.schemaUpdateStrategy = data.schemaUpdateStrategy;
         instance.schemaVersion = data.schemaVersion;
         instance.timestampField = data.timestampField;
         return instance;
@@ -4063,6 +4059,10 @@ export type ConfigurationScope =
     | 'CONTAINER_GLOBAL_CONFIG'
     | 'PIPELINE_ELEMENT_CONFIG';
 
+export type DataLakeMeasureSchemaUpdateStrategy =
+    | 'UPDATE_SCHEMA'
+    | 'EXTEND_EXISTING_SCHEMA';
+
 export type EdgeValidationStatusType = 'COMPLETE' | 'INCOMPLETE' | 'INVALID';
 
 export type EventPropertyUnion =
diff --git a/ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.ts b/ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.ts
index f467d4f7d..9b8ef3ce1 100644
--- a/ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.ts
+++ b/ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.ts
@@ -231,6 +231,10 @@ export class AdapterStartedDialog implements OnInit {
                                     'timestamp_mapping',
                                     's0::' + this.dataLakeTimestampField,
                                 )
+                                .setOneOfStaticProperty(
+                                    'schema_update',
+                                    'Update schema',
+                                )
                                 .build();
 
                         this.pipelineTemplateService
diff --git a/ui/src/app/core-services/template/PipelineInvocationBuilder.ts b/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
index 03715cf48..9192a735f 100644
--- a/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
+++ b/ui/src/app/core-services/template/PipelineInvocationBuilder.ts
@@ -57,6 +57,26 @@ export class PipelineInvocationBuilder {
         return this;
     }
 
+    public setOneOfStaticProperty(name: string, value: string) {
+        this.pipelineTemplateInvocation.staticProperties.forEach(property => {
+            if (
+                // property instanceof OneOfStaticProperty &&
+                property['@class'] ===
+                    'org.apache.streampipes.model.staticproperty.OneOfStaticProperty' &&
+                'jsplumb_domId2' + name === property.internalName
+            ) {
+                // set selected for selected option
+                property.options.forEach(option => {
+                    if (option.name === value) {
+                        option.selected = true;
+                    }
+                });
+            }
+        });
+
+        return this;
+    }
+
     public setMappingPropertyUnary(name: string, value: string) {
         this.pipelineTemplateInvocation.staticProperties.forEach(property => {
             if (
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.ts b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.ts
index f9f12f1bc..b52f66ed8 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.ts
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.ts
@@ -180,7 +180,6 @@ export class SavePipelineComponent implements OnInit {
             entity.elementId.substring(0, lastIdIndex + 1) +
             this.jsplumbService.makeId(5);
         entity.elementId = newElementId;
-        entity.uri = newElementId;
     }
 
     storePipelineCanvasMetadata(pipelineId: string, updateMode: boolean) {
diff --git a/ui/src/app/editor/services/jsplumb.service.ts b/ui/src/app/editor/services/jsplumb.service.ts
index 5303ae18d..cd8271621 100644
--- a/ui/src/app/editor/services/jsplumb.service.ts
+++ b/ui/src/app/editor/services/jsplumb.service.ts
@@ -312,7 +312,6 @@ export class JsplumbService {
         newElementId: string,
     ) {
         pipelineElement.elementId = newElementId;
-        pipelineElement.uri = newElementId;
     }
 
     makeId(count: number) {