You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ho...@apache.org on 2026/08/31 07:35:14 UTC
(streampipes) 02/02: DB migration
This is an automated email from the ASF dual-hosted git repository.
JHoelli pushed a commit to branch 4738-migrate-stored-metadata-to-dataset-naming
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 81effc45d2783e0b39bbce8024d2ad8942a59a40
Author: Jacqueline Höllig <j....@gmx.net>
AuthorDate: Mon Aug 31 09:34:53 2026 +0200
DB migration
---
...rateDataLakeDatabaseToDatasetMigrationTest.java | 102 +++++++++++++++++++
...nameDataLakeMetadataToDatasetMigrationTest.java | 110 +++++++++++++++++++++
.../streampipes/storage/couchdb/utils/Utils.java | 3 +-
3 files changed, 214 insertions(+), 1 deletion(-)
diff --git a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/MigrateDataLakeDatabaseToDatasetMigrationTest.java b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/MigrateDataLakeDatabaseToDatasetMigrationTest.java
new file mode 100644
index 0000000000..decd5915e7
--- /dev/null
+++ b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/MigrateDataLakeDatabaseToDatasetMigrationTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.service.core.migrations.v099;
+
+import org.apache.streampipes.storage.couchdb.utils.Utils;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class MigrateDataLakeDatabaseToDatasetMigrationTest {
+
+ private final Map<String, Boolean> existingDatabases = new HashMap<>();
+ private final Map<String, Integer> documentCounts = new HashMap<>();
+ private boolean replicated;
+ private MigrateDataLakeDatabaseToDatasetMigration migration;
+
+ @BeforeEach
+ void setUp() {
+ this.replicated = false;
+ this.migration = new MigrateDataLakeDatabaseToDatasetMigration() {
+ @Override
+ protected boolean databaseExists(String databaseName) {
+ return existingDatabases.getOrDefault(databaseName, false);
+ }
+
+ @Override
+ protected int getDocumentCount(String databaseName) {
+ return documentCounts.getOrDefault(databaseName, 0);
+ }
+
+ @Override
+ public void executeMigration() {
+ MigrateDataLakeDatabaseToDatasetMigrationTest.this.replicated = true;
+ }
+ };
+ }
+
+ @Test
+ void shouldExecuteReturnsFalseWhenLegacyDatabaseDoesNotExist() {
+ existingDatabases.put(Utils.LEGACY_DATA_LAKE_DB_NAME, false);
+
+ assertFalse(migration.shouldExecute());
+ }
+
+ @Test
+ void shouldExecuteReturnsTrueWhenDatasetDatabaseDoesNotExistYet() {
+ existingDatabases.put(Utils.LEGACY_DATA_LAKE_DB_NAME, true);
+ existingDatabases.put(Utils.DATA_LAKE_DB_NAME, false);
+
+ assertTrue(migration.shouldExecute());
+ }
+
+ @Test
+ void shouldExecuteReturnsTrueWhenDatasetDatabaseHasFewerDocuments() {
+ existingDatabases.put(Utils.LEGACY_DATA_LAKE_DB_NAME, true);
+ existingDatabases.put(Utils.DATA_LAKE_DB_NAME, true);
+ documentCounts.put(Utils.LEGACY_DATA_LAKE_DB_NAME, 4);
+ documentCounts.put(Utils.DATA_LAKE_DB_NAME, 1);
+
+ assertTrue(migration.shouldExecute());
+ }
+
+ @Test
+ void shouldExecuteReturnsFalseWhenDatasetDatabaseAlreadyContainsAllDocuments() {
+ existingDatabases.put(Utils.LEGACY_DATA_LAKE_DB_NAME, true);
+ existingDatabases.put(Utils.DATA_LAKE_DB_NAME, true);
+ documentCounts.put(Utils.LEGACY_DATA_LAKE_DB_NAME, 2);
+ documentCounts.put(Utils.DATA_LAKE_DB_NAME, 2);
+
+ assertFalse(migration.shouldExecute());
+ }
+
+ @Test
+ void executeMigrationReplicatesLegacyDatabase() throws IOException {
+ migration.executeMigration();
+
+ assertTrue(replicated);
+ }
+}
diff --git a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/RenameDataLakeMetadataToDatasetMigrationTest.java b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/RenameDataLakeMetadataToDatasetMigrationTest.java
new file mode 100644
index 0000000000..889f7f5121
--- /dev/null
+++ b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v099/RenameDataLakeMetadataToDatasetMigrationTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.service.core.migrations.v099;
+
+import org.apache.streampipes.model.client.user.Permission;
+import org.apache.streampipes.storage.api.user.IPermissionStorage;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class RenameDataLakeMetadataToDatasetMigrationTest {
+
+ private IPermissionStorage permissionStorage;
+ private RenameDataLakeMetadataToDatasetMigration migration;
+
+ @BeforeEach
+ void setUp() {
+ this.permissionStorage = mock(IPermissionStorage.class);
+ this.migration = new RenameDataLakeMetadataToDatasetMigration(permissionStorage);
+ }
+
+ @Test
+ void shouldExecuteReturnsTrueWhenOldDataLakeClassNamesExist() {
+ var permission = permissionWithClassName(
+ RenameDataLakeMetadataToDatasetMigration.DATA_LAKE_MEASURE_CLASS
+ );
+ when(permissionStorage.findAll()).thenReturn(List.of(permission));
+
+ assertTrue(migration.shouldExecute());
+ }
+
+ @Test
+ void shouldExecuteReturnsFalseWhenMetadataAlreadyUsesDatasetNaming() {
+ var permission = permissionWithClassName(
+ RenameDataLakeMetadataToDatasetMigration.DATASET_MEASURE_CLASS
+ );
+ when(permissionStorage.findAll()).thenReturn(List.of(permission));
+
+ assertFalse(migration.shouldExecute());
+ }
+
+ @Test
+ void executeMigrationRenamesDatasetAndWidgetPermissionMetadata() throws IOException {
+ var measurePermission = permissionWithClassName(
+ RenameDataLakeMetadataToDatasetMigration.DATA_LAKE_MEASURE_CLASS
+ );
+ var widgetPermission = permissionWithClassName(
+ RenameDataLakeMetadataToDatasetMigration.DATA_LAKE_WIDGET_CLASS
+ );
+ var unchangedPermission = permissionWithClassName("org.apache.streampipes.model.dashboard.DashboardModel");
+
+ when(permissionStorage.findAll()).thenReturn(List.of(
+ measurePermission,
+ widgetPermission,
+ unchangedPermission
+ ));
+
+ migration.executeMigration();
+
+ assertEquals(
+ RenameDataLakeMetadataToDatasetMigration.DATASET_MEASURE_CLASS,
+ measurePermission.getObjectClassName()
+ );
+ assertEquals(
+ RenameDataLakeMetadataToDatasetMigration.DATASET_WIDGET_CLASS,
+ widgetPermission.getObjectClassName()
+ );
+ assertEquals(
+ "org.apache.streampipes.model.dashboard.DashboardModel",
+ unchangedPermission.getObjectClassName()
+ );
+
+ verify(permissionStorage).updateElement(measurePermission);
+ verify(permissionStorage).updateElement(widgetPermission);
+ verify(permissionStorage, never()).updateElement(unchangedPermission);
+ }
+
+ private Permission permissionWithClassName(String className) {
+ var permission = new Permission();
+ permission.setObjectClassName(className);
+ return permission;
+ }
+}
diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/utils/Utils.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/utils/Utils.java
index d0eabc1a5c..fc82c437ad 100644
--- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/utils/Utils.java
+++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/utils/Utils.java
@@ -33,7 +33,8 @@ import org.lightcouch.CouchDbProperties;
public class Utils {
public static final String USER_DB_NAME = "users";
- public static final String DATA_LAKE_DB_NAME = "data-lake";
+ public static final String LEGACY_DATA_LAKE_DB_NAME = "data-lake";
+ public static final String DATA_LAKE_DB_NAME = "dataset";
public static CouchDbClient getCouchDbDataProcessorDescriptionClient() {
return getCouchDbGsonClient("data-processor");