You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/12/13 08:31:13 UTC
(streampipes) 01/01: refactor(#2181): Refactor DuplicateFilesRenameMigration
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch 2300-redesign-fileapi-to-use-original-file-name-v2
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 830ba8f6f08f911dffe977cdbecac0a4010cd815
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Wed Dec 13 09:30:55 2023 +0100
refactor(#2181): Refactor DuplicateFilesRenameMigration
---
streampipes-service-core/pom.xml | 6 +
.../v095/DuplicateFilesRenameMigration.java | 124 ++++++++++++++++-----
.../v095/DuplicateFilesRenameMigrationTest.java | 120 ++++++++++++++++++++
3 files changed, 224 insertions(+), 26 deletions(-)
diff --git a/streampipes-service-core/pom.xml b/streampipes-service-core/pom.xml
index 09023a1d6..8d45ccb45 100644
--- a/streampipes-service-core/pom.xml
+++ b/streampipes-service-core/pom.xml
@@ -113,6 +113,12 @@
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
</dependency>
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java
index 3e1cd2e05..de17535c9 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java
@@ -22,10 +22,9 @@ import org.apache.streampipes.model.file.FileMetadata;
import org.apache.streampipes.service.core.migrations.Migration;
import org.apache.streampipes.storage.management.StorageDispatcher;
-
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
public class DuplicateFilesRenameMigration implements Migration {
@Override
@@ -35,34 +34,107 @@ public class DuplicateFilesRenameMigration implements Migration {
@Override
public void executeMigration() {
- var duplicateFiles = new HashMap<String, List<FileMetadata>>();
- var fileMetadataStorage = StorageDispatcher.INSTANCE.getNoSqlStore().getFileMetadataStorage();
- for (var metadata : fileMetadataStorage
- .getAllFileMetadataDescriptions()) {
- var originalFilename = metadata.getOriginalFilename().toLowerCase();
- if (!duplicateFiles.containsKey(originalFilename)) {
- duplicateFiles.put(originalFilename, new ArrayList<FileMetadata>());
- }
- duplicateFiles.get(originalFilename).add(metadata);
- }
-
- for (var metadataList : duplicateFiles.values()) {
- for (var i = 1; i < metadataList.size(); ++i) {
- var metadata = metadataList.get(i);
- var oldOriginalFilename = metadata.getOriginalFilename();
- var indexBeforeFileType = oldOriginalFilename.lastIndexOf('.');
- var newOriginalFilename =
- String.format("%s(%d)%s", oldOriginalFilename.substring(0, indexBeforeFileType), i + 1,
- oldOriginalFilename.substring(indexBeforeFileType));
- metadata.setOriginalFilename(newOriginalFilename);
- fileMetadataStorage.updateFileMetadata(metadata);
- }
- }
+ var fileMetadataStorage = StorageDispatcher.INSTANCE.getNoSqlStore()
+ .getFileMetadataStorage();
+
+ var filesToUpdate = getFilesToUpdate(fileMetadataStorage.getAllFileMetadataDescriptions());
+ filesToUpdate.forEach(fileMetadata -> fileMetadataStorage.updateFileMetadata(fileMetadata));
+ }
+
+
+ /**
+ * Takes the list of files and groups all files with the same name together.
+ * The result is a list for each file name that has more than one file associated with it.
+ */
+ private List<List<FileMetadata>> getListsOfFilesWithSameName(List<FileMetadata> filesWithOldName) {
+ var duplicateFileMap = filesWithOldName.stream()
+ .collect(
+ Collectors.groupingBy(file -> file.getOriginalFilename()
+ .toLowerCase()));
+ return duplicateFileMap.values()
+ .stream()
+ .filter(files -> files.size() > 1)
+ .toList();
}
@Override
public String getDescription() {
return "Rename files with duplicate names.";
}
+
+ /**
+ * Takes the files searches for duplicates and renames them and returns the files that must be updated
+ */
+ protected List<FileMetadata> getFilesToUpdate(List<FileMetadata> filesWithOldName) {
+
+ var groupsOfFilesWithSameName = getListsOfFilesWithSameName(filesWithOldName);
+
+ return groupsOfFilesWithSameName.stream()
+ .flatMap(filesWithSameName ->
+ renameFilesWithSameName(filesWithSameName).stream())
+ .collect(Collectors.toList());
+ }
+
+
+ /**
+ * Takes a list of files with the same name renames them and returns the updated list
+ */
+ private List<FileMetadata> renameFilesWithSameName(List<FileMetadata> filesWithSameName) {
+ return IntStream.range(1, filesWithSameName.size())
+ .mapToObj(i -> {
+ var metadata = filesWithSameName.get(i);
+ return renameFile(metadata, i);
+ })
+ .toList();
+ }
+
+
+ /**
+ * Takes a file and renames it with a new name based the number of occurrences of the file name
+ */
+ private FileMetadata renameFile(FileMetadata fileMetadata, int index) {
+ var oldFilename = fileMetadata.getOriginalFilename();
+
+ var fileNameWithoutType = removeFileType(oldFilename);
+ var fileTypeSuffix = getFileType(oldFilename);
+
+ var newFileName = createNewFileName(index, fileNameWithoutType, fileTypeSuffix);
+
+ fileMetadata.setOriginalFilename(newFileName);
+
+ return fileMetadata;
+ }
+
+ /**
+ * Creates the new file name for a file with a duplicate name.
+ */
+ private String createNewFileName(
+ int index,
+ String fileName,
+ String fileType
+ ) {
+ return String.format(
+ "%s(%d)%s",
+ fileName,
+ index + 1,
+ fileType
+ );
+ }
+
+ /**
+ * Returns file name without file type suffix.
+ */
+ private String removeFileType(String fileName) {
+ var indexBeforeFileType = fileName.lastIndexOf('.');
+ return fileName.substring(0, indexBeforeFileType);
+ }
+
+ /**
+ * Returns the file type a given file name.
+ */
+ private String getFileType(String fileName) {
+ var indexBeforeFileType = fileName.lastIndexOf('.');
+ return fileName.substring(indexBeforeFileType);
+ }
}
diff --git a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java
new file mode 100644
index 000000000..18b35846f
--- /dev/null
+++ b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.service.core.migrations.v095;
+
+
+import org.apache.streampipes.model.file.FileMetadata;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class DuplicateFilesRenameMigrationTest {
+
+ private DuplicateFilesRenameMigration migration;
+
+ private static final String FILE_NAME = "file.txt";
+ private static final String NEW_FILE_NAME_2 = "file(2).txt";
+ private static final String NEW_FILE_NAME_3 = "file(3).txt";
+
+ @Before
+ public void setUp() {
+ migration = new DuplicateFilesRenameMigration();
+ }
+
+ @Test
+ public void getFilesToUpdateHandlesNoDuplicates() {
+ var filesWithOldName = createFiles(List.of(
+ FILE_NAME
+ ));
+ var result = migration.getFilesToUpdate(filesWithOldName);
+ assertTrue(result.isEmpty());
+ }
+
+ @Test
+ public void getFilesToUpdateHandlesSingleDuplicate() {
+ var filesWithOldName = createFiles(List.of(
+ FILE_NAME,
+ FILE_NAME
+ ));
+ var result = migration.getFilesToUpdate(filesWithOldName);
+ assertEquals(1, result.size());
+ assertEquals(
+ NEW_FILE_NAME_2,
+ result.get(0)
+ .getOriginalFilename()
+ );
+ }
+
+ @Test
+ public void getFilesToUpdateHandlesMultipleDuplicates() {
+ var filesWithOldName = createFiles(List.of(
+ FILE_NAME,
+ FILE_NAME,
+ FILE_NAME
+ ));
+ var result = migration.getFilesToUpdate(filesWithOldName);
+ assertEquals(2, result.size());
+ assertEquals(
+ NEW_FILE_NAME_2,
+ result.get(0)
+ .getOriginalFilename()
+ );
+ assertEquals(
+ NEW_FILE_NAME_3,
+ result.get(1)
+ .getOriginalFilename()
+ );
+ }
+
+ @Test
+ public void getFilesToUpdateHandlesMixedDuplicatesAndUniqueFiles() {
+ var filesWithOldName = createFiles(List.of(
+ FILE_NAME,
+ FILE_NAME,
+ "unique.txt"
+ ));
+ var result = migration.getFilesToUpdate(filesWithOldName);
+ assertEquals(1, result.size());
+ assertEquals(
+ NEW_FILE_NAME_2,
+ result.get(0)
+ .getOriginalFilename()
+ );
+ }
+
+ /**
+ * Creates a list of FileMetadata objects from a list of file names.
+ */
+ private List<FileMetadata> createFiles(List<String> fileNames) {
+ return fileNames.stream()
+ .map(this::createFileMetadata)
+ .toList();
+ }
+
+ private FileMetadata createFileMetadata(String fileName) {
+ var fileMetadata = new FileMetadata();
+ fileMetadata.setOriginalFilename(fileName);
+ return fileMetadata;
+ }
+}
\ No newline at end of file