You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/12/13 08:31:13 UTC

(streampipes) 01/01: refactor(#2181): Refactor DuplicateFilesRenameMigration

This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch 2300-redesign-fileapi-to-use-original-file-name-v2
in repository https://gitbox.apache.org/repos/asf/streampipes.git

commit 830ba8f6f08f911dffe977cdbecac0a4010cd815
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Wed Dec 13 09:30:55 2023 +0100

    refactor(#2181): Refactor DuplicateFilesRenameMigration
---
 streampipes-service-core/pom.xml                   |   6 +
 .../v095/DuplicateFilesRenameMigration.java        | 124 ++++++++++++++++-----
 .../v095/DuplicateFilesRenameMigrationTest.java    | 120 ++++++++++++++++++++
 3 files changed, 224 insertions(+), 26 deletions(-)

diff --git a/streampipes-service-core/pom.xml b/streampipes-service-core/pom.xml
index 09023a1d6..8d45ccb45 100644
--- a/streampipes-service-core/pom.xml
+++ b/streampipes-service-core/pom.xml
@@ -113,6 +113,12 @@
             <groupId>jakarta.annotation</groupId>
             <artifactId>jakarta.annotation-api</artifactId>
         </dependency>
+      <!-- Test dependencies -->
+      <dependency>
+        <groupId>junit</groupId>
+        <artifactId>junit</artifactId>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
 
     <build>
diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java
index 3e1cd2e05..de17535c9 100644
--- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java
+++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigration.java
@@ -22,10 +22,9 @@ import org.apache.streampipes.model.file.FileMetadata;
 import org.apache.streampipes.service.core.migrations.Migration;
 import org.apache.streampipes.storage.management.StorageDispatcher;
 
-
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 public class DuplicateFilesRenameMigration implements Migration {
   @Override
@@ -35,34 +34,107 @@ public class DuplicateFilesRenameMigration implements Migration {
 
   @Override
   public void executeMigration() {
-    var duplicateFiles = new HashMap<String, List<FileMetadata>>();
-    var fileMetadataStorage = StorageDispatcher.INSTANCE.getNoSqlStore().getFileMetadataStorage();
-    for (var metadata : fileMetadataStorage
-        .getAllFileMetadataDescriptions()) {
-      var originalFilename = metadata.getOriginalFilename().toLowerCase();
-      if (!duplicateFiles.containsKey(originalFilename)) {
-        duplicateFiles.put(originalFilename, new ArrayList<FileMetadata>());
-      }
-      duplicateFiles.get(originalFilename).add(metadata);
-    }
-
-    for (var metadataList : duplicateFiles.values()) {
-      for (var i = 1; i < metadataList.size(); ++i) {
-        var metadata = metadataList.get(i);
-        var oldOriginalFilename = metadata.getOriginalFilename();
-        var indexBeforeFileType = oldOriginalFilename.lastIndexOf('.');
-        var newOriginalFilename =
-            String.format("%s(%d)%s", oldOriginalFilename.substring(0, indexBeforeFileType), i + 1,
-                oldOriginalFilename.substring(indexBeforeFileType));
-        metadata.setOriginalFilename(newOriginalFilename);
-        fileMetadataStorage.updateFileMetadata(metadata);
-      }
-    }
+    var fileMetadataStorage = StorageDispatcher.INSTANCE.getNoSqlStore()
+                                                        .getFileMetadataStorage();
+
+    var filesToUpdate = getFilesToUpdate(fileMetadataStorage.getAllFileMetadataDescriptions());
 
+    filesToUpdate.forEach(fileMetadata -> fileMetadataStorage.updateFileMetadata(fileMetadata));
+  }
+
+
+  /**
+   * Takes the list of files and groups all files with the same name together.
+   * The result is a list for each file name that has more than one file associated with it.
+   */
+  private List<List<FileMetadata>> getListsOfFilesWithSameName(List<FileMetadata> filesWithOldName) {
+    var duplicateFileMap = filesWithOldName.stream()
+                                           .collect(
+                                               Collectors.groupingBy(file -> file.getOriginalFilename()
+                                                                                 .toLowerCase()));
+    return duplicateFileMap.values()
+                           .stream()
+                           .filter(files -> files.size() > 1)
+                           .toList();
   }
 
   @Override
   public String getDescription() {
     return "Rename files with duplicate names.";
   }
+
+  /**
+   * Takes the files searches for duplicates and renames them and returns the files that must be updated
+   */
+  protected List<FileMetadata> getFilesToUpdate(List<FileMetadata> filesWithOldName) {
+
+    var groupsOfFilesWithSameName = getListsOfFilesWithSameName(filesWithOldName);
+
+    return groupsOfFilesWithSameName.stream()
+                                    .flatMap(filesWithSameName ->
+                                                 renameFilesWithSameName(filesWithSameName).stream())
+                                    .collect(Collectors.toList());
+  }
+
+
+  /**
+   * Takes a list of files with the same name renames them and returns the updated list
+   */
+  private List<FileMetadata> renameFilesWithSameName(List<FileMetadata> filesWithSameName) {
+    return IntStream.range(1, filesWithSameName.size())
+                    .mapToObj(i -> {
+                      var metadata = filesWithSameName.get(i);
+                      return renameFile(metadata, i);
+                    })
+                    .toList();
+  }
+
+
+  /**
+   * Takes a file and renames it with a new name based the number of occurrences of the file name
+   */
+  private FileMetadata renameFile(FileMetadata fileMetadata, int index) {
+    var oldFilename = fileMetadata.getOriginalFilename();
+
+    var fileNameWithoutType = removeFileType(oldFilename);
+    var fileTypeSuffix = getFileType(oldFilename);
+
+    var newFileName = createNewFileName(index, fileNameWithoutType, fileTypeSuffix);
+
+    fileMetadata.setOriginalFilename(newFileName);
+
+    return fileMetadata;
+  }
+
+  /**
+   * Creates the new file name for a file with a duplicate name.
+   */
+  private String createNewFileName(
+      int index,
+      String fileName,
+      String fileType
+  ) {
+    return String.format(
+        "%s(%d)%s",
+        fileName,
+        index + 1,
+        fileType
+    );
+  }
+
+  /**
+   * Returns file name without file type suffix.
+   */
+  private String removeFileType(String fileName) {
+    var indexBeforeFileType = fileName.lastIndexOf('.');
+    return fileName.substring(0, indexBeforeFileType);
+  }
+
+  /**
+   * Returns the file type a given file name.
+   */
+  private String getFileType(String fileName) {
+    var indexBeforeFileType = fileName.lastIndexOf('.');
+    return fileName.substring(indexBeforeFileType);
+  }
 }
diff --git a/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java
new file mode 100644
index 000000000..18b35846f
--- /dev/null
+++ b/streampipes-service-core/src/test/java/org/apache/streampipes/service/core/migrations/v095/DuplicateFilesRenameMigrationTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.service.core.migrations.v095;
+
+
+import org.apache.streampipes.model.file.FileMetadata;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class DuplicateFilesRenameMigrationTest {
+
+  private DuplicateFilesRenameMigration migration;
+
+  private static final String FILE_NAME = "file.txt";
+  private static final String NEW_FILE_NAME_2 = "file(2).txt";
+  private static final String NEW_FILE_NAME_3 = "file(3).txt";
+
+  @Before
+  public void setUp() {
+    migration = new DuplicateFilesRenameMigration();
+  }
+
+  @Test
+  public void getFilesToUpdateHandlesNoDuplicates() {
+    var filesWithOldName = createFiles(List.of(
+        FILE_NAME
+    ));
+    var result = migration.getFilesToUpdate(filesWithOldName);
+    assertTrue(result.isEmpty());
+  }
+
+  @Test
+  public void getFilesToUpdateHandlesSingleDuplicate() {
+    var filesWithOldName = createFiles(List.of(
+        FILE_NAME,
+        FILE_NAME
+    ));
+    var result = migration.getFilesToUpdate(filesWithOldName);
+    assertEquals(1, result.size());
+    assertEquals(
+        NEW_FILE_NAME_2,
+        result.get(0)
+              .getOriginalFilename()
+    );
+  }
+
+  @Test
+  public void getFilesToUpdateHandlesMultipleDuplicates() {
+    var filesWithOldName = createFiles(List.of(
+        FILE_NAME,
+        FILE_NAME,
+        FILE_NAME
+    ));
+    var result = migration.getFilesToUpdate(filesWithOldName);
+    assertEquals(2, result.size());
+    assertEquals(
+        NEW_FILE_NAME_2,
+        result.get(0)
+              .getOriginalFilename()
+    );
+    assertEquals(
+        NEW_FILE_NAME_3,
+        result.get(1)
+              .getOriginalFilename()
+    );
+  }
+
+  @Test
+  public void getFilesToUpdateHandlesMixedDuplicatesAndUniqueFiles() {
+    var filesWithOldName = createFiles(List.of(
+        FILE_NAME,
+        FILE_NAME,
+        "unique.txt"
+    ));
+    var result = migration.getFilesToUpdate(filesWithOldName);
+    assertEquals(1, result.size());
+    assertEquals(
+        NEW_FILE_NAME_2,
+        result.get(0)
+              .getOriginalFilename()
+    );
+  }
+
+  /**
+   * Creates a list of FileMetadata objects from a list of file names.
+   */
+  private List<FileMetadata> createFiles(List<String> fileNames) {
+    return fileNames.stream()
+                    .map(this::createFileMetadata)
+                    .toList();
+  }
+
+  private FileMetadata createFileMetadata(String fileName) {
+    var fileMetadata = new FileMetadata();
+    fileMetadata.setOriginalFilename(fileName);
+    return fileMetadata;
+  }
+}
\ No newline at end of file