[incubator-streampipes] branch STREAMPIPES-379 created (now abc4413)

[ze...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a change to branch STREAMPIPES-379
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git.


      at abc4413  Remove bom in csv files

This branch includes the following new commits:

     new abc4413  Remove bom in csv files

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[incubator-streampipes] 01/01: Remove bom in csv files

[ze...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch STREAMPIPES-379
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git

commit abc44137dea78964ddd3663116493c9f8726e6fd
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Tue Jun 8 09:33:59 2021 +0200

    Remove bom in csv files
---
 .../streampipes/manager/file/FileManager.java      | 11 ++++
 .../streampipes/manager/file/TestFileManager.java  | 64 ++++++++++++++++++++++
 2 files changed, 75 insertions(+)

diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java
index af1e7a6..382ee3b 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java
@@ -17,6 +17,7 @@
  */
 package org.apache.streampipes.manager.file;
 
+import org.apache.commons.io.input.BOMInputStream;
 import org.apache.streampipes.model.client.file.FileMetadata;
 import org.apache.streampipes.storage.api.IFileMetadataStorage;
 import org.apache.streampipes.storage.management.StorageDispatcher;
@@ -33,6 +34,11 @@ public class FileManager {
                                InputStream fileInputStream) throws IOException {
 
     String filetype = filename.substring(filename.lastIndexOf(".") + 1);
+
+    if ("csv".equals(filetype)) {
+      fileInputStream = removeBom(fileInputStream);
+    }
+
     String internalFilename = makeInternalFilename(filetype);
     FileMetadata fileMetadata = makeFileMetadata(user, filename, internalFilename, filetype);
     new FileHandler().storeFile(internalFilename, fileInputStream);
@@ -79,4 +85,9 @@ public class FileManager {
   private static String makeInternalFilename(String filetype) {
     return UUID.randomUUID().toString() + "." + filetype;
   }
+
+  public static InputStream removeBom(InputStream stream) {
+    return new BOMInputStream(stream);
+//      return stream;
+  }
 }
diff --git a/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java
new file mode 100644
index 0000000..14b449e
--- /dev/null
+++ b/streampipes-pipeline-management/src/test/java/org/apache/streampipes/manager/file/TestFileManager.java
@@ -0,0 +1,64 @@
+package org.apache.streampipes.manager.file;
+
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.*;
+
+/*
+Copyright 2021 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+public class TestFileManager {
+
+    @Test
+    public void testRemoveBomWithNoBom() throws IOException {
+        String expected = "test";
+        InputStream inputStream = IOUtils.toInputStream(expected, StandardCharsets.UTF_8);
+        InputStream resultStream = FileManager.removeBom(inputStream);
+        String resultString = IOUtils.toString(resultStream, StandardCharsets.UTF_8);
+
+        assertEquals(expected, resultString);
+    }
+
+    @Test
+    public void testRemoveBomWithBom() throws IOException {
+        String expected = "test";
+        String UTF8_BOM = "\uFEFF";
+        String inputString = UTF8_BOM + expected;
+        InputStream inputStream = IOUtils.toInputStream(inputString, StandardCharsets.UTF_8);
+        InputStream resultStream = FileManager.removeBom(inputStream);
+        String resultString = IOUtils.toString(resultStream, StandardCharsets.UTF_8);
+
+        assertEquals(expected, resultString);
+    }
+
+    @Test
+    public void testRemoveBomWithBomAndUmlauts() throws IOException {
+        String expected = "testäüö";
+        String UTF8_BOM = "\uFEFF";
+        String inputString = UTF8_BOM + expected;
+        InputStream inputStream = IOUtils.toInputStream(inputString, StandardCharsets.UTF_8);
+        InputStream resultStream = FileManager.removeBom(inputStream);
+        String resultString = IOUtils.toString(resultStream, StandardCharsets.ISO_8859_1);
+
+        assertEquals(expected, resultString);
+    }
+}
\ No newline at end of file