You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by da...@apache.org on 2023/07/06 13:39:56 UTC

[flink] branch release-1.16 updated: [FLINK-27415][Connectors / FileSystem] Read empty csv file throws exception in FileSystem table connector (#20006)

This is an automated email from the ASF dual-hosted git repository.

dannycranmer pushed a commit to branch release-1.16
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.16 by this push:
     new ad1b739358e [FLINK-27415][Connectors / FileSystem] Read empty csv file throws exception in FileSystem table connector (#20006)
ad1b739358e is described below

commit ad1b739358efa3c58187f80ead80a81377029c48
Author: Jia Liu <li...@gmail.com>
AuthorDate: Wed Jul 5 02:35:58 2023 +0800

    [FLINK-27415][Connectors / FileSystem] Read empty csv file throws exception in FileSystem table connector (#20006)
    
    * [FLINK-27415][Connectors / FileSystem] Remove size limit, so `StreamFormatAdapter` can read empty files.
    
    ---------
    
    Co-authored-by: Weijie Guo <re...@163.com>
---
 .../file/src/impl/StreamFormatAdapter.java         |  1 -
 .../file/src/impl/StreamFormatAdapterTest.java     | 22 ++++++++++++++++++++++
 .../formats/csv/CsvFilesystemBatchITCase.java      | 11 +++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapter.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapter.java
index d52de0da1f7..39ed50e4236 100644
--- a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapter.java
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapter.java
@@ -261,7 +261,6 @@ public final class StreamFormatAdapter<T> implements BulkFormat<T, FileSourceSpl
         private int remainingInBatch;
 
         TrackingFsDataInputStream(FSDataInputStream stream, long fileLength, int batchSize) {
-            checkArgument(fileLength > 0L);
             checkArgument(batchSize > 0);
             this.stream = stream;
             this.fileLength = fileLength;
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapterTest.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapterTest.java
index cb39c58e763..61ca188e234 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapterTest.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapterTest.java
@@ -27,9 +27,11 @@ import org.apache.flink.connector.file.src.reader.BulkFormat;
 import org.apache.flink.connector.file.src.reader.SimpleStreamFormat;
 import org.apache.flink.connector.file.src.reader.StreamFormat;
 import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.Path;
 
 import org.junit.jupiter.api.Test;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -83,6 +85,26 @@ class StreamFormatAdapterTest extends AdapterTestBase<StreamFormat<Integer>> {
         simpleReadTest(20);
     }
 
+    @Test
+    void testReadEmptyFile() throws IOException {
+        final StreamFormatAdapter<Integer> format =
+                new StreamFormatAdapter<>(new CheckpointedIntFormat());
+
+        final File emptyFile = new File(tmpDir.toFile(), "testFile-empty");
+        emptyFile.createNewFile();
+        Path emptyFilePath = Path.fromLocalFile(emptyFile);
+
+        final BulkFormat.Reader<Integer> reader =
+                format.createReader(
+                        new Configuration(),
+                        new FileSourceSplit("test-id", emptyFilePath, 0L, 0, 0L, 0));
+
+        final List<Integer> result = new ArrayList<>();
+        readNumbers(reader, result, 0);
+
+        assertThat(result).isEmpty();
+    }
+
     private void simpleReadTest(int batchSize) throws IOException {
         final Configuration config = new Configuration();
         config.set(StreamFormat.FETCH_IO_SIZE, new MemorySize(batchSize));
diff --git a/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java b/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java
index 9938c9b3e0a..832c6d5b9a2 100644
--- a/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java
+++ b/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java
@@ -30,6 +30,7 @@ import java.io.File;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 /** ITCase to test csv format for {@link CsvFileFormatFactory} in batch mode. */
@@ -114,5 +115,15 @@ public class CsvFilesystemBatchITCase {
                     "select * from nonPartitionedTable",
                     Arrays.asList(Row.of("x5", 5, 1, 1), Row.of("x5", 5, 2, 2)));
         }
+
+        @Test
+        public void testEmpty() throws Exception {
+            String path = new URI(resultPath()).getPath();
+            new File(path).mkdirs();
+            File file = new File(path, "test_file");
+            file.createNewFile();
+
+            check("select * from nonPartitionedTable", Collections.emptyList());
+        }
     }
 }