You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by da...@apache.org on 2023/07/06 13:39:56 UTC
[flink] branch release-1.16 updated: [FLINK-27415][Connectors / FileSystem] Read empty csv file throws exception in FileSystem table connector (#20006)
This is an automated email from the ASF dual-hosted git repository.
dannycranmer pushed a commit to branch release-1.16
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.16 by this push:
new ad1b739358e [FLINK-27415][Connectors / FileSystem] Read empty csv file throws exception in FileSystem table connector (#20006)
ad1b739358e is described below
commit ad1b739358efa3c58187f80ead80a81377029c48
Author: Jia Liu <li...@gmail.com>
AuthorDate: Wed Jul 5 02:35:58 2023 +0800
[FLINK-27415][Connectors / FileSystem] Read empty csv file throws exception in FileSystem table connector (#20006)
* [FLINK-27415][Connectors / FileSystem] Remove size limit, so `StreamFormatAdapter` can read empty files.
---------
Co-authored-by: Weijie Guo <re...@163.com>
---
.../file/src/impl/StreamFormatAdapter.java | 1 -
.../file/src/impl/StreamFormatAdapterTest.java | 22 ++++++++++++++++++++++
.../formats/csv/CsvFilesystemBatchITCase.java | 11 +++++++++++
3 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapter.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapter.java
index d52de0da1f7..39ed50e4236 100644
--- a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapter.java
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapter.java
@@ -261,7 +261,6 @@ public final class StreamFormatAdapter<T> implements BulkFormat<T, FileSourceSpl
private int remainingInBatch;
TrackingFsDataInputStream(FSDataInputStream stream, long fileLength, int batchSize) {
- checkArgument(fileLength > 0L);
checkArgument(batchSize > 0);
this.stream = stream;
this.fileLength = fileLength;
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapterTest.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapterTest.java
index cb39c58e763..61ca188e234 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapterTest.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/impl/StreamFormatAdapterTest.java
@@ -27,9 +27,11 @@ import org.apache.flink.connector.file.src.reader.BulkFormat;
import org.apache.flink.connector.file.src.reader.SimpleStreamFormat;
import org.apache.flink.connector.file.src.reader.StreamFormat;
import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.Path;
import org.junit.jupiter.api.Test;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -83,6 +85,26 @@ class StreamFormatAdapterTest extends AdapterTestBase<StreamFormat<Integer>> {
simpleReadTest(20);
}
+ @Test
+ void testReadEmptyFile() throws IOException {
+ final StreamFormatAdapter<Integer> format =
+ new StreamFormatAdapter<>(new CheckpointedIntFormat());
+
+ final File emptyFile = new File(tmpDir.toFile(), "testFile-empty");
+ emptyFile.createNewFile();
+ Path emptyFilePath = Path.fromLocalFile(emptyFile);
+
+ final BulkFormat.Reader<Integer> reader =
+ format.createReader(
+ new Configuration(),
+ new FileSourceSplit("test-id", emptyFilePath, 0L, 0, 0L, 0));
+
+ final List<Integer> result = new ArrayList<>();
+ readNumbers(reader, result, 0);
+
+ assertThat(result).isEmpty();
+ }
+
private void simpleReadTest(int batchSize) throws IOException {
final Configuration config = new Configuration();
config.set(StreamFormat.FETCH_IO_SIZE, new MemorySize(batchSize));
diff --git a/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java b/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java
index 9938c9b3e0a..832c6d5b9a2 100644
--- a/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java
+++ b/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvFilesystemBatchITCase.java
@@ -30,6 +30,7 @@ import java.io.File;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
/** ITCase to test csv format for {@link CsvFileFormatFactory} in batch mode. */
@@ -114,5 +115,15 @@ public class CsvFilesystemBatchITCase {
"select * from nonPartitionedTable",
Arrays.asList(Row.of("x5", 5, 1, 1), Row.of("x5", 5, 2, 2)));
}
+
+ @Test
+ public void testEmpty() throws Exception {
+ String path = new URI(resultPath()).getPath();
+ new File(path).mkdirs();
+ File file = new File(path, "test_file");
+ file.createNewFile();
+
+ check("select * from nonPartitionedTable", Collections.emptyList());
+ }
}
}