You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by zh...@apache.org on 2022/06/15 02:57:12 UTC

[flink] branch master updated (6fe543bc856 -> 0cf5abacbb2)

This is an automated email from the ASF dual-hosted git repository.

zhuzh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


    from 6fe543bc856 [FLINK-24960][yarn-tests] change order of statements to avoid race condition
     new 0cf8208ede5 [hotfix] Rework BinaryInputFormatTest to be based on AssertJ
     new 0cf5abacbb2 [FLINK-28018][core] correct the start index for creating empty splits in BinaryInputFormat#createInputSplits

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../flink/api/common/io/BinaryInputFormat.java     |   2 +-
 .../flink/api/common/io/BinaryInputFormatTest.java | 115 +++++++++++++++------
 2 files changed, 84 insertions(+), 33 deletions(-)


[flink] 02/02: [FLINK-28018][core] correct the start index for creating empty splits in BinaryInputFormat#createInputSplits

Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 0cf5abacbb2777b507d0bccfe5fd0239e62c9339
Author: zhouli <zh...@foxmail.com>
AuthorDate: Tue Jun 14 15:40:46 2022 +0800

    [FLINK-28018][core] correct the start index for creating empty splits in BinaryInputFormat#createInputSplits
---
 .../flink/api/common/io/BinaryInputFormat.java     |  2 +-
 .../flink/api/common/io/BinaryInputFormatTest.java | 62 ++++++++++++++++++----
 2 files changed, 53 insertions(+), 11 deletions(-)

diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/BinaryInputFormat.java b/flink-core/src/main/java/org/apache/flink/api/common/io/BinaryInputFormat.java
index 93bf43d72c8..75f4bb2a7fa 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/BinaryInputFormat.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/BinaryInputFormat.java
@@ -147,7 +147,7 @@ public abstract class BinaryInputFormat<T> extends FileInputFormat<T>
             FileStatus last = files.get(files.size() - 1);
             final BlockLocation[] blocks =
                     last.getPath().getFileSystem().getFileBlockLocations(last, 0, last.getLen());
-            for (int index = files.size(); index < minNumSplits; index++) {
+            for (int index = inputSplits.size(); index < minNumSplits; index++) {
                 inputSplits.add(
                         new FileInputSplit(
                                 index, last.getPath(), last.getLen(), 0, blocks[0].getHosts()));
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/BinaryInputFormatTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/BinaryInputFormatTest.java
index 480995f1132..7de496606ae 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/BinaryInputFormatTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/BinaryInputFormatTest.java
@@ -24,7 +24,9 @@ import org.apache.flink.core.fs.FileInputSplit;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.types.Record;
 
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -35,6 +37,8 @@ import static org.assertj.core.api.Fail.fail;
 
 public class BinaryInputFormatTest {
 
+    @Rule public TemporaryFolder tempFolder = new TemporaryFolder();
+
     private static final class MyBinaryInputFormat extends BinaryInputFormat<Record> {
 
         private static final long serialVersionUID = 1L;
@@ -52,17 +56,14 @@ public class BinaryInputFormatTest {
 
     @Test
     public void testCreateInputSplitsWithOneFile() throws IOException {
-        // create temporary file with 3 blocks
-        final File tempFile = File.createTempFile("binary_input_format_test", "tmp");
-        tempFile.deleteOnExit();
         final int blockInfoSize = new BlockInfo().getInfoSize();
         final int blockSize = blockInfoSize + 8;
         final int numBlocks = 3;
-        FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
-        for (int i = 0; i < blockSize * numBlocks; i++) {
-            fileOutputStream.write(new byte[] {1});
-        }
-        fileOutputStream.close();
+
+        // create temporary file with 3 blocks
+        final File tempFile =
+                createBinaryInputFile(
+                        "test_create_input_splits_with_one_file", blockSize, numBlocks);
 
         final Configuration config = new Configuration();
         config.setLong("input.block_size", blockSize + 10);
@@ -169,11 +170,52 @@ public class BinaryInputFormatTest {
                 .isEqualTo(blockSize * (numBlocks1 + numBlocks2));
     }
 
+    @Test
+    public void testCreateInputSplitsWithEmptySplit() throws IOException {
+        final int blockInfoSize = new BlockInfo().getInfoSize();
+        final int blockSize = blockInfoSize + 8;
+        final int numBlocks = 3;
+        final int minNumSplits = 5;
+
+        // create temporary file with 3 blocks
+        final File tempFile =
+                createBinaryInputFile(
+                        "test_create_input_splits_with_empty_split", blockSize, numBlocks);
+
+        final Configuration config = new Configuration();
+        config.setLong("input.block_size", blockSize + 10);
+
+        final BinaryInputFormat<Record> inputFormat = new MyBinaryInputFormat();
+        inputFormat.setFilePath(tempFile.toURI().toString());
+        inputFormat.setBlockSize(blockSize);
+
+        inputFormat.configure(config);
+
+        FileInputSplit[] inputSplits = inputFormat.createInputSplits(minNumSplits);
+
+        assertThat(inputSplits).as("Returns requested numbers of splits.").hasSize(minNumSplits);
+
+        assertThat(inputSplits[0].getLength())
+                .as("1. split should have block size length.")
+                .isEqualTo(blockSize);
+
+        assertThat(inputSplits[1].getLength())
+                .as("2. split should have block size length.")
+                .isEqualTo(blockSize);
+
+        assertThat(inputSplits[2].getLength())
+                .as("3. split should have block size length.")
+                .isEqualTo(blockSize);
+
+        assertThat(inputSplits[3].getLength()).as("4. split should be an empty split.").isZero();
+
+        assertThat(inputSplits[4].getLength()).as("5. split should be an empty split.").isZero();
+    }
+
     /** Creates a temp file with a certain number of blocks of a certain size. */
     private File createBinaryInputFile(String fileName, int blockSize, int numBlocks)
             throws IOException {
-        final File tempFile = File.createTempFile(fileName, "tmp");
-        tempFile.deleteOnExit();
+        final File tempFile = tempFolder.newFile(fileName);
         try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) {
             for (int i = 0; i < blockSize * numBlocks; i++) {
                 fileOutputStream.write(new byte[] {1});


[flink] 01/02: [hotfix] Rework BinaryInputFormatTest to be based on AssertJ

Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 0cf8208ede55097987abb243874d670ed5f504ae
Author: zhouli <zh...@foxmail.com>
AuthorDate: Tue Jun 14 15:35:39 2022 +0800

    [hotfix] Rework BinaryInputFormatTest to be based on AssertJ
---
 .../flink/api/common/io/BinaryInputFormatTest.java | 53 +++++++++++++---------
 1 file changed, 31 insertions(+), 22 deletions(-)

diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/BinaryInputFormatTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/BinaryInputFormatTest.java
index a6c9140619e..480995f1132 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/BinaryInputFormatTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/BinaryInputFormatTest.java
@@ -24,13 +24,15 @@ import org.apache.flink.core.fs.FileInputSplit;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.types.Record;
 
-import org.junit.Assert;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Fail.fail;
+
 public class BinaryInputFormatTest {
 
     private static final class MyBinaryInputFormat extends BinaryInputFormat<Record> {
@@ -73,13 +75,19 @@ public class BinaryInputFormatTest {
 
         FileInputSplit[] inputSplits = inputFormat.createInputSplits(numBlocks);
 
-        Assert.assertEquals("Returns requested numbers of splits.", numBlocks, inputSplits.length);
-        Assert.assertEquals(
-                "1. split has block size length.", blockSize, inputSplits[0].getLength());
-        Assert.assertEquals(
-                "2. split has block size length.", blockSize, inputSplits[1].getLength());
-        Assert.assertEquals(
-                "3. split has block size length.", blockSize, inputSplits[2].getLength());
+        assertThat(inputSplits).as("Returns requested numbers of splits.").hasSize(numBlocks);
+
+        assertThat(inputSplits[0].getLength())
+                .as("1. split should have block size length.")
+                .isEqualTo(blockSize);
+
+        assertThat(inputSplits[1].getLength())
+                .as("2. split should have block size length.")
+                .isEqualTo(blockSize);
+
+        assertThat(inputSplits[2].getLength())
+                .as("3. split should have block size length.")
+                .isEqualTo(blockSize);
     }
 
     @Test
@@ -106,23 +114,24 @@ public class BinaryInputFormatTest {
         int numSplitsFile1 = 0;
         int numSplitsFile2 = 0;
 
-        Assert.assertEquals(
-                "Returns requested numbers of splits.", numBlocksTotal, inputSplits.length);
+        assertThat(inputSplits).as("Returns requested numbers of splits.").hasSize(numBlocksTotal);
+
         for (int i = 0; i < inputSplits.length; i++) {
-            Assert.assertEquals(
-                    String.format("%d. split has block size length.", i),
-                    blockSize,
-                    inputSplits[i].getLength());
+
+            assertThat(inputSplits[i].getLength())
+                    .as("%d. split should have block size length.", i)
+                    .isEqualTo(blockSize);
+
             if (inputSplits[i].getPath().toString().equals(pathFile1)) {
                 numSplitsFile1++;
             } else if (inputSplits[i].getPath().toString().equals(pathFile2)) {
                 numSplitsFile2++;
             } else {
-                Assert.fail("Split does not belong to any input file.");
+                fail("Split does not belong to any input file.");
             }
         }
-        Assert.assertEquals(numBlocks1, numSplitsFile1);
-        Assert.assertEquals(numBlocks2, numSplitsFile2);
+        assertThat(numSplitsFile1).isEqualTo(numBlocks1);
+        assertThat(numSplitsFile2).isEqualTo(numBlocks2);
     }
 
     @Test
@@ -134,7 +143,7 @@ public class BinaryInputFormatTest {
         format.configure(new Configuration());
 
         BaseStatistics stats = format.getStatistics(null);
-        Assert.assertNull("The file statistics should be null.", stats);
+        assertThat(stats).as("The file statistics should be null.").isNull();
     }
 
     @Test
@@ -154,10 +163,10 @@ public class BinaryInputFormatTest {
         inputFormat.setBlockSize(blockSize);
 
         BaseStatistics stats = inputFormat.getStatistics(null);
-        Assert.assertEquals(
-                "The file size statistics is wrong",
-                blockSize * (numBlocks1 + numBlocks2),
-                stats.getTotalInputSize());
+
+        assertThat(stats.getTotalInputSize())
+                .as("The file size statistics is wrong")
+                .isEqualTo(blockSize * (numBlocks1 + numBlocks2));
     }
 
     /** Creates a temp file with a certain number of blocks of a certain size. */