You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2023/12/18 08:13:27 UTC

(iceberg) branch main updated: Flink: Fix TestIcebergSourceWithWatermarkExtractor flakiness (#9309)

This is an automated email from the ASF dual-hosted git repository.

pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 9342f64a71 Flink: Fix TestIcebergSourceWithWatermarkExtractor flakiness (#9309)
9342f64a71 is described below

commit 9342f64a713ed4a1c540046ae5db91b3ac86ac02
Author: pvary <pe...@gmail.com>
AuthorDate: Mon Dec 18 09:13:21 2023 +0100

    Flink: Fix TestIcebergSourceWithWatermarkExtractor flakiness (#9309)
---
 flink/v1.16/build.gradle                           |  3 +
 .../TestIcebergSourceWithWatermarkExtractor.java   | 72 ++++++++++++----------
 flink/v1.17/build.gradle                           |  3 +
 .../TestIcebergSourceWithWatermarkExtractor.java   | 71 ++++++++++++---------
 flink/v1.18/build.gradle                           |  3 +
 .../TestIcebergSourceWithWatermarkExtractor.java   | 71 +++++++++++----------
 6 files changed, 130 insertions(+), 93 deletions(-)

diff --git a/flink/v1.16/build.gradle b/flink/v1.16/build.gradle
index 2d5e6f326b..335a471e24 100644
--- a/flink/v1.16/build.gradle
+++ b/flink/v1.16/build.gradle
@@ -63,6 +63,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       // These artifacts are shaded and included in the orc-core fat jar
       exclude group: 'com.google.protobuf', module: 'protobuf-java'
       exclude group: 'org.apache.hive', module: 'hive-storage-api'
+      exclude group: 'org.slf4j'
     }
 
     testImplementation libs.flink116.connector.test.utils
@@ -111,6 +112,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.tdunning', module: 'json'
       exclude group: 'javax.transaction', module: 'transaction-api'
       exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'org.slf4j'
     }
 
     testImplementation libs.awaitility
@@ -202,6 +204,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'com.tdunning', module: 'json'
       exclude group: 'javax.transaction', module: 'transaction-api'
       exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'org.slf4j'
     }
 
     integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
diff --git a/flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java b/flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
index aa4b9cd79e..7428d30f42 100644
--- a/flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
+++ b/flink/v1.16/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
@@ -40,11 +40,11 @@ import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.core.execution.JobClient;
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.runtime.metrics.MetricNames;
 import org.apache.flink.runtime.minicluster.RpcServiceSharing;
+import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.runtime.testutils.InMemoryReporter;
 import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
 import org.apache.flink.streaming.api.datastream.DataStream;
@@ -78,7 +78,6 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
-  private static final InMemoryReporter reporter = InMemoryReporter.createWithRetainedMetrics();
   private static final int PARALLELISM = 4;
   private static final String SOURCE_NAME = "IcebergSource";
   private static final int RECORD_NUM_FOR_2_SPLITS = 200;
@@ -86,6 +85,8 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
   @ClassRule public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
 
+  private final InMemoryReporter reporter = InMemoryReporter.createWithRetainedMetrics();
+
   @Rule
   public final MiniClusterWithClientResource miniClusterResource =
       new MiniClusterWithClientResource(
@@ -93,10 +94,7 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
               .setNumberTaskManagers(1)
               .setNumberSlotsPerTaskManager(PARALLELISM)
               .setRpcServiceSharing(RpcServiceSharing.DEDICATED)
-              .setConfiguration(
-                  reporter.addToConfiguration(
-                      // disable classloader check as Avro may cache class in the serializers.
-                      new Configuration().set(CoreOptions.CHECK_LEAKED_CLASSLOADER, false)))
+              .setConfiguration(reporter.addToConfiguration(new Configuration()))
               .withHaLeadershipControl()
               .build());
 
@@ -273,23 +271,40 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
   public void testThrottling() throws Exception {
     GenericAppenderHelper dataAppender = appender();
 
-    // Generate records with the following pattern:
-    // - File 1 - Later records (Watermark 6000000)
-    //    - Split 1 - 2 records (100, "file_1-recordTs_100"), (103, "file_1-recordTs_103")
-    // - File 2 - First records (Watermark 0)
-    //    - Split 1 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
-    //    - Split 2 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
-    List<Record> batch =
+    // Generate records in advance
+
+    // File 1 - Later records (Watermark 6.000.000 - 100 min)
+    //  - Split 1 - 2 records (100, "file_1-recordTs_100"), (103, "file_1-recordTs_103")
+    List<Record> batch1 =
         ImmutableList.of(
             generateRecord(100, "file_1-recordTs_100"), generateRecord(103, "file_1-recordTs_103"));
-    dataAppender.appendToTable(batch);
 
-    batch = Lists.newArrayListWithCapacity(100);
+    // File 2 - First records (Watermark 0 - 0 min)
+    //  - Split 1 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
+    //  - Split 2 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
+    List<Record> batch2 = Lists.newArrayListWithCapacity(100);
     for (int i = 0; i < RECORD_NUM_FOR_2_SPLITS; ++i) {
-      batch.add(generateRecord(4 - i % 5, "file_2-recordTs_" + i));
+      batch2.add(generateRecord(4 - i % 5, "file_2-recordTs_" + i));
     }
 
-    dataAppender.appendToTable(batch);
+    // File 3 - Some records will be blocked (Watermark 900.000 - 15 min)
+    List<Record> batch3 =
+        ImmutableList.of(
+            generateRecord(15, "file_3-recordTs_15"),
+            generateRecord(16, "file_3-recordTs_16"),
+            generateRecord(17, "file_3-recordTs_17"));
+
+    // File 4 - Some records will be blocked (Watermark 900.000 - 15 min)
+    List<Record> batch4 =
+        ImmutableList.of(
+            generateRecord(15, "file_4-recordTs_15"),
+            generateRecord(16, "file_4-recordTs_16"),
+            generateRecord(17, "file_4-recordTs_17"));
+
+    // File 5 - Records which will remove the block (Watermark 5.400.000 - 90 min)
+    List<Record> batch5 =
+        ImmutableList.of(
+            generateRecord(90, "file_5-recordTs_90"), generateRecord(91, "file_5-recordTs_91"));
 
     StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
     env.setParallelism(2);
@@ -304,6 +319,11 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
     try (CloseableIterator<RowData> resultIterator = stream.collectAsync()) {
       JobClient jobClient = env.executeAsync("Iceberg Source Throttling Test");
+      CommonTestUtils.waitForAllTaskRunning(
+          miniClusterResource.getMiniCluster(), jobClient.getJobID(), false);
+
+      // Insert the first data into the table
+      dataAppender.appendToTable(dataAppender.writeFile(batch1), dataAppender.writeFile(batch2));
 
       // Check that the read the non-blocked data
       // The first RECORD_NUM_FOR_2_SPLITS should be read
@@ -325,18 +345,7 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
       // Add some old records with 2 splits, so even if the blocked gets one split, the other reader
       // one gets one as well
-      List<Record> newBatch1 =
-          ImmutableList.of(
-              generateRecord(15, "file_3-recordTs_15"),
-              generateRecord(16, "file_3-recordTs_16"),
-              generateRecord(17, "file_3-recordTs_17"));
-      List<Record> newBatch2 =
-          ImmutableList.of(
-              generateRecord(15, "file_4-recordTs_15"),
-              generateRecord(16, "file_4-recordTs_16"),
-              generateRecord(17, "file_4-recordTs_17"));
-      dataAppender.appendToTable(
-          dataAppender.writeFile(newBatch1), dataAppender.writeFile(newBatch2));
+      dataAppender.appendToTable(dataAppender.writeFile(batch3), dataAppender.writeFile(batch4));
       // The records received will highly depend on scheduling
       // We minimally get 3 records from the non-blocked reader
       // We might get 1 record from the blocked reader (as part of the previous batch -
@@ -352,10 +361,7 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
           .until(() -> drift.getValue() == TimeUnit.MINUTES.toMillis(65));
 
       // Add some new records which should unblock the throttled reader
-      batch =
-          ImmutableList.of(
-              generateRecord(90, "file_5-recordTs_90"), generateRecord(91, "file_5-recordTs_91"));
-      dataAppender.appendToTable(batch);
+      dataAppender.appendToTable(batch5);
       // We should get all the records at this point
       waitForRecords(resultIterator, 6);
 
diff --git a/flink/v1.17/build.gradle b/flink/v1.17/build.gradle
index 2e081b7608..f2bd37a6d1 100644
--- a/flink/v1.17/build.gradle
+++ b/flink/v1.17/build.gradle
@@ -63,6 +63,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       // These artifacts are shaded and included in the orc-core fat jar
       exclude group: 'com.google.protobuf', module: 'protobuf-java'
       exclude group: 'org.apache.hive', module: 'hive-storage-api'
+      exclude group: 'org.slf4j'
     }
 
     testImplementation libs.flink117.connector.test.utils
@@ -111,6 +112,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.tdunning', module: 'json'
       exclude group: 'javax.transaction', module: 'transaction-api'
       exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'org.slf4j'
     }
 
     testImplementation libs.awaitility
@@ -202,6 +204,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'com.tdunning', module: 'json'
       exclude group: 'javax.transaction', module: 'transaction-api'
       exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'org.slf4j'
     }
 
     integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
diff --git a/flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java b/flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
index 0bb2eb7766..c4070212df 100644
--- a/flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
+++ b/flink/v1.17/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
@@ -40,10 +40,12 @@ import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.PipelineOptions;
 import org.apache.flink.core.execution.JobClient;
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.runtime.metrics.MetricNames;
 import org.apache.flink.runtime.minicluster.RpcServiceSharing;
+import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.runtime.testutils.InMemoryReporter;
 import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
 import org.apache.flink.streaming.api.datastream.DataStream;
@@ -77,7 +79,6 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
-  private static final InMemoryReporter reporter = InMemoryReporter.createWithRetainedMetrics();
   private static final int PARALLELISM = 4;
   private static final String SOURCE_NAME = "IcebergSource";
   private static final int RECORD_NUM_FOR_2_SPLITS = 200;
@@ -85,6 +86,8 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
   @ClassRule public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
 
+  private final InMemoryReporter reporter = InMemoryReporter.createWithRetainedMetrics();
+
   @Rule
   public final MiniClusterWithClientResource miniClusterResource =
       new MiniClusterWithClientResource(
@@ -92,7 +95,9 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
               .setNumberTaskManagers(1)
               .setNumberSlotsPerTaskManager(PARALLELISM)
               .setRpcServiceSharing(RpcServiceSharing.DEDICATED)
-              .setConfiguration(reporter.addToConfiguration(new Configuration()))
+              .setConfiguration(
+                  reporter.addToConfiguration(
+                      new Configuration().set(PipelineOptions.ALLOW_UNALIGNED_SOURCE_SPLITS, true)))
               .withHaLeadershipControl()
               .build());
 
@@ -269,23 +274,40 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
   public void testThrottling() throws Exception {
     GenericAppenderHelper dataAppender = appender();
 
-    // Generate records with the following pattern:
-    // - File 1 - Later records (Watermark 6000000)
-    //    - Split 1 - 2 records (100, "file_1-recordTs_100"), (103, "file_1-recordTs_103")
-    // - File 2 - First records (Watermark 0)
-    //    - Split 1 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
-    //    - Split 2 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
-    List<Record> batch =
+    // Generate records in advance
+
+    // File 1 - Later records (Watermark 6.000.000 - 100 min)
+    //  - Split 1 - 2 records (100, "file_1-recordTs_100"), (103, "file_1-recordTs_103")
+    List<Record> batch1 =
         ImmutableList.of(
             generateRecord(100, "file_1-recordTs_100"), generateRecord(103, "file_1-recordTs_103"));
-    dataAppender.appendToTable(batch);
 
-    batch = Lists.newArrayListWithCapacity(100);
+    // File 2 - First records (Watermark 0 - 0 min)
+    //  - Split 1 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
+    //  - Split 2 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
+    List<Record> batch2 = Lists.newArrayListWithCapacity(100);
     for (int i = 0; i < RECORD_NUM_FOR_2_SPLITS; ++i) {
-      batch.add(generateRecord(4 - i % 5, "file_2-recordTs_" + i));
+      batch2.add(generateRecord(4 - i % 5, "file_2-recordTs_" + i));
     }
 
-    dataAppender.appendToTable(batch);
+    // File 3 - Some records will be blocked (Watermark 900.000 - 15 min)
+    List<Record> batch3 =
+        ImmutableList.of(
+            generateRecord(15, "file_3-recordTs_15"),
+            generateRecord(16, "file_3-recordTs_16"),
+            generateRecord(17, "file_3-recordTs_17"));
+
+    // File 4 - Some records will be blocked (Watermark 900.000 - 15 min)
+    List<Record> batch4 =
+        ImmutableList.of(
+            generateRecord(15, "file_4-recordTs_15"),
+            generateRecord(16, "file_4-recordTs_16"),
+            generateRecord(17, "file_4-recordTs_17"));
+
+    // File 5 - Records which will remove the block (Watermark 5.400.000 - 90 min)
+    List<Record> batch5 =
+        ImmutableList.of(
+            generateRecord(90, "file_5-recordTs_90"), generateRecord(91, "file_5-recordTs_91"));
 
     StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
     env.setParallelism(2);
@@ -300,6 +322,11 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
     try (CloseableIterator<RowData> resultIterator = stream.collectAsync()) {
       JobClient jobClient = env.executeAsync("Iceberg Source Throttling Test");
+      CommonTestUtils.waitForAllTaskRunning(
+          miniClusterResource.getMiniCluster(), jobClient.getJobID(), false);
+
+      // Insert the first data into the table
+      dataAppender.appendToTable(dataAppender.writeFile(batch1), dataAppender.writeFile(batch2));
 
       // Check that the read the non-blocked data
       // The first RECORD_NUM_FOR_2_SPLITS should be read
@@ -321,18 +348,7 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
       // Add some old records with 2 splits, so even if the blocked gets one split, the other reader
       // one gets one as well
-      List<Record> newBatch1 =
-          ImmutableList.of(
-              generateRecord(15, "file_3-recordTs_15"),
-              generateRecord(16, "file_3-recordTs_16"),
-              generateRecord(17, "file_3-recordTs_17"));
-      List<Record> newBatch2 =
-          ImmutableList.of(
-              generateRecord(15, "file_4-recordTs_15"),
-              generateRecord(16, "file_4-recordTs_16"),
-              generateRecord(17, "file_4-recordTs_17"));
-      dataAppender.appendToTable(
-          dataAppender.writeFile(newBatch1), dataAppender.writeFile(newBatch2));
+      dataAppender.appendToTable(dataAppender.writeFile(batch3), dataAppender.writeFile(batch4));
       // The records received will highly depend on scheduling
       // We minimally get 3 records from the non-blocked reader
       // We might get 1 record from the blocked reader (as part of the previous batch -
@@ -348,10 +364,7 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
           .until(() -> drift.getValue() == TimeUnit.MINUTES.toMillis(65));
 
       // Add some new records which should unblock the throttled reader
-      batch =
-          ImmutableList.of(
-              generateRecord(90, "file_5-recordTs_90"), generateRecord(91, "file_5-recordTs_91"));
-      dataAppender.appendToTable(batch);
+      dataAppender.appendToTable(batch5);
       // We should get all the records at this point
       waitForRecords(resultIterator, 6);
 
diff --git a/flink/v1.18/build.gradle b/flink/v1.18/build.gradle
index 55578d3b11..c08ae5d8cc 100644
--- a/flink/v1.18/build.gradle
+++ b/flink/v1.18/build.gradle
@@ -63,6 +63,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       // These artifacts are shaded and included in the orc-core fat jar
       exclude group: 'com.google.protobuf', module: 'protobuf-java'
       exclude group: 'org.apache.hive', module: 'hive-storage-api'
+      exclude group: 'org.slf4j'
     }
 
     testImplementation libs.flink118.connector.test.utils
@@ -111,6 +112,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
       exclude group: 'com.tdunning', module: 'json'
       exclude group: 'javax.transaction', module: 'transaction-api'
       exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'org.slf4j'
     }
 
     testImplementation libs.awaitility
@@ -202,6 +204,7 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") {
       exclude group: 'com.tdunning', module: 'json'
       exclude group: 'javax.transaction', module: 'transaction-api'
       exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'org.slf4j'
     }
 
     integrationImplementation("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
diff --git a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
index aa4b9cd79e..c4070212df 100644
--- a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
+++ b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceWithWatermarkExtractor.java
@@ -40,11 +40,12 @@ import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.PipelineOptions;
 import org.apache.flink.core.execution.JobClient;
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.runtime.metrics.MetricNames;
 import org.apache.flink.runtime.minicluster.RpcServiceSharing;
+import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.runtime.testutils.InMemoryReporter;
 import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
 import org.apache.flink.streaming.api.datastream.DataStream;
@@ -78,7 +79,6 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
-  private static final InMemoryReporter reporter = InMemoryReporter.createWithRetainedMetrics();
   private static final int PARALLELISM = 4;
   private static final String SOURCE_NAME = "IcebergSource";
   private static final int RECORD_NUM_FOR_2_SPLITS = 200;
@@ -86,6 +86,8 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
   @ClassRule public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
 
+  private final InMemoryReporter reporter = InMemoryReporter.createWithRetainedMetrics();
+
   @Rule
   public final MiniClusterWithClientResource miniClusterResource =
       new MiniClusterWithClientResource(
@@ -95,8 +97,7 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
               .setRpcServiceSharing(RpcServiceSharing.DEDICATED)
               .setConfiguration(
                   reporter.addToConfiguration(
-                      // disable classloader check as Avro may cache class in the serializers.
-                      new Configuration().set(CoreOptions.CHECK_LEAKED_CLASSLOADER, false)))
+                      new Configuration().set(PipelineOptions.ALLOW_UNALIGNED_SOURCE_SPLITS, true)))
               .withHaLeadershipControl()
               .build());
 
@@ -273,23 +274,40 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
   public void testThrottling() throws Exception {
     GenericAppenderHelper dataAppender = appender();
 
-    // Generate records with the following pattern:
-    // - File 1 - Later records (Watermark 6000000)
-    //    - Split 1 - 2 records (100, "file_1-recordTs_100"), (103, "file_1-recordTs_103")
-    // - File 2 - First records (Watermark 0)
-    //    - Split 1 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
-    //    - Split 2 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
-    List<Record> batch =
+    // Generate records in advance
+
+    // File 1 - Later records (Watermark 6.000.000 - 100 min)
+    //  - Split 1 - 2 records (100, "file_1-recordTs_100"), (103, "file_1-recordTs_103")
+    List<Record> batch1 =
         ImmutableList.of(
             generateRecord(100, "file_1-recordTs_100"), generateRecord(103, "file_1-recordTs_103"));
-    dataAppender.appendToTable(batch);
 
-    batch = Lists.newArrayListWithCapacity(100);
+    // File 2 - First records (Watermark 0 - 0 min)
+    //  - Split 1 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
+    //  - Split 2 - 100 records (0, "file_2-recordTs_0"), (1, "file_2-recordTs_1"),...
+    List<Record> batch2 = Lists.newArrayListWithCapacity(100);
     for (int i = 0; i < RECORD_NUM_FOR_2_SPLITS; ++i) {
-      batch.add(generateRecord(4 - i % 5, "file_2-recordTs_" + i));
+      batch2.add(generateRecord(4 - i % 5, "file_2-recordTs_" + i));
     }
 
-    dataAppender.appendToTable(batch);
+    // File 3 - Some records will be blocked (Watermark 900.000 - 15 min)
+    List<Record> batch3 =
+        ImmutableList.of(
+            generateRecord(15, "file_3-recordTs_15"),
+            generateRecord(16, "file_3-recordTs_16"),
+            generateRecord(17, "file_3-recordTs_17"));
+
+    // File 4 - Some records will be blocked (Watermark 900.000 - 15 min)
+    List<Record> batch4 =
+        ImmutableList.of(
+            generateRecord(15, "file_4-recordTs_15"),
+            generateRecord(16, "file_4-recordTs_16"),
+            generateRecord(17, "file_4-recordTs_17"));
+
+    // File 5 - Records which will remove the block (Watermark 5.400.000 - 90 min)
+    List<Record> batch5 =
+        ImmutableList.of(
+            generateRecord(90, "file_5-recordTs_90"), generateRecord(91, "file_5-recordTs_91"));
 
     StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
     env.setParallelism(2);
@@ -304,6 +322,11 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
     try (CloseableIterator<RowData> resultIterator = stream.collectAsync()) {
       JobClient jobClient = env.executeAsync("Iceberg Source Throttling Test");
+      CommonTestUtils.waitForAllTaskRunning(
+          miniClusterResource.getMiniCluster(), jobClient.getJobID(), false);
+
+      // Insert the first data into the table
+      dataAppender.appendToTable(dataAppender.writeFile(batch1), dataAppender.writeFile(batch2));
 
       // Check that the read the non-blocked data
       // The first RECORD_NUM_FOR_2_SPLITS should be read
@@ -325,18 +348,7 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
 
       // Add some old records with 2 splits, so even if the blocked gets one split, the other reader
       // one gets one as well
-      List<Record> newBatch1 =
-          ImmutableList.of(
-              generateRecord(15, "file_3-recordTs_15"),
-              generateRecord(16, "file_3-recordTs_16"),
-              generateRecord(17, "file_3-recordTs_17"));
-      List<Record> newBatch2 =
-          ImmutableList.of(
-              generateRecord(15, "file_4-recordTs_15"),
-              generateRecord(16, "file_4-recordTs_16"),
-              generateRecord(17, "file_4-recordTs_17"));
-      dataAppender.appendToTable(
-          dataAppender.writeFile(newBatch1), dataAppender.writeFile(newBatch2));
+      dataAppender.appendToTable(dataAppender.writeFile(batch3), dataAppender.writeFile(batch4));
       // The records received will highly depend on scheduling
       // We minimally get 3 records from the non-blocked reader
       // We might get 1 record from the blocked reader (as part of the previous batch -
@@ -352,10 +364,7 @@ public class TestIcebergSourceWithWatermarkExtractor implements Serializable {
           .until(() -> drift.getValue() == TimeUnit.MINUTES.toMillis(65));
 
       // Add some new records which should unblock the throttled reader
-      batch =
-          ImmutableList.of(
-              generateRecord(90, "file_5-recordTs_90"), generateRecord(91, "file_5-recordTs_91"));
-      dataAppender.appendToTable(batch);
+      dataAppender.appendToTable(batch5);
       // We should get all the records at this point
       waitForRecords(resultIterator, 6);