You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2024/08/17 08:55:50 UTC
(flink) 01/11: [hotfix] Refactor SourceOperatorSlitWatermarkAlignmentTest and support pausing splits in MockSourceReader
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 308f05a911d82971138f35f75ccb33ee1e73e891
Author: Piotr Nowojski <pi...@gmail.com>
AuthorDate: Mon Aug 5 15:24:46 2024 +0200
[hotfix] Refactor SourceOperatorSlitWatermarkAlignmentTest and support pausing splits in MockSourceReader
---
.../connector/source/mocks/MockSourceReader.java | 25 ++++++++-
.../SourceOperatorSplitWatermarkAlignmentTest.java | 65 +++++++++-------------
2 files changed, 48 insertions(+), 42 deletions(-)
diff --git a/flink-core/src/test/java/org/apache/flink/api/connector/source/mocks/MockSourceReader.java b/flink-core/src/test/java/org/apache/flink/api/connector/source/mocks/MockSourceReader.java
index 2c1d4b40867..facfcf346cd 100644
--- a/flink-core/src/test/java/org/apache/flink/api/connector/source/mocks/MockSourceReader.java
+++ b/flink-core/src/test/java/org/apache/flink/api/connector/source/mocks/MockSourceReader.java
@@ -26,11 +26,15 @@ import org.apache.flink.core.io.InputStatus;
import javax.annotation.concurrent.GuardedBy;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
/** A mock {@link SourceReader} for unit tests. */
public class MockSourceReader implements SourceReader<Integer, MockSourceSplit> {
+ private final Set<String> pausedSplits = new HashSet<>();
private final List<MockSourceSplit> assignedSplits = new ArrayList<>();
private final List<SourceEvent> receivedSourceEvents = new ArrayList<>();
private final List<Long> completedCheckpoints = new ArrayList<>();
@@ -100,11 +104,16 @@ public class MockSourceReader implements SourceReader<Integer, MockSourceSplit>
|| waitingForSplitsBehaviour == WaitingForSplits.DO_NOT_WAIT_FOR_SPLITS;
currentSplitIndex = 0;
// Find first splits with available records.
- while (currentSplitIndex < assignedSplits.size()
- && !assignedSplits.get(currentSplitIndex).isAvailable()) {
- finished &= assignedSplits.get(currentSplitIndex).isFinished();
+ for (MockSourceSplit assignedSplit : assignedSplits) {
+ finished &= assignedSplit.isFinished();
+ if (!pausedSplits.contains(assignedSplit.splitId())) {
+ if (assignedSplit.isAvailable()) {
+ break;
+ }
+ }
currentSplitIndex++;
}
+
// Read from the split with available record.
if (currentSplitIndex < assignedSplits.size()) {
if (idle) {
@@ -151,6 +160,12 @@ public class MockSourceReader implements SourceReader<Integer, MockSourceSplit>
markAvailable();
}
+ public void pauseOrResumeSplits(
+ Collection<String> splitsToPause, Collection<String> splitsToResume) {
+ pausedSplits.removeAll(splitsToResume);
+ pausedSplits.addAll(splitsToPause);
+ }
+
@Override
public void notifyNoMoreSplits() {
splitsAssignmentState = SplitsAssignmentState.NO_MORE_SPLITS;
@@ -224,4 +239,8 @@ public class MockSourceReader implements SourceReader<Integer, MockSourceSplit>
public List<Long> getAbortedCheckpoints() {
return abortedCheckpoints;
}
+
+ public Set<String> getPausedSplits() {
+ return pausedSplits;
+ }
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorSplitWatermarkAlignmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorSplitWatermarkAlignmentTest.java
index bef779e778a..eee59752e2d 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorSplitWatermarkAlignmentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorSplitWatermarkAlignmentTest.java
@@ -24,6 +24,7 @@ import org.apache.flink.api.common.eventtime.WatermarkGenerator;
import org.apache.flink.api.common.eventtime.WatermarkOutput;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.connector.source.mocks.MockSourceReader;
+import org.apache.flink.api.connector.source.mocks.MockSourceReader.WaitingForSplits;
import org.apache.flink.api.connector.source.mocks.MockSourceSplit;
import org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer;
import org.apache.flink.configuration.Configuration;
@@ -42,46 +43,28 @@ import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService;
import org.apache.flink.streaming.util.MockOutput;
import org.apache.flink.streaming.util.MockStreamConfig;
+import org.assertj.core.api.Condition;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.Optional;
+import java.util.function.Predicate;
import static org.assertj.core.api.Assertions.assertThat;
/** Unit test for split alignment in {@link SourceOperator}. */
-class SourceOperatorSplitWatermarkAlignmentTest {
- private static final WatermarkGenerator<Integer> WATERMARK_GENERATOR =
- new WatermarkGenerator<Integer>() {
-
- private long maxWatermark = Long.MIN_VALUE;
-
- @Override
- public void onEvent(Integer event, long eventTimestamp, WatermarkOutput output) {
- if (eventTimestamp > maxWatermark) {
- this.maxWatermark = eventTimestamp;
- output.emitWatermark(new Watermark(maxWatermark));
- }
- }
-
- @Override
- public void onPeriodicEmit(WatermarkOutput output) {
- output.emitWatermark(new Watermark(maxWatermark));
- }
- };
+class SourceOperatorSplitWatermarkAlignmentTest {
@Test
void testSplitWatermarkAlignment() throws Exception {
- final SplitAligningSourceReader sourceReader = new SplitAligningSourceReader();
+ MockSourceReader sourceReader = new MockSourceReader(WaitingForSplits.DO_NOT_WAIT_FOR_SPLITS, false, true);
SourceOperator<Integer, MockSourceSplit> operator =
new TestingSourceOperator<>(
sourceReader,
- WatermarkStrategy.forGenerator(ctx -> WATERMARK_GENERATOR)
+ WatermarkStrategy.forGenerator(ctx -> new TestWatermarkGenerator())
.withTimestampAssigner((r, l) -> r)
.withWatermarkAlignment("group-1", Duration.ofMillis(1)),
new TestProcessingTimeService(),
@@ -97,8 +80,8 @@ class SourceOperatorSplitWatermarkAlignmentTest {
operator.initializeState(new StreamTaskStateInitializerImpl(env, new MemoryStateBackend()));
operator.open();
- final MockSourceSplit split1 = new MockSourceSplit(0, 0, 10);
- final MockSourceSplit split2 = new MockSourceSplit(1, 10, 20);
+ MockSourceSplit split1 = new MockSourceSplit(0, 0, 10);
+ MockSourceSplit split2 = new MockSourceSplit(1, 10, 20);
split1.addRecord(5);
split1.addRecord(11);
split2.addRecord(3);
@@ -107,25 +90,25 @@ class SourceOperatorSplitWatermarkAlignmentTest {
operator.handleOperatorEvent(
new AddSplitEvent<>(
Arrays.asList(split1, split2), new MockSourceSplitSerializer()));
- final CollectingDataOutput<Integer> dataOutput = new CollectingDataOutput<>();
+ CollectingDataOutput<Integer> dataOutput = new CollectingDataOutput<>();
operator.emitNext(dataOutput); // split 1 emits 5
operator.handleOperatorEvent(
new WatermarkAlignmentEvent(4)); // pause by coordinator message
- assertThat(sourceReader.pausedSplits).containsExactly("0");
+ assertThat(sourceReader.getPausedSplits()).containsExactly("0");
operator.handleOperatorEvent(new WatermarkAlignmentEvent(5));
- assertThat(sourceReader.pausedSplits).isEmpty();
+ assertThat(sourceReader.getPausedSplits()).isEmpty();
operator.emitNext(dataOutput); // split 1 emits 11
operator.emitNext(dataOutput); // split 2 emits 3
- assertThat(sourceReader.pausedSplits).containsExactly("0");
+ assertThat(sourceReader.getPausedSplits()).containsExactly("0");
operator.emitNext(dataOutput); // split 2 emits 6
- assertThat(sourceReader.pausedSplits).containsExactly("0", "1");
+ assertThat(sourceReader.getPausedSplits()).containsExactly("0", "1");
}
private Environment getTestingEnvironment() {
@@ -139,17 +122,21 @@ class SourceOperatorSplitWatermarkAlignmentTest {
new TestTaskStateManager());
}
- private static class SplitAligningSourceReader extends MockSourceReader {
- Set<String> pausedSplits = new HashSet<>();
+ private static class TestWatermarkGenerator implements WatermarkGenerator<Integer> {
- public SplitAligningSourceReader() {
- super(WaitingForSplits.DO_NOT_WAIT_FOR_SPLITS, false, true);
+ private long maxWatermark = Long.MIN_VALUE;
+
+ @Override
+ public void onEvent(Integer event, long eventTimestamp, WatermarkOutput output) {
+ if (eventTimestamp > maxWatermark) {
+ this.maxWatermark = eventTimestamp;
+ output.emitWatermark(new Watermark(maxWatermark));
+ }
}
- public void pauseOrResumeSplits(
- Collection<String> splitsToPause, Collection<String> splitsToResume) {
- pausedSplits.removeAll(splitsToResume);
- pausedSplits.addAll(splitsToPause);
+ @Override
+ public void onPeriodicEmit(WatermarkOutput output) {
+ output.emitWatermark(new Watermark(maxWatermark));
}
}
}