You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fa...@apache.org on 2023/06/28 02:34:21 UTC
[flink] branch release-1.16 updated (780f675430f -> 06d4c043249)
This is an automated email from the ASF dual-hosted git repository.
fanrui pushed a change to branch release-1.16
in repository https://gitbox.apache.org/repos/asf/flink.git
from 780f675430f [FLINK-32316] [connectors/common] Scheduling announceCombinedWatermark period task during SourceCoordinator::start not in construct function to avoid duplicated task be scheduled
new 4ab88c9226d [hotfix] Migrate SourceOperatorAlignmentTest to AssertJ
new 06d4c043249 [FLINK-32414][connectors/common] Don't emitLatestWatermark when lastEmittedWatermark is UNINITIALIZED
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../streaming/api/operators/SourceOperator.java | 3 +
.../api/operators/SourceOperatorAlignmentTest.java | 112 ++++++++++++++-------
2 files changed, 78 insertions(+), 37 deletions(-)
[flink] 01/02: [hotfix] Migrate SourceOperatorAlignmentTest to AssertJ
Posted by fa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fanrui pushed a commit to branch release-1.16
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4ab88c9226d41e0b1de67b626af3193b352c7d85
Author: 1996fanrui <19...@gmail.com>
AuthorDate: Thu Jun 22 23:12:19 2023 +0800
[hotfix] Migrate SourceOperatorAlignmentTest to AssertJ
---
.../api/operators/SourceOperatorAlignmentTest.java | 71 +++++++++++-----------
1 file changed, 34 insertions(+), 37 deletions(-)
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorAlignmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorAlignmentTest.java
index f8da5632859..3f8849c5011 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorAlignmentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorAlignmentTest.java
@@ -46,22 +46,17 @@ import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.contains;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.assertj.core.api.Assertions.assertThat;
/** Unit test for {@link SourceOperator} watermark alignment. */
@SuppressWarnings("serial")
-public class SourceOperatorAlignmentTest {
+class SourceOperatorAlignmentTest {
@Nullable private SourceOperatorTestContext context;
@Nullable private SourceOperator<Integer, MockSourceSplit> operator;
@BeforeEach
- public void setup() throws Exception {
+ void setup() throws Exception {
context =
new SourceOperatorTestContext(
false,
@@ -73,14 +68,14 @@ public class SourceOperatorAlignmentTest {
}
@AfterEach
- public void tearDown() throws Exception {
+ void tearDown() throws Exception {
context.close();
context = null;
operator = null;
}
@Test
- public void testWatermarkAlignment() throws Exception {
+ void testWatermarkAlignment() throws Exception {
operator.initializeState(context.createStateContext());
operator.open();
MockSourceSplit newSplit = new MockSourceSplit(2);
@@ -98,39 +93,39 @@ public class SourceOperatorAlignmentTest {
CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
List<Integer> expectedOutput = new ArrayList<>();
- assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
+ assertThat(operator.emitNext(actualOutput)).isEqualTo(DataInputStatus.MORE_AVAILABLE);
expectedOutput.add(record1);
context.getTimeService().advance(1);
assertLatestReportedWatermarkEvent(record1);
assertOutput(actualOutput, expectedOutput);
- assertTrue(operator.isAvailable());
+ assertThat(operator.isAvailable()).isTrue();
operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 - 1));
- assertFalse(operator.isAvailable());
- assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
+ assertThat(operator.isAvailable()).isFalse();
+ assertThat(operator.emitNext(actualOutput)).isEqualTo(DataInputStatus.NOTHING_AVAILABLE);
assertLatestReportedWatermarkEvent(record1);
assertOutput(actualOutput, expectedOutput);
- assertFalse(operator.isAvailable());
+ assertThat(operator.isAvailable()).isFalse();
operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 + 1));
- assertTrue(operator.isAvailable());
+ assertThat(operator.isAvailable()).isTrue();
operator.emitNext(actualOutput);
// Try to poll a record second time. Technically speaking previous emitNext call could have
// already switch the operator status to unavailable, but that's an implementation detail.
// However, this second call can not emit anything and should after that second call
// operator must be unavailable.
- assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
+ assertThat(operator.emitNext(actualOutput)).isEqualTo(DataInputStatus.NOTHING_AVAILABLE);
expectedOutput.add(record2);
context.getTimeService().advance(1);
assertLatestReportedWatermarkEvent(record2);
assertOutput(actualOutput, expectedOutput);
- assertFalse(operator.isAvailable());
+ assertThat(operator.isAvailable()).isFalse();
}
@Test
- public void testWatermarkAlignmentWithIdleness() throws Exception {
+ void testWatermarkAlignmentWithIdleness() throws Exception {
// we use a separate context, because we need to enable idleness
try (SourceOperatorTestContext context =
new SourceOperatorTestContext(
@@ -156,17 +151,18 @@ public class SourceOperatorAlignmentTest {
CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
List<Integer> expectedOutput = new ArrayList<>();
- assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
+ assertThat(operator.emitNext(actualOutput)).isEqualTo(DataInputStatus.MORE_AVAILABLE);
expectedOutput.add(record1);
context.getTimeService().advance(1);
assertLatestReportedWatermarkEvent(context, record1);
// mock WatermarkAlignmentEvent from SourceCoordinator
operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 + 100));
assertOutput(actualOutput, expectedOutput);
- assertTrue(operator.isAvailable());
+ assertThat(operator.isAvailable()).isTrue();
// source becomes idle, it should report Long.MAX_VALUE as the watermark
- assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
+ assertThat(operator.emitNext(actualOutput))
+ .isEqualTo(DataInputStatus.NOTHING_AVAILABLE);
context.getTimeService().advance(1);
assertLatestReportedWatermarkEvent(context, Long.MAX_VALUE);
// If all source subtasks of the watermark group are idle,
@@ -184,7 +180,7 @@ public class SourceOperatorAlignmentTest {
new AddSplitEvent<>(
Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
- assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
+ assertThat(operator.emitNext(actualOutput)).isEqualTo(DataInputStatus.MORE_AVAILABLE);
expectedOutput.add(record2);
context.getTimeService().advance(1);
// becomes active again, should go back to the previously emitted
@@ -192,23 +188,23 @@ public class SourceOperatorAlignmentTest {
assertLatestReportedWatermarkEvent(context, record1);
operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 + 100));
assertOutput(actualOutput, expectedOutput);
- assertTrue(operator.isAvailable());
+ assertThat(operator.isAvailable()).isTrue();
}
}
@Test
- public void testStopWhileWaitingForWatermarkAlignment() throws Exception {
+ void testStopWhileWaitingForWatermarkAlignment() throws Exception {
testWatermarkAlignment();
CompletableFuture<?> availableFuture = operator.getAvailableFuture();
- assertFalse(availableFuture.isDone());
+ assertThat(availableFuture).isNotDone();
operator.stop(StopMode.NO_DRAIN);
- assertTrue(availableFuture.isDone());
- assertTrue(operator.isAvailable());
+ assertThat(availableFuture).isDone();
+ assertThat(operator.isAvailable()).isTrue();
}
@Test
- public void testReportedWatermarkDoNotDecrease() throws Exception {
+ void testReportedWatermarkDoNotDecrease() throws Exception {
operator.initializeState(context.createStateContext());
operator.open();
MockSourceSplit split1 = new MockSourceSplit(2);
@@ -240,12 +236,12 @@ public class SourceOperatorAlignmentTest {
private void assertOutput(
CollectingDataOutput<Integer> actualOutput, List<Integer> expectedOutput) {
assertThat(
- actualOutput.getEvents().stream()
- .filter(o -> o instanceof StreamRecord)
- .mapToInt(object -> ((StreamRecord<Integer>) object).getValue())
- .boxed()
- .collect(Collectors.toList()),
- contains(expectedOutput.toArray()));
+ actualOutput.getEvents().stream()
+ .filter(o -> o instanceof StreamRecord)
+ .mapToInt(object -> ((StreamRecord<Integer>) object).getValue())
+ .boxed()
+ .collect(Collectors.toList()))
+ .containsExactly(expectedOutput.toArray(new Integer[0]));
}
private void assertLatestReportedWatermarkEvent(long expectedWatermark) {
@@ -259,8 +255,9 @@ public class SourceOperatorAlignmentTest {
.filter(event -> event instanceof ReportedWatermarkEvent)
.collect(Collectors.toList());
- assertFalse(events.isEmpty());
- assertEquals(new ReportedWatermarkEvent(expectedWatermark), events.get(events.size() - 1));
+ assertThat(events).isNotEmpty();
+ assertThat(events.get(events.size() - 1))
+ .isEqualTo(new ReportedWatermarkEvent(expectedWatermark));
}
private static class PunctuatedGenerator implements WatermarkGenerator<Integer> {
[flink] 02/02: [FLINK-32414][connectors/common] Don't emitLatestWatermark when lastEmittedWatermark is UNINITIALIZED
Posted by fa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fanrui pushed a commit to branch release-1.16
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 06d4c04324962826c47698d432fa5e839cb34889
Author: 1996fanrui <19...@gmail.com>
AuthorDate: Thu Jun 22 23:39:45 2023 +0800
[FLINK-32414][connectors/common] Don't emitLatestWatermark when lastEmittedWatermark is UNINITIALIZED
---
.../streaming/api/operators/SourceOperator.java | 3 ++
.../api/operators/SourceOperatorAlignmentTest.java | 41 ++++++++++++++++++++++
2 files changed, 44 insertions(+)
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
index 948d33b7081..e636b9a7bba 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
@@ -469,6 +469,9 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
private void emitLatestWatermark(long time) {
checkState(currentMainOutput != null);
+ if (lastEmittedWatermark == Watermark.UNINITIALIZED.getTimestamp()) {
+ return;
+ }
operatorEventGateway.sendEventToCoordinator(
new ReportedWatermarkEvent(lastEmittedWatermark));
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorAlignmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorAlignmentTest.java
index 3f8849c5011..ce7637ceb25 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorAlignmentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/SourceOperatorAlignmentTest.java
@@ -192,6 +192,39 @@ class SourceOperatorAlignmentTest {
}
}
+ @Test
+ void testWatermarkAlignmentWithoutSplit() throws Exception {
+ operator.initializeState(context.createStateContext());
+ operator.open();
+
+ CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
+ assertThat(operator.emitNext(actualOutput)).isEqualTo(DataInputStatus.NOTHING_AVAILABLE);
+
+ // Don't report any ReportedWatermarkEvent
+ context.getTimeService().advance(1);
+ assertNoReportedWatermarkEvent(context);
+
+ context.getTimeService().advance(1);
+ assertNoReportedWatermarkEvent(context);
+
+ assertThat(operator.emitNext(actualOutput)).isEqualTo(DataInputStatus.NOTHING_AVAILABLE);
+
+ MockSourceSplit newSplit = new MockSourceSplit(2);
+ int record = 10;
+ newSplit.addRecord(record);
+ operator.handleOperatorEvent(
+ new AddSplitEvent<>(
+ Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
+
+ List<Integer> expectedOutput = new ArrayList<>();
+ assertThat(operator.emitNext(actualOutput)).isEqualTo(DataInputStatus.MORE_AVAILABLE);
+ expectedOutput.add(record);
+
+ context.getTimeService().advance(1);
+ assertLatestReportedWatermarkEvent(record);
+ assertOutput(actualOutput, expectedOutput);
+ }
+
@Test
void testStopWhileWaitingForWatermarkAlignment() throws Exception {
testWatermarkAlignment();
@@ -260,6 +293,14 @@ class SourceOperatorAlignmentTest {
.isEqualTo(new ReportedWatermarkEvent(expectedWatermark));
}
+ private void assertNoReportedWatermarkEvent(SourceOperatorTestContext context) {
+ List<OperatorEvent> events =
+ context.getGateway().getEventsSent().stream()
+ .filter(event -> event instanceof ReportedWatermarkEvent)
+ .collect(Collectors.toList());
+ assertThat(events).isEmpty();
+ }
+
private static class PunctuatedGenerator implements WatermarkGenerator<Integer> {
private enum GenerationMode {