You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by jq...@apache.org on 2020/12/12 11:08:34 UTC
[flink] 31/32: [FLINK-20270][runtime] Add support for
ExternallyInducedSource based on FLIP-27 to SourceOperatorStreamTask.
This is an automated email from the ASF dual-hosted git repository.
jqin pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 54884a7ad4057863c2fc3fa956e2149edc904688
Author: Jiangjie (Becket) Qin <ji...@alibaba-inc.com>
AuthorDate: Mon Nov 23 23:52:22 2020 +0800
[FLINK-20270][runtime] Add support for ExternallyInducedSource based on FLIP-27 to SourceOperatorStreamTask.
---
.../source/ExternallyInducedSourceReader.java | 57 +++++++++
.../io/StreamTaskExternallyInducedSourceInput.java | 51 ++++++++
.../runtime/io/StreamTaskSourceInput.java | 2 +-
.../runtime/tasks/SourceOperatorStreamTask.java | 57 ++++++++-
.../tasks/SourceOperatorStreamTaskTest.java | 130 ++++++++++++++++++++-
5 files changed, 291 insertions(+), 6 deletions(-)
diff --git a/flink-core/src/main/java/org/apache/flink/api/connector/source/ExternallyInducedSourceReader.java b/flink-core/src/main/java/org/apache/flink/api/connector/source/ExternallyInducedSourceReader.java
new file mode 100644
index 0000000..96ffd92
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/connector/source/ExternallyInducedSourceReader.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.connector.source;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import java.util.Optional;
+
+/**
+ * Sources that implement this interface do not trigger checkpoints when receiving a
+ * trigger message from the checkpoint coordinator, but when their input data/events
+ * indicate that a checkpoint should be triggered.
+ *
+ * <p>The ExternallyInducedSourceReader tells the Flink runtime that a checkpoint needs to
+ * be made by returning a checkpointId when shouldTriggerCheckpoint() is invoked.
+ *
+ * <p>The implementations typically works together with the SplitEnumerator which informs
+ * the external system to trigger a checkpoint.
+ *
+ * @param <T> The type of records produced by the source.
+ * @param <SplitT> The type of splits handled by the source.
+ */
+@PublicEvolving
+public interface ExternallyInducedSourceReader<T, SplitT extends SourceSplit>
+ extends SourceReader<T, SplitT> {
+
+ /**
+ * A method that informs the Flink runtime whether a checkpoint should be triggered on
+ * this Source.
+ *
+ * <p>This method is invoked when the previous {@link #pollNext(ReaderOutput)}
+ * returns {@link org.apache.flink.core.io.InputStatus#NOTHING_AVAILABLE}, to check
+ * if the source needs to be checkpointed.
+ *
+ * <p>If a CheckpointId is returned, a checkpoint will be triggered on this source reader.
+ * Otherwise, Flink runtime will continue to process the records.
+ *
+ * @return An optional checkpoint ID that Flink runtime should take a checkpoint for.
+ */
+ Optional<Long> shouldTriggerCheckpoint();
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskExternallyInducedSourceInput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskExternallyInducedSourceInput.java
new file mode 100644
index 0000000..03581d5
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskExternallyInducedSourceInput.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.io;
+
+import org.apache.flink.api.connector.source.ExternallyInducedSourceReader;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.streaming.api.operators.SourceOperator;
+
+import java.util.function.Consumer;
+
+/**
+ * A subclass of {@link StreamTaskSourceInput} for {@link ExternallyInducedSourceReader}.
+ */
+public class StreamTaskExternallyInducedSourceInput<T> extends StreamTaskSourceInput<T> {
+ private final Consumer<Long> checkpointTriggeringHook;
+ private final ExternallyInducedSourceReader<T, ?> sourceReader;
+
+ @SuppressWarnings("unchecked")
+ public StreamTaskExternallyInducedSourceInput(
+ SourceOperator<T, ?> operator,
+ Consumer<Long> checkpointTriggeringHook) {
+ super(operator);
+ this.checkpointTriggeringHook = checkpointTriggeringHook;
+ this.sourceReader = (ExternallyInducedSourceReader<T, ?>) operator.getSourceReader();
+ }
+
+ @Override
+ public InputStatus emitNext(DataOutput<T> output) throws Exception {
+ InputStatus status = super.emitNext(output);
+ if (status == InputStatus.NOTHING_AVAILABLE) {
+ sourceReader.shouldTriggerCheckpoint().ifPresent(checkpointTriggeringHook);
+ }
+ return status;
+ }
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskSourceInput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskSourceInput.java
index 0b2d065..722afa9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskSourceInput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskSourceInput.java
@@ -34,7 +34,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* unavailable or finished.
*/
@Internal
-public final class StreamTaskSourceInput<T> implements StreamTaskInput<T> {
+public class StreamTaskSourceInput<T> implements StreamTaskInput<T> {
private final SourceOperator<T, ?> operator;
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTask.java
index bd4fe74..d55e542 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTask.java
@@ -19,6 +19,10 @@
package org.apache.flink.streaming.runtime.tasks;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.connector.source.ExternallyInducedSourceReader;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
+import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.streaming.api.operators.Output;
import org.apache.flink.streaming.api.operators.SourceOperator;
@@ -26,12 +30,16 @@ import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.io.AbstractDataOutput;
import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput;
import org.apache.flink.streaming.runtime.io.StreamOneInputProcessor;
+import org.apache.flink.streaming.runtime.io.StreamTaskExternallyInducedSourceInput;
import org.apache.flink.streaming.runtime.io.StreamTaskInput;
import org.apache.flink.streaming.runtime.io.StreamTaskSourceInput;
import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
@@ -39,7 +47,9 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
*/
@Internal
public class SourceOperatorStreamTask<T> extends StreamTask<T, SourceOperator<T, ?>> {
+
private AsyncDataOutputToOutput<T> output;
+ private boolean isExternallyInducedSource;
public SourceOperatorStreamTask(Environment env) throws Exception {
super(env);
@@ -54,7 +64,21 @@ public class SourceOperatorStreamTask<T> extends StreamTask<T, SourceOperator<T,
// input processors
sourceOperator.initReader();
- StreamTaskInput<T> input = new StreamTaskSourceInput<>(headOperator);
+ final SourceReader<T, ?> sourceReader = headOperator.getSourceReader();
+ final StreamTaskInput<T> input;
+
+ if (sourceReader instanceof ExternallyInducedSourceReader) {
+ isExternallyInducedSource = true;
+
+ input = new StreamTaskExternallyInducedSourceInput<>(
+ sourceOperator,
+ this::triggerCheckpointForExternallyInducedSource);
+ } else {
+ input = new StreamTaskSourceInput<>(sourceOperator);
+ }
+
+ // The SourceOperatorStreamTask doesn't have any inputs, so there is no need for
+ // a WatermarkGauge on the input.
output = new AsyncDataOutputToOutput<>(
operatorChain.getChainEntryPoint(),
getStreamStatusMaintainer());
@@ -66,6 +90,21 @@ public class SourceOperatorStreamTask<T> extends StreamTask<T, SourceOperator<T,
}
@Override
+ public Future<Boolean> triggerCheckpointAsync(
+ CheckpointMetaData checkpointMetaData,
+ CheckpointOptions checkpointOptions,
+ boolean advanceToEndOfEventTime) {
+ if (!isExternallyInducedSource) {
+ return super.triggerCheckpointAsync(
+ checkpointMetaData,
+ checkpointOptions,
+ advanceToEndOfEventTime);
+ } else {
+ return CompletableFuture.completedFuture(isRunning());
+ }
+ }
+
+ @Override
protected void advanceToEndOfEventTime() {
output.emitWatermark(Watermark.MAX_WATERMARK);
}
@@ -78,6 +117,22 @@ public class SourceOperatorStreamTask<T> extends StreamTask<T, SourceOperator<T,
super.afterInvoke();
}
+ // --------------------------
+
+ private void triggerCheckpointForExternallyInducedSource(long checkpointId) {
+ final CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation(
+ configuration.isExactlyOnceCheckpointMode(),
+ configuration.isUnalignedCheckpointsEnabled());
+ final long timestamp = System.currentTimeMillis();
+
+ final CheckpointMetaData checkpointMetaData =
+ new CheckpointMetaData(checkpointId, timestamp);
+
+ super.triggerCheckpointAsync(checkpointMetaData, checkpointOptions, false);
+ }
+
+ // ---------------------------
+
/**
* Implementation of {@link DataOutput} that wraps a specific {@link Output}.
*/
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTaskTest.java
index 8c4dc0a..ce60719 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTaskTest.java
@@ -22,10 +22,15 @@ import org.apache.flink.api.common.eventtime.TimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.ExternallyInducedSourceReader;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
import org.apache.flink.api.connector.source.mocks.MockSource;
import org.apache.flink.api.connector.source.mocks.MockSourceReader;
import org.apache.flink.api.connector.source.mocks.MockSourceSplit;
import org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer;
+import org.apache.flink.core.io.InputStatus;
import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
@@ -43,10 +48,13 @@ import org.apache.flink.util.SerializedValue;
import org.junit.Test;
+import java.io.Serializable;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
+import java.util.Optional;
import java.util.Queue;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.function.Supplier;
import java.util.stream.IntStream;
@@ -56,6 +64,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
/**
* Tests for verifying that the {@link SourceOperator} as a task input can be integrated
@@ -125,6 +134,26 @@ public class SourceOperatorStreamTaskTest {
}
}
+ @Test
+ public void testExternallyInducedSource() throws Exception {
+ final int numEventsBeforeCheckpoint = 10;
+ final int totalNumEvents = 20;
+ TestingExternallyInducedSourceReader testingReader =
+ new TestingExternallyInducedSourceReader(numEventsBeforeCheckpoint, totalNumEvents);
+ try (StreamTaskMailboxTestHarness<Integer> testHarness =
+ createTestHarness(new TestingExternallyInducedSource(testingReader), 0, null)) {
+ TestingExternallyInducedSourceReader runtimeTestingReader =
+ (TestingExternallyInducedSourceReader) ((SourceOperator) testHarness.getStreamTask().headOperator).getSourceReader();
+
+ testHarness.processWhileAvailable();
+
+ assertEquals(totalNumEvents, runtimeTestingReader.numEmittedEvents);
+ assertTrue(runtimeTestingReader.checkpointed);
+ assertEquals(TestingExternallyInducedSourceReader.CHECKPOINT_ID, runtimeTestingReader.checkpointedId);
+ assertEquals(numEventsBeforeCheckpoint, runtimeTestingReader.checkpointedAt);
+ }
+ }
+
private TaskStateSnapshot executeAndWaitForCheckpoint(
long checkpointId,
TaskStateSnapshot initialSnapshot,
@@ -168,7 +197,7 @@ public class SourceOperatorStreamTaskTest {
// Wait until the checkpoint finishes.
// We have to mark the source reader as available here, otherwise the runMailboxStep() call after
- // checkpiont is completed will block.
+ // checkpoint is completed will block.
getSourceReaderFromTask(testHarness).markAvailable();
processUntil(testHarness, checkpointFuture::isDone);
waitForAcknowledgeLatch.await();
@@ -187,10 +216,20 @@ public class SourceOperatorStreamTaskTest {
private StreamTaskMailboxTestHarness<Integer> createTestHarness(
long checkpointId,
TaskStateSnapshot snapshot) throws Exception {
+ return createTestHarness(
+ new MockSource(Boundedness.BOUNDED, 1),
+ checkpointId,
+ snapshot
+ );
+ }
+
+ private StreamTaskMailboxTestHarness<Integer> createTestHarness(
+ MockSource source,
+ long checkpointId,
+ TaskStateSnapshot snapshot) throws Exception {
// get a source operator.
- SourceOperatorFactory<Integer> sourceOperatorFactory = new SourceOperatorFactory<>(
- new MockSource(Boundedness.BOUNDED, 1),
- WatermarkStrategy.noWatermarks());
+ SourceOperatorFactory<Integer> sourceOperatorFactory =
+ new SourceOperatorFactory<>(source, WatermarkStrategy.noWatermarks());
// build a test harness.
StreamTaskMailboxTestHarnessBuilder<Integer> builder =
@@ -236,4 +275,87 @@ public class SourceOperatorStreamTaskTest {
private MockSourceReader getSourceReaderFromTask(StreamTaskMailboxTestHarness<Integer> testHarness) {
return (MockSourceReader) ((SourceOperator) testHarness.getStreamTask().headOperator).getSourceReader();
}
+
+ // ------------- private testing classes ----------
+
+ private static class TestingExternallyInducedSource extends MockSource {
+ private static final long serialVersionUID = 3078454109555893721L;
+ private final TestingExternallyInducedSourceReader reader;
+
+ private TestingExternallyInducedSource(TestingExternallyInducedSourceReader reader) {
+ super(Boundedness.CONTINUOUS_UNBOUNDED, 1);
+ this.reader = reader;
+ }
+
+ @Override
+ public SourceReader<Integer, MockSourceSplit> createReader(SourceReaderContext readerContext) {
+ return reader;
+ }
+ }
+
+ private static class TestingExternallyInducedSourceReader
+ implements ExternallyInducedSourceReader<Integer, MockSourceSplit>, Serializable {
+ private static final long CHECKPOINT_ID = 1234L;
+ private final int numEventsBeforeCheckpoint;
+ private final int totalNumEvents;
+ private int numEmittedEvents;
+
+ private boolean checkpointed;
+ private int checkpointedAt;
+ private long checkpointedId;
+
+ TestingExternallyInducedSourceReader(int numEventsBeforeCheckpoint, int totalNumEvents) {
+ this.numEventsBeforeCheckpoint = numEventsBeforeCheckpoint;
+ this.totalNumEvents = totalNumEvents;
+ this.numEmittedEvents = 0;
+ this.checkpointed = false;
+ this.checkpointedAt = -1;
+ }
+
+ @Override
+ public Optional<Long> shouldTriggerCheckpoint() {
+ if (numEmittedEvents == numEventsBeforeCheckpoint && !checkpointed) {
+ return Optional.of(CHECKPOINT_ID);
+ } else {
+ return Optional.empty();
+ }
+ }
+
+ @Override
+ public void start() {}
+
+ @Override
+ public InputStatus pollNext(ReaderOutput<Integer> output) throws Exception {
+ numEmittedEvents++;
+ if (numEmittedEvents == numEventsBeforeCheckpoint) {
+ return InputStatus.NOTHING_AVAILABLE;
+ } else if (numEmittedEvents < totalNumEvents) {
+ return InputStatus.MORE_AVAILABLE;
+ } else {
+ return InputStatus.END_OF_INPUT;
+ }
+ }
+
+ @Override
+ public List<MockSourceSplit> snapshotState(long checkpointId) {
+ checkpointed = true;
+ checkpointedAt = numEmittedEvents;
+ checkpointedId = checkpointId;
+ return Collections.emptyList();
+ }
+
+ @Override
+ public CompletableFuture<Void> isAvailable() {
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ public void addSplits(List<MockSourceSplit> splits) {}
+
+ @Override
+ public void notifyNoMoreSplits() {}
+
+ @Override
+ public void close() throws Exception {}
+ }
}