You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2019/08/08 08:31:33 UTC

[flink] 01/02: [FLINK-12479][operators] Integrate StreamInputProcessor with mailbox

This is an automated email from the ASF dual-hosted git repository.

pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 04b5cbf07775c086b1df33f94b77a99acc3f4615
Author: Piotr Nowojski <pi...@gmail.com>
AuthorDate: Fri Jun 21 11:57:24 2019 +0200

    [FLINK-12479][operators] Integrate StreamInputProcessor with mailbox
---
 .../streaming/runtime/io/StreamInputProcessor.java |   7 +-
 .../runtime/io/StreamOneInputProcessor.java        |  43 ++++---
 .../runtime/io/StreamTwoInputProcessor.java        |  21 +++-
 .../io/StreamTwoInputSelectableProcessor.java      | 125 ++++++++++-----------
 .../flink/streaming/runtime/tasks/StreamTask.java  |   9 +-
 .../tasks/StreamTaskSelectiveReadingTest.java      |  10 +-
 6 files changed, 122 insertions(+), 93 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
index 0b263d0..1de31bf 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
@@ -19,6 +19,7 @@
 package org.apache.flink.streaming.runtime.io;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.io.AvailabilityListener;
 
 import java.io.Closeable;
 
@@ -26,9 +27,11 @@ import java.io.Closeable;
  * Interface for processing records by {@link org.apache.flink.streaming.runtime.tasks.StreamTask}.
  */
 @Internal
-public interface StreamInputProcessor extends Closeable {
+public interface StreamInputProcessor extends AvailabilityListener, Closeable {
 	/**
-	 * @return true if {@link StreamTaskInput} is finished.
+	 * @return true if {@link StreamInputProcessor} estimates that more records can be processed
+	 * immediately. Otherwise false, which means that there are no more records available at the
+	 * moment and the caller should check {@link #isFinished()} and/or {@link #isAvailable()}.
 	 */
 	boolean processInput() throws Exception;
 }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamOneInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamOneInputProcessor.java
index 4be6d54..3be72ee 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamOneInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamOneInputProcessor.java
@@ -43,6 +43,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 import static org.apache.flink.util.Preconditions.checkState;
@@ -129,29 +130,29 @@ public final class StreamOneInputProcessor<IN> implements StreamInputProcessor {
 	}
 
 	@Override
+	public boolean isFinished() {
+		return input.isFinished();
+	}
+
+	@Override
+	public CompletableFuture<?> isAvailable() {
+		return input.isAvailable();
+	}
+
+	@Override
 	public boolean processInput() throws Exception {
 		initializeNumRecordsIn();
 
 		StreamElement recordOrMark = input.pollNextNullable();
-		if (recordOrMark == null) {
-			input.isAvailable().get();
-			return !checkFinished();
-		}
-		int channel = input.getLastChannel();
-		checkState(channel != StreamTaskInput.UNSPECIFIED);
+		if (recordOrMark != null) {
+			int channel = input.getLastChannel();
+			checkState(channel != StreamTaskInput.UNSPECIFIED);
 
-		processElement(recordOrMark, channel);
-		return true;
-	}
-
-	private boolean checkFinished() throws Exception {
-		boolean isFinished = input.isFinished();
-		if (isFinished) {
-			synchronized (lock) {
-				operatorChain.endInput(1);
-			}
+			processElement(recordOrMark, channel);
 		}
-		return isFinished;
+		checkFinished();
+
+		return recordOrMark != null;
 	}
 
 	private void processElement(StreamElement recordOrMark, int channel) throws Exception {
@@ -180,6 +181,14 @@ public final class StreamOneInputProcessor<IN> implements StreamInputProcessor {
 		}
 	}
 
+	private void checkFinished() throws Exception {
+		if (input.isFinished()) {
+			synchronized (lock) {
+				operatorChain.endInput(1);
+			}
+		}
+	}
+
 	private void initializeNumRecordsIn() {
 		if (numRecordsIn == null) {
 			try {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
index 00bdab9..c888477 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
@@ -56,6 +56,7 @@ import java.io.IOException;
 import java.util.BitSet;
 import java.util.Collection;
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 
@@ -207,6 +208,19 @@ public final class StreamTwoInputProcessor<IN1, IN2> implements StreamInputProce
 	}
 
 	@Override
+	public boolean isFinished() {
+		return isFinished;
+	}
+
+	@Override
+	public CompletableFuture<?> isAvailable() {
+		if (currentRecordDeserializer != null) {
+			return AVAILABLE;
+		}
+		return barrierHandler.isAvailable();
+	}
+
+	@Override
 	public boolean processInput() throws Exception {
 		if (isFinished) {
 			return false;
@@ -259,7 +273,6 @@ public final class StreamTwoInputProcessor<IN1, IN2> implements StreamInputProce
 								streamOperator.processElement1(record);
 							}
 							return true;
-
 						}
 					}
 					else {
@@ -295,15 +308,13 @@ public final class StreamTwoInputProcessor<IN1, IN2> implements StreamInputProce
 			if (bufferOrEvent.isPresent()) {
 				processBufferOrEvent(bufferOrEvent.get());
 			} else {
-				if (!barrierHandler.isFinished()) {
-					barrierHandler.isAvailable().get();
-				} else {
+				if (barrierHandler.isFinished()) {
 					isFinished = true;
 					if (!barrierHandler.isEmpty()) {
 						throw new IllegalStateException("Trailing data in checkpoint barrier handler.");
 					}
-					return false;
 				}
+				return false;
 			}
 		}
 	}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputSelectableProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputSelectableProcessor.java
index 65464db..4adc2dc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputSelectableProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputSelectableProcessor.java
@@ -47,7 +47,6 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
 import static org.apache.flink.util.Preconditions.checkState;
@@ -156,7 +155,21 @@ public final class StreamTwoInputSelectableProcessor<IN1, IN2> implements Stream
 		this.lastReadInputIndex = 1; // always try to read from the first input
 
 		this.isPrepared = false;
+	}
 
+	@Override
+	public boolean isFinished() {
+		return input1.isFinished() && input2.isFinished();
+	}
+
+	@Override
+	public CompletableFuture<?> isAvailable() {
+		if (inputSelection.isALLMaskOf2()) {
+			return isAnyInputAvailable();
+		} else {
+			StreamTaskInput input = (inputSelection.getInputMask() == InputSelection.FIRST.getInputMask()) ? input1 : input2;
+			return input.isAvailable();
+		}
 	}
 
 	@Override
@@ -179,18 +192,29 @@ public final class StreamTwoInputSelectableProcessor<IN1, IN2> implements Stream
 			if (recordOrMark != null) {
 				processElement1(recordOrMark, input1.getLastChannel());
 			}
+			checkFinished(input1, lastReadInputIndex);
 		} else {
 			recordOrMark = input2.pollNextNullable();
 			if (recordOrMark != null) {
 				processElement2(recordOrMark, input2.getLastChannel());
 			}
+			checkFinished(input2, lastReadInputIndex);
 		}
 
 		if (recordOrMark == null) {
 			setUnavailableInput(readingInputIndex);
 		}
 
-		return !checkFinished();
+		return recordOrMark != null;
+	}
+
+	private void checkFinished(StreamTaskInput input, int inputIndex) throws Exception {
+		if (input.isFinished()) {
+			synchronized (lock) {
+				operatorChain.endInput(getInputId(inputIndex));
+				inputSelection = inputSelector.nextSelection();
+			}
+		}
 	}
 
 	@Override
@@ -213,14 +237,12 @@ public final class StreamTwoInputSelectableProcessor<IN1, IN2> implements Stream
 		}
 	}
 
-	private int selectNextReadingInputIndex()
-		throws InterruptedException, ExecutionException, IOException {
-
-		int readingInputIndex;
-		while ((readingInputIndex = inputSelection.fairSelectNextIndexOutOf2(availableInputsMask, lastReadInputIndex)) == -1) {
-			if (!waitForAvailableInput(inputSelection)) {
-				return -1;
-			}
+	private int selectNextReadingInputIndex() throws IOException {
+		updateAvailability();
+		checkInputSelectionAgainstIsFinished();
+		int readingInputIndex = inputSelection.fairSelectNextIndexOutOf2(availableInputsMask, lastReadInputIndex);
+		if (readingInputIndex == -1) {
+			return -1;
 		}
 
 		// to avoid starvation, if the input selection is ALL and availableInputsMask is not ALL,
@@ -234,6 +256,27 @@ public final class StreamTwoInputSelectableProcessor<IN1, IN2> implements Stream
 		return readingInputIndex;
 	}
 
+	private void checkInputSelectionAgainstIsFinished() throws IOException {
+		if (inputSelection.isALLMaskOf2()) {
+			return;
+		}
+		if (inputSelection.isInputSelected(1) && input1.isFinished()) {
+			throw new IOException("Can not make a progress: only first input is selected but it is already finished");
+		}
+		if (inputSelection.isInputSelected(2) && input2.isFinished()) {
+			throw new IOException("Can not make a progress: only second input is selected but it is already finished");
+		}
+	}
+
+	private void updateAvailability() {
+		if (!input1.isFinished() && input1.isAvailable() == AVAILABLE) {
+			setAvailableInput(input1.getInputIndex());
+		}
+		if (!input2.isFinished() && input2.isAvailable() == AVAILABLE) {
+			setAvailableInput(input2.getInputIndex());
+		}
+	}
+
 	private void processElement1(StreamElement recordOrMark, int channel) throws Exception {
 		if (recordOrMark.isRecord()) {
 			StreamRecord<IN1> record = recordOrMark.asRecord();
@@ -304,64 +347,20 @@ public final class StreamTwoInputSelectableProcessor<IN1, IN2> implements Stream
 		}
 	}
 
-	/**
-	 * @return false if both of the inputs are finished, true otherwise.
-	 */
-	private boolean waitForAvailableInput(InputSelection inputSelection)
-		throws ExecutionException, InterruptedException, IOException {
-
-		if (inputSelection.isALLMaskOf2()) {
-			return waitForAvailableEitherInput();
-		} else {
-			waitForOneInput(
-				(inputSelection.getInputMask() == InputSelection.FIRST.getInputMask()) ? input1 : input2);
-			return true;
+	private CompletableFuture<?> isAnyInputAvailable() {
+		if (input1.isFinished()) {
+			return input2.isFinished() ? AVAILABLE : input2.isAvailable();
 		}
-	}
-
-	private boolean waitForAvailableEitherInput()
-		throws ExecutionException, InterruptedException {
 
-		CompletableFuture<?> future1 = input1.isFinished() ? UNAVAILABLE : input1.isAvailable();
-		CompletableFuture<?> future2 = input2.isFinished() ? UNAVAILABLE : input2.isAvailable();
-
-		if (future1 == UNAVAILABLE && future2 == UNAVAILABLE) {
-			return false;
+		if (input2.isFinished()) {
+			return input1.isAvailable();
 		}
 
-		// block to wait for a available input
-		CompletableFuture.anyOf(future1, future2).get();
-
-		if (future1.isDone()) {
-			setAvailableInput(input1.getInputIndex());
-		}
-		if (future2.isDone()) {
-			setAvailableInput(input2.getInputIndex());
-		}
+		CompletableFuture<?> input1Available = input1.isAvailable();
+		CompletableFuture<?> input2Available = input2.isAvailable();
 
-		return true;
-	}
-
-	private void waitForOneInput(StreamTaskInput input)
-		throws IOException, ExecutionException, InterruptedException {
-
-		if (input.isFinished()) {
-			throw new IOException("Could not read the finished input: input" + (input.getInputIndex() + 1) +  ".");
-		}
-
-		input.isAvailable().get();
-		setAvailableInput(input.getInputIndex());
-	}
-
-	private boolean checkFinished() throws Exception {
-		if (getInput(lastReadInputIndex).isFinished()) {
-			synchronized (lock) {
-				operatorChain.endInput(getInputId(lastReadInputIndex));
-				inputSelection = inputSelector.nextSelection();
-			}
-		}
-
-		return input1.isFinished() && input2.isFinished();
+		return (input1Available == AVAILABLE || input2Available == AVAILABLE) ?
+			AVAILABLE : CompletableFuture.anyOf(input1Available, input2Available);
 	}
 
 	private void setAvailableInput(int inputIndex) {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 7960a2f..53142a6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -61,6 +61,7 @@ import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer;
 import org.apache.flink.streaming.runtime.tasks.mailbox.execution.DefaultActionContext;
 import org.apache.flink.streaming.runtime.tasks.mailbox.execution.MailboxExecutor;
 import org.apache.flink.streaming.runtime.tasks.mailbox.execution.MailboxProcessor;
+import org.apache.flink.streaming.runtime.tasks.mailbox.execution.SuspendedMailboxDefaultAction;
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.Preconditions;
 
@@ -267,7 +268,13 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>>
 	 */
 	protected void performDefaultAction(DefaultActionContext context) throws Exception {
 		if (!inputProcessor.processInput()) {
-			context.allActionsCompleted();
+			if (inputProcessor.isFinished()) {
+				context.allActionsCompleted();
+			}
+			else {
+				SuspendedMailboxDefaultAction suspendedDefaultAction = context.suspendDefaultAction();
+				inputProcessor.isAvailable().thenRun(suspendedDefaultAction::resume);
+			}
 		}
 	}
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskSelectiveReadingTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskSelectiveReadingTest.java
index c7c1440..16e5bd1 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskSelectiveReadingTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskSelectiveReadingTest.java
@@ -39,7 +39,6 @@ import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -104,13 +103,14 @@ public class StreamTaskSelectiveReadingTest {
 	}
 
 	@Test
-	public void testReadFinishedInput() {
+	public void testReadFinishedInput() throws Exception {
 		try {
 			testBase(new TestReadFinishedInputStreamOperator(), false, new ConcurrentLinkedQueue<>(), true);
 			fail("should throw an IOException");
-		} catch (Throwable t) {
-			assertTrue("wrong exception, should be IOException",
-				ExceptionUtils.findThrowableWithMessage(t, "Could not read the finished input: input1").isPresent());
+		} catch (Exception t) {
+			if (!ExceptionUtils.findThrowableWithMessage(t, "only first input is selected but it is already finished").isPresent()) {
+				throw t;
+			}
 		}
 	}