You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2018/03/09 18:58:15 UTC

[1/6] flink git commit: [hotfix] [network] Rename RecordWriter#closeBufferConsumer() to closeBufferBuilder()

Repository: flink
Updated Branches:
  refs/heads/release-1.5 a75815aae -> d1a969f7a


[hotfix] [network] Rename RecordWriter#closeBufferConsumer() to closeBufferBuilder()


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/adae7d96
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/adae7d96
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/adae7d96

Branch: refs/heads/release-1.5
Commit: adae7d9672bf42fc44a9ceb31b0802007870af81
Parents: a75815a
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Thu Feb 22 14:11:13 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 9 17:01:42 2018 +0100

----------------------------------------------------------------------
 .../apache/flink/runtime/io/network/api/writer/RecordWriter.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/adae7d96/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
index 4ec2863..c35c7f3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
@@ -175,7 +175,7 @@ public class RecordWriter<T extends IOReadableWritable> {
 	public void clearBuffers() {
 		for (int targetChannel = 0; targetChannel < numChannels; targetChannel++) {
 			RecordSerializer<?> serializer = serializers[targetChannel];
-			closeBufferConsumer(targetChannel);
+			closeBufferBuilder(targetChannel);
 			serializer.clear();
 		}
 	}
@@ -213,7 +213,7 @@ public class RecordWriter<T extends IOReadableWritable> {
 		return bufferBuilder;
 	}
 
-	private void closeBufferConsumer(int targetChannel) {
+	private void closeBufferBuilder(int targetChannel) {
 		if (bufferBuilders[targetChannel].isPresent()) {
 			bufferBuilders[targetChannel].get().finish();
 			bufferBuilders[targetChannel] = Optional.empty();


[3/6] flink git commit: [hotfix] [network] [tests] Make AwaitableBufferAvailablityListener thread-safe

Posted by se...@apache.org.
[hotfix] [network] [tests] Make AwaitableBufferAvailablityListener thread-safe

This is called asynchronously by the spill writer and thus may need
synchronization on incrementing the counter but definately had visibility
issues with the counter. Using an AtomicLong fixes that.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b3a93c56
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b3a93c56
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b3a93c56

Branch: refs/heads/release-1.5
Commit: b3a93c569a3eca42b99323491d40f5bf0ad40628
Parents: 7e2e25c
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Fri Feb 23 10:35:41 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 9 17:01:48 2018 +0100

----------------------------------------------------------------------
 .../partition/AwaitableBufferAvailablityListener.java   | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/b3a93c56/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java
index 2b6b834..6cf9d64 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java
@@ -18,29 +18,31 @@
 
 package org.apache.flink.runtime.io.network.partition;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 /**
  * Test implementation of {@link BufferAvailabilityListener}.
  */
 class AwaitableBufferAvailablityListener implements BufferAvailabilityListener {
 
-	private long numNotifications;
+	private final AtomicLong numNotifications = new AtomicLong();
 
 	@Override
 	public void notifyDataAvailable() {
-		++numNotifications;
+		numNotifications.getAndIncrement();
 	}
 
 	public long getNumNotifications() {
-		return numNotifications;
+		return numNotifications.get();
 	}
 
 	public void resetNotificationCounters() {
-		numNotifications = 0;
+		numNotifications.set(0L);
 	}
 
 	void awaitNotifications(long awaitedNumNotifications, long timeoutMillis) throws InterruptedException {
 		long deadline = System.currentTimeMillis() + timeoutMillis;
-		while (numNotifications < awaitedNumNotifications && System.currentTimeMillis() < deadline) {
+		while (numNotifications.get() < awaitedNumNotifications && System.currentTimeMillis() < deadline) {
 			Thread.sleep(1);
 		}
 	}


[6/6] flink git commit: [FLINK-8755] [FLINK-8786] [network] Add and improve subpartition tests

Posted by se...@apache.org.
[FLINK-8755] [FLINK-8786] [network] Add and improve subpartition tests

+ also improve the subpartition tests in general to reduce some duplication

This closes #5581


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d1a969f7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d1a969f7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d1a969f7

Branch: refs/heads/release-1.5
Commit: d1a969f7ad018ef44f40f974eb49ba004494fcdf
Parents: 835adcc
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Fri Feb 23 12:13:20 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 9 17:01:54 2018 +0100

----------------------------------------------------------------------
 .../partition/SpillableSubpartitionView.java    |   2 +-
 .../partition/PipelinedSubpartitionTest.java    |  11 +-
 .../partition/SpillableSubpartitionTest.java    | 130 ++++++-------------
 .../network/partition/SubpartitionTestBase.java |  78 ++++++++++-
 4 files changed, 121 insertions(+), 100 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/d1a969f7/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java
index 0f51bc8..65790d7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java
@@ -167,7 +167,7 @@ class SpillableSubpartitionView implements ResultSubpartitionView {
 
 				parent.updateStatistics(current);
 				// if we are spilled (but still process a non-spilled nextBuffer), we don't know the
-				// state of nextBufferIsEvent...
+				// state of nextBufferIsEvent or whether more buffers are available
 				if (spilledView == null) {
 					return new BufferAndBacklog(current, isMoreAvailable, newBacklog, nextBufferIsEvent);
 				}

http://git-wip-us.apache.org/repos/asf/flink/blob/d1a969f7/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java
index ee678ab..bc66c9d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java
@@ -135,7 +135,8 @@ public class PipelinedSubpartitionTest extends SubpartitionTestBase {
 			bufferBuilder.appendAndCommit(ByteBuffer.allocate(1024));
 			subpartition.add(bufferBuilder.createBufferConsumer());
 
-			assertNextBuffer(readView, 1024, false, 1);
+			// note that since the buffer builder is not finished, there is still a retained instance!
+			assertNextBuffer(readView, 1024, false, 1, false, false);
 			assertEquals(1, subpartition.getBuffersInBacklog());
 		} finally {
 			readView.releaseAllResources();
@@ -157,7 +158,7 @@ public class PipelinedSubpartitionTest extends SubpartitionTestBase {
 			subpartition.add(createFilledBufferConsumer(1025)); // finished
 			subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished
 
-			assertNextBuffer(readView, 1025, false, 1);
+			assertNextBuffer(readView, 1025, false, 1, false, true);
 		} finally {
 			subpartition.release();
 		}
@@ -178,8 +179,8 @@ public class PipelinedSubpartitionTest extends SubpartitionTestBase {
 			subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished
 			subpartition.flush();
 
-			assertNextBuffer(readView, 1025, true, 1);
-			assertNextBuffer(readView, 1024, false, 1);
+			assertNextBuffer(readView, 1025, true, 1, false, true);
+			assertNextBuffer(readView, 1024, false, 1, false, false);
 		} finally {
 			subpartition.release();
 		}
@@ -208,7 +209,7 @@ public class PipelinedSubpartitionTest extends SubpartitionTestBase {
 			subpartition.add(createFilledBufferConsumer(1024));
 			assertEquals(2, availablityListener.getNumNotifications());
 
-			assertNextBuffer(readView, 1024, false, 0);
+			assertNextBuffer(readView, 1024, false, 0, false, true);
 		} finally {
 			readView.releaseAllResources();
 			subpartition.release();

http://git-wip-us.apache.org/repos/asf/flink/blob/d1a969f7/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java
index e41a85c..840669e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java
@@ -24,13 +24,13 @@ import org.apache.flink.runtime.io.disk.iomanager.FileIOChannel;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
 import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
 import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsyncWithNoOpBufferFileWriter;
+import org.apache.flink.runtime.io.network.api.CancelCheckpointMarker;
 import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
 import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
+import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.io.network.buffer.BufferBuilder;
-import org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils;
 import org.apache.flink.runtime.io.network.buffer.BufferConsumer;
 import org.apache.flink.runtime.io.network.buffer.BufferProvider;
-import org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog;
 
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -52,7 +52,6 @@ import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.
 import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
@@ -190,10 +189,13 @@ public class SpillableSubpartitionTest extends SubpartitionTestBase {
 		SpillableSubpartition partition = createSubpartition();
 
 		BufferConsumer bufferConsumer = createFilledBufferConsumer(BUFFER_DATA_SIZE, BUFFER_DATA_SIZE);
+		BufferConsumer eventBufferConsumer =
+			EventSerializer.toBufferConsumer(new CancelCheckpointMarker(1));
+		final int eventSize = eventBufferConsumer.getWrittenBytes();
 
 		partition.add(bufferConsumer.copy());
 		partition.add(bufferConsumer.copy());
-		partition.add(BufferBuilderTestUtils.createEventBufferConsumer(BUFFER_DATA_SIZE));
+		partition.add(eventBufferConsumer);
 		partition.add(bufferConsumer);
 
 		assertEquals(4, partition.getTotalNumberOfBuffers());
@@ -207,13 +209,13 @@ public class SpillableSubpartitionTest extends SubpartitionTestBase {
 		// still same statistics
 		assertEquals(4, partition.getTotalNumberOfBuffers());
 		assertEquals(3, partition.getBuffersInBacklog());
-		assertEquals(BUFFER_DATA_SIZE * 4, partition.getTotalNumberOfBytes());
+		assertEquals(BUFFER_DATA_SIZE * 3 + eventSize, partition.getTotalNumberOfBytes());
 
 		partition.finish();
 		// + one EndOfPartitionEvent
 		assertEquals(5, partition.getTotalNumberOfBuffers());
 		assertEquals(3, partition.getBuffersInBacklog());
-		assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes());
+		assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes());
 
 		AwaitableBufferAvailablityListener listener = new AwaitableBufferAvailablityListener();
 		SpilledSubpartitionView reader = (SpilledSubpartitionView) partition.createReadView(listener);
@@ -221,59 +223,24 @@ public class SpillableSubpartitionTest extends SubpartitionTestBase {
 		assertEquals(1, listener.getNumNotifications());
 
 		assertFalse(reader.nextBufferIsEvent()); // buffer
-		BufferAndBacklog read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertTrue(read.buffer().isBuffer());
+		assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 2, false, true);
 		assertEquals(2, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		assertFalse(read.buffer().isRecycled());
-		read.buffer().recycleBuffer();
-		assertTrue(read.buffer().isRecycled());
-		assertFalse(read.nextBufferIsEvent());
 
 		assertFalse(reader.nextBufferIsEvent()); // buffer
-		read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertTrue(read.buffer().isBuffer());
+		assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 1, true, true);
 		assertEquals(1, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		assertFalse(read.buffer().isRecycled());
-		read.buffer().recycleBuffer();
-		assertTrue(read.buffer().isRecycled());
-		assertTrue(read.nextBufferIsEvent());
 
 		assertTrue(reader.nextBufferIsEvent()); // event
-		read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertFalse(read.buffer().isBuffer());
+		assertNextEvent(reader, eventSize, CancelCheckpointMarker.class, true, 1, false, true);
 		assertEquals(1, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		read.buffer().recycleBuffer();
-		assertFalse(read.nextBufferIsEvent());
 
 		assertFalse(reader.nextBufferIsEvent()); // buffer
-		read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertTrue(read.buffer().isBuffer());
+		assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 0, true, true);
 		assertEquals(0, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		assertFalse(read.buffer().isRecycled());
-		read.buffer().recycleBuffer();
-		assertTrue(read.buffer().isRecycled());
-		assertTrue(read.nextBufferIsEvent());
 
 		assertTrue(reader.nextBufferIsEvent()); // end of partition event
-		read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertFalse(read.buffer().isBuffer());
+		assertNextEvent(reader, 4, EndOfPartitionEvent.class, false, 0, false, true);
 		assertEquals(0, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		assertEquals(EndOfPartitionEvent.class,
-			EventSerializer.fromBuffer(read.buffer(), ClassLoader.getSystemClassLoader()).getClass());
-		assertFalse(read.buffer().isRecycled());
-		read.buffer().recycleBuffer();
-		assertTrue(read.buffer().isRecycled());
-		assertFalse(read.nextBufferIsEvent());
 
 		// finally check that the bufferConsumer has been freed after a successful (or failed) write
 		final long deadline = System.currentTimeMillis() + 30_000L; // 30 secs
@@ -292,10 +259,13 @@ public class SpillableSubpartitionTest extends SubpartitionTestBase {
 		SpillableSubpartition partition = createSubpartition();
 
 		BufferConsumer bufferConsumer = createFilledBufferConsumer(BUFFER_DATA_SIZE, BUFFER_DATA_SIZE);
+		BufferConsumer eventBufferConsumer =
+			EventSerializer.toBufferConsumer(new CancelCheckpointMarker(1));
+		final int eventSize = eventBufferConsumer.getWrittenBytes();
 
 		partition.add(bufferConsumer.copy());
 		partition.add(bufferConsumer.copy());
-		partition.add(BufferBuilderTestUtils.createEventBufferConsumer(BUFFER_DATA_SIZE));
+		partition.add(eventBufferConsumer);
 		partition.add(bufferConsumer);
 		partition.finish();
 
@@ -311,17 +281,12 @@ public class SpillableSubpartitionTest extends SubpartitionTestBase {
 		assertFalse(bufferConsumer.isRecycled());
 
 		assertFalse(reader.nextBufferIsEvent());
-		BufferAndBacklog read = reader.getNextBuffer(); // first buffer (non-spilled)
-		assertNotNull(read);
-		assertTrue(read.buffer().isBuffer());
+		// first buffer (non-spilled)
+		assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 2, false, false);
 		assertEquals(BUFFER_DATA_SIZE, partition.getTotalNumberOfBytes()); // only updated when getting/spilling the buffers
 		assertEquals(2, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		read.buffer().recycleBuffer();
-		assertTrue(read.isMoreAvailable());
 		assertEquals(1, listener.getNumNotifications()); // since isMoreAvailable is set to true, no need for notification
 		assertFalse(bufferConsumer.isRecycled());
-		assertFalse(read.nextBufferIsEvent());
 
 		// Spill now
 		assertEquals(3, partition.releaseMemory());
@@ -330,59 +295,44 @@ public class SpillableSubpartitionTest extends SubpartitionTestBase {
 		assertEquals(5, partition.getTotalNumberOfBuffers());
 		assertEquals(2, partition.getBuffersInBacklog());
 		// only updated when getting/spilling the buffers but without the nextBuffer (kept in memory)
-		assertEquals(BUFFER_DATA_SIZE * 3 + 4, partition.getTotalNumberOfBytes());
+		assertEquals(BUFFER_DATA_SIZE * 2 + eventSize + 4, partition.getTotalNumberOfBytes());
 
+		// wait for successfully spilling all buffers (before that we may not access any spilled buffer and cannot rely on isMoreAvailable!)
 		listener.awaitNotifications(2, 30_000);
 		// Spiller finished
 		assertEquals(2, listener.getNumNotifications());
 
+		// after consuming and releasing the next buffer, the bufferConsumer may be freed,
+		// depending on the timing of the last write operation
+		// -> retain once so that we can check below
+		Buffer buffer = bufferConsumer.build();
+		buffer.retainBuffer();
+
 		assertFalse(reader.nextBufferIsEvent()); // second buffer (retained in SpillableSubpartition#nextBuffer)
-		read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertTrue(read.buffer().isBuffer());
-		assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // finally integrates the nextBuffer statistics
+		assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 1, true, false);
+		assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); // finally integrates the nextBuffer statistics
 		assertEquals(1, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		read.buffer().recycleBuffer();
-		// now the bufferConsumer may be freed, depending on the timing of the write operation
-		// -> let's do this check at the end of the test (to save some time)
-		assertTrue(read.nextBufferIsEvent());
+
+		bufferConsumer.close(); // recycle the retained buffer from above (should be the last reference!)
 
 		assertTrue(reader.nextBufferIsEvent()); // the event (spilled)
-		read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertFalse(read.buffer().isBuffer());
-		assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling
+		assertNextEvent(reader, eventSize, CancelCheckpointMarker.class, true, 1, false, true);
+		assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); // already updated during spilling
 		assertEquals(1, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		read.buffer().recycleBuffer();
-		assertFalse(read.nextBufferIsEvent());
 
 		assertFalse(reader.nextBufferIsEvent()); // last buffer (spilled)
-		read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertTrue(read.buffer().isBuffer());
-		assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling
+		assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 0, true, true);
+		assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); // already updated during spilling
 		assertEquals(0, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		assertFalse(read.buffer().isRecycled());
-		read.buffer().recycleBuffer();
-		assertTrue(read.buffer().isRecycled());
-		assertTrue(read.nextBufferIsEvent());
+
+		buffer.recycleBuffer();
+		assertTrue(buffer.isRecycled());
 
 		// End of partition
 		assertTrue(reader.nextBufferIsEvent());
-		read = reader.getNextBuffer();
-		assertNotNull(read);
-		assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling
+		assertNextEvent(reader, 4, EndOfPartitionEvent.class, false, 0, false, true);
+		assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); // already updated during spilling
 		assertEquals(0, partition.getBuffersInBacklog());
-		assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog());
-		assertEquals(EndOfPartitionEvent.class,
-			EventSerializer.fromBuffer(read.buffer(), ClassLoader.getSystemClassLoader()).getClass());
-		assertFalse(read.buffer().isRecycled());
-		read.buffer().recycleBuffer();
-		assertTrue(read.buffer().isRecycled());
-		assertFalse(read.nextBufferIsEvent());
 
 		// finally check that the bufferConsumer has been freed after a successful (or failed) write
 		final long deadline = System.currentTimeMillis() + 30_000L; // 30 secs

http://git-wip-us.apache.org/repos/asf/flink/blob/d1a969f7/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
index a3f18f6..8c90215 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
@@ -18,19 +18,26 @@
 
 package org.apache.flink.runtime.io.network.partition;
 
+import org.apache.flink.runtime.event.AbstractEvent;
+import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
 import org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils;
 import org.apache.flink.runtime.io.network.buffer.BufferConsumer;
 import org.apache.flink.util.TestLogger;
 
 import org.junit.Test;
 
+import javax.annotation.Nullable;
+
 import java.io.IOException;
 
 import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
 
@@ -138,11 +145,74 @@ public abstract class SubpartitionTestBase extends TestLogger {
 			ResultSubpartitionView readView,
 			int expectedReadableBufferSize,
 			boolean expectedIsMoreAvailable,
-			int expectedBuffersInBacklog) throws IOException, InterruptedException {
+			int expectedBuffersInBacklog,
+			boolean expectedNextBufferIsEvent,
+			boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException {
+		assertNextBufferOrEvent(
+			readView,
+			expectedReadableBufferSize,
+			true,
+			null,
+			expectedIsMoreAvailable,
+			expectedBuffersInBacklog,
+			expectedNextBufferIsEvent,
+			expectedRecycledAfterRecycle);
+	}
+
+	static void assertNextEvent(
+			ResultSubpartitionView readView,
+			int expectedReadableBufferSize,
+			Class<? extends AbstractEvent> expectedEventClass,
+			boolean expectedIsMoreAvailable,
+			int expectedBuffersInBacklog,
+			boolean expectedNextBufferIsEvent,
+			boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException {
+		assertNextBufferOrEvent(
+			readView,
+			expectedReadableBufferSize,
+			false,
+			expectedEventClass,
+			expectedIsMoreAvailable,
+			expectedBuffersInBacklog,
+			expectedNextBufferIsEvent,
+			expectedRecycledAfterRecycle);
+	}
+
+	private static void assertNextBufferOrEvent(
+			ResultSubpartitionView readView,
+			int expectedReadableBufferSize,
+			boolean expectedIsBuffer,
+			@Nullable Class<? extends AbstractEvent> expectedEventClass,
+			boolean expectedIsMoreAvailable,
+			int expectedBuffersInBacklog,
+			boolean expectedNextBufferIsEvent,
+			boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException {
+		checkArgument(expectedEventClass == null || !expectedIsBuffer);
+
 		ResultSubpartition.BufferAndBacklog bufferAndBacklog = readView.getNextBuffer();
-		assertEquals(expectedReadableBufferSize, bufferAndBacklog.buffer().readableBytes());
-		assertEquals(expectedIsMoreAvailable, bufferAndBacklog.isMoreAvailable());
-		assertEquals(expectedBuffersInBacklog, bufferAndBacklog.buffersInBacklog());
+		assertNotNull(bufferAndBacklog);
+		try {
+			assertEquals("buffer size", expectedReadableBufferSize,
+				bufferAndBacklog.buffer().readableBytes());
+			assertEquals("buffer or event", expectedIsBuffer,
+				bufferAndBacklog.buffer().isBuffer());
+			if (expectedEventClass != null) {
+				assertThat(EventSerializer
+						.fromBuffer(bufferAndBacklog.buffer(), ClassLoader.getSystemClassLoader()),
+					instanceOf(expectedEventClass));
+			}
+			assertEquals("more available", expectedIsMoreAvailable,
+				bufferAndBacklog.isMoreAvailable());
+			assertEquals("more available", expectedIsMoreAvailable, readView.isAvailable());
+			assertEquals("backlog", expectedBuffersInBacklog, bufferAndBacklog.buffersInBacklog());
+			assertEquals("next is event", expectedNextBufferIsEvent,
+				bufferAndBacklog.nextBufferIsEvent());
+
+			assertFalse("not recycled", bufferAndBacklog.buffer().isRecycled());
+		} finally {
+			bufferAndBacklog.buffer().recycleBuffer();
+		}
+		assertEquals("recycled", expectedRecycledAfterRecycle, bufferAndBacklog.buffer().isRecycled());
 	}
 
 	protected void assertNoNextBuffer(ResultSubpartitionView readView) throws IOException, InterruptedException {


[2/6] flink git commit: [hotfix] [network] Various minor improvements

Posted by se...@apache.org.
[hotfix] [network] Various minor improvements


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7e2e25cf
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7e2e25cf
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7e2e25cf

Branch: refs/heads/release-1.5
Commit: 7e2e25cfa6be737803a74b75ffb231568a51513e
Parents: adae7d9
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Thu Feb 22 14:17:06 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 9 17:01:45 2018 +0100

----------------------------------------------------------------------
 .../network/netty/CreditBasedSequenceNumberingViewReader.java   | 3 ++-
 .../flink/runtime/io/network/netty/PartitionRequestQueue.java   | 5 ++++-
 .../runtime/io/network/netty/SequenceNumberingViewReader.java   | 1 +
 .../runtime/io/network/partition/SubpartitionTestBase.java      | 2 +-
 4 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/7e2e25cf/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java
index 9acbbac..8fc7ef4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java
@@ -114,7 +114,7 @@ class CreditBasedSequenceNumberingViewReader implements BufferAvailabilityListen
 	 */
 	@Override
 	public boolean isAvailable() {
-		// BEWARE: this must be in sync with #isAvailable()!
+		// BEWARE: this must be in sync with #isAvailable(BufferAndBacklog)!
 		return hasBuffersAvailable() &&
 			(numCreditsAvailable > 0 || subpartitionView.nextBufferIsEvent());
 	}
@@ -130,6 +130,7 @@ class CreditBasedSequenceNumberingViewReader implements BufferAvailabilityListen
 	 * 		current buffer and backlog including information about the next buffer
 	 */
 	private boolean isAvailable(BufferAndBacklog bufferAndBacklog) {
+		// BEWARE: this must be in sync with #isAvailable()!
 		return bufferAndBacklog.isMoreAvailable() &&
 			(numCreditsAvailable > 0 || bufferAndBacklog.nextBufferIsEvent());
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/7e2e25cf/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java
index d63a88e..8c05b82 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java
@@ -38,6 +38,8 @@ import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdap
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 import java.io.IOException;
 import java.util.ArrayDeque;
 import java.util.Set;
@@ -52,7 +54,7 @@ import static org.apache.flink.runtime.io.network.netty.NettyMessage.BufferRespo
  */
 class PartitionRequestQueue extends ChannelInboundHandlerAdapter {
 
-	private final Logger LOG = LoggerFactory.getLogger(PartitionRequestQueue.class);
+	private static final Logger LOG = LoggerFactory.getLogger(PartitionRequestQueue.class);
 
 	private final ChannelFutureListener writeListener = new WriteAndFlushNextMessageIfPossibleListener();
 
@@ -278,6 +280,7 @@ class PartitionRequestQueue extends ChannelInboundHandlerAdapter {
 		reader.setRegisteredAsAvailable(true);
 	}
 
+	@Nullable
 	private NetworkSequenceViewReader pollAvailableReader() {
 		NetworkSequenceViewReader reader = availableReaders.poll();
 		if (reader != null) {

http://git-wip-us.apache.org/repos/asf/flink/blob/7e2e25cf/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java
index 6a83af1..054046f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java
@@ -148,6 +148,7 @@ class SequenceNumberingViewReader implements BufferAvailabilityListener, Network
 			"requestLock=" + requestLock +
 			", receiverId=" + receiverId +
 			", sequenceNumber=" + sequenceNumber +
+			", isRegisteredAsAvailable=" + isRegisteredAvailable +
 			'}';
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/7e2e25cf/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
index 215726b..a3f18f6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java
@@ -134,7 +134,7 @@ public abstract class SubpartitionTestBase extends TestLogger {
 		assertTrue(view.isReleased());
 	}
 
-	protected void assertNextBuffer(
+	static void assertNextBuffer(
 			ResultSubpartitionView readView,
 			int expectedReadableBufferSize,
 			boolean expectedIsMoreAvailable,


[4/6] flink git commit: [FLINK-8755] [network] Fix SpilledSubpartitionView relying on the backlog for determining whether more data is available

Posted by se...@apache.org.
[FLINK-8755] [network] Fix SpilledSubpartitionView relying on the backlog for determining whether more data is available

Fix SpilledSubpartitionView#getNextBuffer() to not only rely on the backlog:
instead it is sufficient to also return true if the next buffer is an event
since either there is a real buffer enqueued (reflected by the backlog) or at
least one event.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5c7457aa
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5c7457aa
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5c7457aa

Branch: refs/heads/release-1.5
Commit: 5c7457aa2aece89b77e9a9402cc514f90e083a69
Parents: b3a93c5
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Fri Feb 23 10:19:58 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 9 17:01:50 2018 +0100

----------------------------------------------------------------------
 .../runtime/io/network/partition/SpilledSubpartitionView.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/5c7457aa/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java
index 378b086..2a6a71f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java
@@ -148,7 +148,7 @@ class SpilledSubpartitionView implements ResultSubpartitionView, NotificationLis
 		}
 
 		int newBacklog = parent.decreaseBuffersInBacklog(current);
-		return new BufferAndBacklog(current, newBacklog > 0, newBacklog, nextBufferIsEvent);
+		return new BufferAndBacklog(current, newBacklog > 0 || nextBufferIsEvent, newBacklog, nextBufferIsEvent);
 	}
 
 	@Nullable


[5/6] flink git commit: [FLINK-8786] [network] Fix SpillableSubpartitionView#getNextBuffer returning wrong isMoreAvailable when processing last in-memory buffer

Posted by se...@apache.org.
[FLINK-8786] [network] Fix SpillableSubpartitionView#getNextBuffer returning wrong isMoreAvailable when processing last in-memory buffer

When processing the last in-memory buffer in
SpillableSubpartitionView#getNextBuffer while the rest of the buffers are
spilled, need to rely on the spilled view's isAvailable instead of always
setting the isMoreAvailable flag of the returned BufferAndBacklog to false.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/835adcc3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/835adcc3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/835adcc3

Branch: refs/heads/release-1.5
Commit: 835adcc373ce169f202055e9b4f9dc3fb9123772
Parents: 5c7457a
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Mon Feb 26 16:27:44 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 9 17:01:52 2018 +0100

----------------------------------------------------------------------
 .../runtime/io/network/partition/SpillableSubpartitionView.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/835adcc3/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java
index 3c73e43..0f51bc8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java
@@ -177,7 +177,7 @@ class SpillableSubpartitionView implements ResultSubpartitionView {
 		SpilledSubpartitionView spilled = spilledView;
 		if (spilled != null) {
 			if (current != null) {
-				return new BufferAndBacklog(current, isMoreAvailable, newBacklog, spilled.nextBufferIsEvent());
+				return new BufferAndBacklog(current, spilled.isAvailable(), newBacklog, spilled.nextBufferIsEvent());
 			} else {
 				return spilled.getNextBuffer();
 			}