You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2017/03/09 12:02:31 UTC
[6/9] flink git commit: [FLINK-4545] [network] remove fixed-size
BufferPool instances
[FLINK-4545] [network] remove fixed-size BufferPool instances
This closes #3467
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8b49ee5a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8b49ee5a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8b49ee5a
Branch: refs/heads/master
Commit: 8b49ee5aa2e17b1787764c3265e1ebda47d89840
Parents: 3ab91cc
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Fri Feb 10 16:11:08 2017 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Mar 9 13:00:56 2017 +0100
----------------------------------------------------------------------
.../plantranslate/JobGraphGenerator.java | 4 +-
.../BackPressureStatsTrackerITCase.java | 2 +-
.../runtime/io/network/NetworkEnvironment.java | 4 +-
.../serialization/SpanningRecordSerializer.java | 2 +-
.../io/network/buffer/BufferPoolFactory.java | 4 +-
.../io/network/buffer/LocalBufferPool.java | 36 +++++++++---
.../io/network/buffer/NetworkBufferPool.java | 24 +++-----
.../netty/PartitionRequestServerHandler.java | 2 +-
.../network/partition/ResultPartitionType.java | 16 +----
.../runtime/jobgraph/IntermediateDataSet.java | 8 ---
.../flink/runtime/jobgraph/JobVertex.java | 4 --
.../ExecutionGraphConstructionTest.java | 38 ++++++------
.../ExecutionGraphDeploymentTest.java | 6 +-
.../ExecutionGraphSignalsTest.java | 11 ++--
.../ExecutionVertexLocalityTest.java | 3 +-
.../executiongraph/PointwisePatternTest.java | 15 ++---
.../executiongraph/VertexSlotSharingTest.java | 5 +-
.../io/network/MockNetworkEnvironment.java | 2 +-
.../io/network/api/writer/RecordWriterTest.java | 2 +-
.../network/buffer/BufferPoolFactoryTest.java | 31 +++-------
.../network/buffer/NetworkBufferPoolTest.java | 55 +++++-------------
.../consumer/LocalInputChannelTest.java | 4 +-
.../flink/runtime/jobgraph/JobGraphTest.java | 61 ++++++++++----------
.../runtime/jobgraph/JobTaskVertexTest.java | 5 +-
.../jobgraph/jsonplan/JsonGeneratorTest.java | 4 +-
.../LeaderChangeJobRecoveryTest.java | 3 +-
.../LeaderChangeStateCleanupTest.java | 3 +-
.../TaskCancelAsyncProducerConsumerITCase.java | 3 +-
.../jobmanager/CoLocationConstraintITCase.scala | 11 ++--
.../runtime/jobmanager/JobManagerITCase.scala | 40 ++++++++-----
.../runtime/jobmanager/RecoveryITCase.scala | 18 +++---
.../runtime/jobmanager/SlotSharingITCase.scala | 19 +++---
.../TaskManagerFailsWithSlotSharingITCase.scala | 12 ++--
.../io/BarrierBufferMassiveRandomTest.java | 4 +-
.../runtime/NetworkStackThroughputITCase.java | 10 +++-
.../ZooKeeperLeaderElectionITCase.java | 4 +-
.../taskmanager/TaskManagerFailsITCase.scala | 7 ++-
37 files changed, 238 insertions(+), 244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
index caeb43f..61e5327 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
@@ -1289,7 +1289,7 @@ public class JobGraphGenerator implements Visitor<PlanNode> {
syncConfig.setNumberOfIterations(maxNumIterations);
// connect the sync task
- sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+ sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
// ----------------------------- create the iteration tail ------------------------------
@@ -1425,7 +1425,7 @@ public class JobGraphGenerator implements Visitor<PlanNode> {
syncConfig.setNumberOfIterations(maxNumIterations);
// connect the sync task
- sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+ sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
}
// ----------------------------- create the iteration tails -----------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java
index 30a86a2..1943129 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/BackPressureStatsTrackerITCase.java
@@ -108,7 +108,7 @@ public class BackPressureStatsTrackerITCase extends TestLogger {
//
// 1) Consume all buffers at first (no buffers for the test task)
//
- testBufferPool = networkBufferPool.createBufferPool(1, false);
+ testBufferPool = networkBufferPool.createBufferPool(1);
final List<Buffer> buffers = new ArrayList<>();
while (true) {
Buffer buffer = testBufferPool.requestBuffer();
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
index 5cf2c26..8e85ffe 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
@@ -171,7 +171,7 @@ public class NetworkEnvironment {
BufferPool bufferPool = null;
try {
- bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions(), false);
+ bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions());
partition.registerBufferPool(bufferPool);
resultPartitionManager.registerResultPartition(partition);
@@ -198,7 +198,7 @@ public class NetworkEnvironment {
BufferPool bufferPool = null;
try {
- bufferPool = networkBufferPool.createBufferPool(gate.getNumberOfInputChannels(), false);
+ bufferPool = networkBufferPool.createBufferPool(gate.getNumberOfInputChannels());
gate.setBufferPool(bufferPool);
} catch (Throwable t) {
if (bufferPool != null) {
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
index cb5665b..6c541a9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
@@ -43,7 +43,7 @@ public class SpanningRecordSerializer<T extends IOReadableWritable> implements R
/** Intermediate data serialization */
private final DataOutputSerializer serializationBuffer;
- /** Intermediate buffer for data serialization */
+ /** Intermediate buffer for data serialization (wrapped from {@link #serializationBuffer}) */
private ByteBuffer dataBuffer;
/** Intermediate buffer for length serialization */
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactory.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactory.java
index 23321f4..e953158 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactory.java
@@ -29,9 +29,9 @@ public interface BufferPoolFactory {
* Tries to create a buffer pool, which is guaranteed to provide at least the number of required
* buffers.
*
- * <p> The buffer pool is either of dynamic size or fixed.
+ * <p> The buffer pool is of dynamic size with at least <tt>numRequiredBuffers</tt> buffers.
*/
- BufferPool createBufferPool(int numRequiredBuffers, boolean isFixedSize) throws IOException;
+ BufferPool createBufferPool(int numRequiredBuffers) throws IOException;
/**
* Destroy callback for updating factory book keeping.
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java
index 86e6870..d6a4cf7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/LocalBufferPool.java
@@ -45,30 +45,46 @@ import static org.apache.flink.util.Preconditions.checkState;
*/
class LocalBufferPool implements BufferPool {
+ /** Global network buffer pool to get buffers from. */
private final NetworkBufferPool networkBufferPool;
- // The minimum number of required segments for this pool
+ /** The minimum number of required segments for this pool */
private final int numberOfRequiredMemorySegments;
- // The currently available memory segments. These are segments, which have been requested from
- // the network buffer pool and are currently not handed out as Buffer instances.
+ /**
+ * The currently available memory segments. These are segments, which have been requested from
+ * the network buffer pool and are currently not handed out as Buffer instances.
+ */
private final Queue<MemorySegment> availableMemorySegments = new ArrayDeque<MemorySegment>();
- // Buffer availability listeners, which need to be notified when a Buffer becomes available.
- // Listeners can only be registered at a time/state where no Buffer instance was available.
+ /**
+ * Buffer availability listeners, which need to be notified when a Buffer becomes available.
+ * Listeners can only be registered at a time/state where no Buffer instance was available.
+ */
private final Queue<EventListener<Buffer>> registeredListeners = new ArrayDeque<EventListener<Buffer>>();
- // The current size of this pool
+ /** The current size of this pool */
private int currentPoolSize;
- // Number of all memory segments, which have been requested from the network buffer pool and are
- // somehow referenced through this pool (e.g. wrapped in Buffer instances or as available segments).
+ /**
+ * Number of all memory segments, which have been requested from the network buffer pool and are
+ * somehow referenced through this pool (e.g. wrapped in Buffer instances or as available segments).
+ */
private int numberOfRequestedMemorySegments;
private boolean isDestroyed;
private BufferPoolOwner owner;
+ /**
+ * Local buffer pool based on the given <tt>networkBufferPool</tt> with a minimal number of
+ * network buffers being available.
+ *
+ * @param networkBufferPool
+ * global network buffer pool to get buffers from
+ * @param numberOfRequiredMemorySegments
+ * minimum number of network buffers
+ */
LocalBufferPool(NetworkBufferPool networkBufferPool, int numberOfRequiredMemorySegments) {
this.networkBufferPool = networkBufferPool;
this.numberOfRequiredMemorySegments = numberOfRequiredMemorySegments;
@@ -265,11 +281,15 @@ class LocalBufferPool implements BufferPool {
// ------------------------------------------------------------------------
private void returnMemorySegment(MemorySegment segment) {
+ assert Thread.holdsLock(availableMemorySegments);
+
numberOfRequestedMemorySegments--;
networkBufferPool.recycle(segment);
}
private void returnExcessMemorySegments() {
+ assert Thread.holdsLock(availableMemorySegments);
+
while (numberOfRequestedMemorySegments > currentPoolSize) {
MemorySegment segment = availableMemorySegments.poll();
if (segment == null) {
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java
index dc23341..5345fbb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPool.java
@@ -58,9 +58,7 @@ public class NetworkBufferPool implements BufferPoolFactory {
private final Object factoryLock = new Object();
- private final Set<LocalBufferPool> managedBufferPools = new HashSet<LocalBufferPool>();
-
- public final Set<LocalBufferPool> allBufferPools = new HashSet<LocalBufferPool>();
+ private final Set<LocalBufferPool> allBufferPools = new HashSet<LocalBufferPool>();
private int numTotalRequiredBuffers;
@@ -182,7 +180,7 @@ public class NetworkBufferPool implements BufferPoolFactory {
// ------------------------------------------------------------------------
@Override
- public BufferPool createBufferPool(int numRequiredBuffers, boolean isFixedSize) throws IOException {
+ public BufferPool createBufferPool(int numRequiredBuffers) throws IOException {
// It is necessary to use a separate lock from the one used for buffer
// requests to ensure deadlock freedom for failure cases.
synchronized (factoryLock) {
@@ -209,12 +207,6 @@ public class NetworkBufferPool implements BufferPoolFactory {
// non-fixed size buffers.
LocalBufferPool localBufferPool = new LocalBufferPool(this, numRequiredBuffers);
- // The fixed size pools get their share of buffers and don't change
- // it during their lifetime.
- if (!isFixedSize) {
- managedBufferPools.add(localBufferPool);
- }
-
allBufferPools.add(localBufferPool);
redistributeBuffers();
@@ -231,8 +223,6 @@ public class NetworkBufferPool implements BufferPoolFactory {
synchronized (factoryLock) {
if (allBufferPools.remove(bufferPool)) {
- managedBufferPools.remove(bufferPool);
-
numTotalRequiredBuffers -= bufferPool.getNumberOfRequiredMemorySegments();
try {
@@ -246,7 +236,7 @@ public class NetworkBufferPool implements BufferPoolFactory {
/**
* Destroys all buffer pools that allocate their buffers from this
- * buffer pool (created via {@link #createBufferPool(int, boolean)}).
+ * buffer pool (created via {@link #createBufferPool(int)}).
*/
public void destroyAllBufferPools() {
synchronized (factoryLock) {
@@ -258,7 +248,7 @@ public class NetworkBufferPool implements BufferPoolFactory {
}
// some sanity checks
- if (allBufferPools.size() > 0 || managedBufferPools.size() > 0 || numTotalRequiredBuffers > 0) {
+ if (allBufferPools.size() > 0 || numTotalRequiredBuffers > 0) {
throw new IllegalStateException("NetworkBufferPool is not empty after destroying all LocalBufferPools");
}
}
@@ -266,7 +256,9 @@ public class NetworkBufferPool implements BufferPoolFactory {
// Must be called from synchronized block
private void redistributeBuffers() throws IOException {
- int numManagedBufferPools = managedBufferPools.size();
+ assert Thread.holdsLock(factoryLock);
+
+ int numManagedBufferPools = allBufferPools.size();
if (numManagedBufferPools == 0) {
return; // necessary to avoid div by zero when no managed pools
@@ -283,7 +275,7 @@ public class NetworkBufferPool implements BufferPoolFactory {
int bufferPoolIndex = 0;
- for (LocalBufferPool bufferPool : managedBufferPools) {
+ for (LocalBufferPool bufferPool : allBufferPools) {
int leftoverBuffers = bufferPoolIndex++ < numLeftoverBuffers ? 1 : 0;
bufferPool.setNumBuffers(bufferPool.getNumberOfRequiredMemorySegments() + numExcessBuffersPerPool + leftoverBuffers);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
index 12b52ec..36c1234 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
@@ -67,7 +67,7 @@ class PartitionRequestServerHandler extends SimpleChannelInboundHandler<NettyMes
public void channelRegistered(ChannelHandlerContext ctx) throws Exception {
super.channelRegistered(ctx);
- bufferPool = networkBufferPool.createBufferPool(1, false);
+ bufferPool = networkBufferPool.createBufferPool(1);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionType.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionType.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionType.java
index 65d49ed..43d3a52 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionType.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionType.java
@@ -20,14 +20,9 @@ package org.apache.flink.runtime.io.network.partition;
public enum ResultPartitionType {
- BLOCKING(true, false, false),
+ BLOCKING(false, false),
- PIPELINED(false, true, true),
-
- PIPELINED_PERSISTENT(true, true, true);
-
- /** Does the partition live longer than the consuming task? */
- private final boolean isPersistent;
+ PIPELINED(true, true);
/** Can the partition be consumed while being produced? */
private final boolean isPipelined;
@@ -38,8 +33,7 @@ public enum ResultPartitionType {
/**
* Specifies the behaviour of an intermediate result partition at runtime.
*/
- ResultPartitionType(boolean isPersistent, boolean isPipelined, boolean hasBackPressure) {
- this.isPersistent = isPersistent;
+ ResultPartitionType(boolean isPipelined, boolean hasBackPressure) {
this.isPipelined = isPipelined;
this.hasBackPressure = hasBackPressure;
}
@@ -55,8 +49,4 @@ public enum ResultPartitionType {
public boolean isPipelined() {
return isPipelined;
}
-
- public boolean isPersistent() {
- return isPersistent;
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateDataSet.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateDataSet.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateDataSet.java
index 2d9faa8..f02aaa3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateDataSet.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateDataSet.java
@@ -47,14 +47,6 @@ public class IntermediateDataSet implements java.io.Serializable {
private final ResultPartitionType resultType;
// --------------------------------------------------------------------------------------------
-
- public IntermediateDataSet(JobVertex producer) {
- this(new IntermediateDataSetID(), producer);
- }
-
- public IntermediateDataSet(IntermediateDataSetID id, JobVertex producer) {
- this(id, ResultPartitionType.PIPELINED, producer);
- }
public IntermediateDataSet(IntermediateDataSetID id, ResultPartitionType resultType, JobVertex producer) {
this.id = checkNotNull(id);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java
index 9dcaeeb..260bd74 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobVertex.java
@@ -400,10 +400,6 @@ public class JobVertex implements java.io.Serializable {
return edge;
}
- public JobEdge connectNewDataSetAsInput(JobVertex input, DistributionPattern distPattern) {
- return connectNewDataSetAsInput(input, distPattern, ResultPartitionType.PIPELINED);
- }
-
public JobEdge connectNewDataSetAsInput(
JobVertex input,
DistributionPattern distPattern,
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java
index fa48384..e1bad56 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphConstructionTest.java
@@ -104,11 +104,11 @@ public class ExecutionGraphConstructionTest {
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL);
- v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL);
- v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL);
- v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL);
- v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
@@ -153,7 +153,7 @@ public class ExecutionGraphConstructionTest {
v3.setInvokableClass(AbstractInvokable.class);
// this creates an intermediate result for v1
- v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
// create results for v2 and v3
IntermediateDataSet v2result = v2.createAndAddResultDataSet(ResultPartitionType.PIPELINED);
@@ -193,7 +193,7 @@ public class ExecutionGraphConstructionTest {
v4.connectDataSetAsInput(v2result, DistributionPattern.ALL_TO_ALL);
v4.connectDataSetAsInput(v3result_1, DistributionPattern.ALL_TO_ALL);
- v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL);
+ v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectDataSetAsInput(v3result_2, DistributionPattern.ALL_TO_ALL);
List<JobVertex> ordered2 = new ArrayList<JobVertex>(Arrays.asList(v4, v5));
@@ -230,7 +230,7 @@ public class ExecutionGraphConstructionTest {
v3.setInvokableClass(AbstractInvokable.class);
// this creates an intermediate result for v1
- v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
// create results for v2 and v3
IntermediateDataSet v2result = v2.createAndAddResultDataSet(ResultPartitionType.PIPELINED);
@@ -269,7 +269,7 @@ public class ExecutionGraphConstructionTest {
v4.connectIdInput(v2result.getId(), DistributionPattern.ALL_TO_ALL);
v4.connectIdInput(v3result_1.getId(), DistributionPattern.ALL_TO_ALL);
- v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL);
+ v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v5.connectIdInput(v3result_2.getId(), DistributionPattern.ALL_TO_ALL);
List<JobVertex> ordered2 = new ArrayList<JobVertex>(Arrays.asList(v4, v5));
@@ -558,11 +558,11 @@ public class ExecutionGraphConstructionTest {
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL);
- v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL);
- v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL);
- v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL);
- v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v5, v4));
@@ -625,11 +625,11 @@ public class ExecutionGraphConstructionTest {
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL);
- v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL);
- v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL);
- v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL);
- v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
v3.setInputSplitSource(source1);
v5.setInputSplitSource(source2);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
index 3d2913f..30824e0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
@@ -90,9 +90,9 @@ public class ExecutionGraphDeploymentTest {
v3.setInvokableClass(BatchTask.class);
v4.setInvokableClass(BatchTask.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL);
- v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL);
- v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
ExecutionGraph eg = new ExecutionGraph(
TestingUtils.defaultExecutor(),
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
index 64b9aa2..27844c1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSignalsTest.java
@@ -31,6 +31,7 @@ import org.apache.flink.runtime.execution.SuppressRestartsException;
import org.apache.flink.runtime.executiongraph.restart.InfiniteDelayRestartStrategy;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
@@ -119,15 +120,15 @@ public class ExecutionGraphSignalsTest {
v4.setParallelism(dop[3]);
v5.setParallelism(dop[4]);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
mockNumberOfInputs(1,0);
- v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL);
+ v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
mockNumberOfInputs(3,1);
- v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL);
+ v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
mockNumberOfInputs(3,2);
- v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL);
+ v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
mockNumberOfInputs(4,3);
- v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL);
+ v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
mockNumberOfInputs(4,2);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
index cfd4665..eb85a33 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy;
import org.apache.flink.runtime.instance.SimpleSlot;
import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -200,7 +201,7 @@ public class ExecutionVertexLocalityTest extends TestLogger {
target.setInvokableClass(NoOpInvokable.class);
DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
- target.connectNewDataSetAsInput(source, connectionPattern);
+ target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);
JobGraph testJob = new JobGraph(jobId, "test job", source, target);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java
index 5629c0b..8ff0032 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PointwisePatternTest.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.fail;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.testingUtils.TestingUtils;
@@ -62,7 +63,7 @@ public class PointwisePatternTest {
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
@@ -109,7 +110,7 @@ public class PointwisePatternTest {
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
@@ -157,7 +158,7 @@ public class PointwisePatternTest {
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
@@ -206,7 +207,7 @@ public class PointwisePatternTest {
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
@@ -253,7 +254,7 @@ public class PointwisePatternTest {
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
@@ -320,7 +321,7 @@ public class PointwisePatternTest {
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
@@ -378,7 +379,7 @@ public class PointwisePatternTest {
v1.setInvokableClass(AbstractInvokable.class);
v2.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java
index bf17485..90e3368 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/VertexSlotSharingTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.api.common.JobID;
@@ -67,8 +68,8 @@ public class VertexSlotSharingTest {
v4.setInvokableClass(AbstractInvokable.class);
v5.setInvokableClass(AbstractInvokable.class);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
- v5.connectNewDataSetAsInput(v4, DistributionPattern.POINTWISE);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ v5.connectNewDataSetAsInput(v4, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
SlotSharingGroup jg1 = new SlotSharingGroup();
v2.setSlotSharingGroup(jg1);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/MockNetworkEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/MockNetworkEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/MockNetworkEnvironment.java
index 718ab45..dcdf44c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/MockNetworkEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/MockNetworkEnvironment.java
@@ -36,7 +36,7 @@ public class MockNetworkEnvironment {
static {
try {
- when(networkBufferPool.createBufferPool(anyInt(), anyBoolean())).thenReturn(mock(BufferPool.class));
+ when(networkBufferPool.createBufferPool(anyInt())).thenReturn(mock(BufferPool.class));
when(networkEnvironment.getNetworkBufferPool()).thenReturn(networkBufferPool);
when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java
index 900b5c3..ca1d0a5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java
@@ -177,7 +177,7 @@ public class RecordWriterTest {
try {
buffers = new NetworkBufferPool(1, 1024, MemoryType.HEAP);
- bufferPool = spy(buffers.createBufferPool(1, true));
+ bufferPool = spy(buffers.createBufferPool(1));
ResultPartitionWriter partitionWriter = mock(ResultPartitionWriter.class);
when(partitionWriter.getBufferProvider()).thenReturn(checkNotNull(bufferPool));
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactoryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactoryTest.java
index 0ac84dc..49808c9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactoryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferPoolFactoryTest.java
@@ -52,37 +52,20 @@ public class BufferPoolFactoryTest {
@Test(expected = IOException.class)
public void testRequireMoreThanPossible() throws IOException {
- networkBufferPool.createBufferPool(networkBufferPool.getTotalNumberOfMemorySegments() * 2, false);
- }
-
- @Test
- public void testFixedPool() throws IOException {
- BufferPool lbp = networkBufferPool.createBufferPool(1, true);
-
- assertEquals(1, lbp.getNumBuffers());
+ networkBufferPool.createBufferPool(networkBufferPool.getTotalNumberOfMemorySegments() * 2);
}
@Test
public void testSingleManagedPoolGetsAll() throws IOException {
- BufferPool lbp = networkBufferPool.createBufferPool(1, false);
+ BufferPool lbp = networkBufferPool.createBufferPool(1);
assertEquals(networkBufferPool.getTotalNumberOfMemorySegments(), lbp.getNumBuffers());
}
@Test
- public void testSingleManagedPoolGetsAllExceptFixedOnes() throws IOException {
- BufferPool fixed = networkBufferPool.createBufferPool(24, true);
-
- BufferPool lbp = networkBufferPool.createBufferPool(1, false);
-
- assertEquals(24, fixed.getNumBuffers());
- assertEquals(networkBufferPool.getTotalNumberOfMemorySegments() - fixed.getNumBuffers(), lbp.getNumBuffers());
- }
-
- @Test
public void testUniformDistribution() throws IOException {
- BufferPool first = networkBufferPool.createBufferPool(0, false);
- BufferPool second = networkBufferPool.createBufferPool(0, false);
+ BufferPool first = networkBufferPool.createBufferPool(0);
+ BufferPool second = networkBufferPool.createBufferPool(0);
assertEquals(networkBufferPool.getTotalNumberOfMemorySegments() / 2, first.getNumBuffers());
assertEquals(networkBufferPool.getTotalNumberOfMemorySegments() / 2, second.getNumBuffers());
@@ -97,7 +80,7 @@ public class BufferPoolFactoryTest {
int numPools = numBuffers / 32;
for (int i = 0; i < numPools; i++) {
- pools.add(networkBufferPool.createBufferPool(random.nextInt(7 + 1), random.nextBoolean()));
+ pools.add(networkBufferPool.createBufferPool(random.nextInt(7 + 1)));
}
int numDistributedBuffers = 0;
@@ -115,11 +98,11 @@ public class BufferPoolFactoryTest {
@Test
public void testCreateDestroy() throws IOException {
- BufferPool first = networkBufferPool.createBufferPool(0, false);
+ BufferPool first = networkBufferPool.createBufferPool(0);
assertEquals(networkBufferPool.getTotalNumberOfMemorySegments(), first.getNumBuffers());
- BufferPool second = networkBufferPool.createBufferPool(0, false);
+ BufferPool second = networkBufferPool.createBufferPool(0);
assertEquals(networkBufferPool.getTotalNumberOfMemorySegments() / 2, first.getNumBuffers());
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPoolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPoolTest.java
index fd5c7a5..ab32685 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPoolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/NetworkBufferPoolTest.java
@@ -47,15 +47,7 @@ public class NetworkBufferPoolTest {
assertTrue(globalPool.isDestroyed());
try {
- globalPool.createBufferPool(2, true);
- fail("Should throw an IllegalStateException");
- }
- catch (IllegalStateException e) {
- // yippie!
- }
-
- try {
- globalPool.createBufferPool(2, false);
+ globalPool.createBufferPool(2);
fail("Should throw an IllegalStateException");
}
catch (IllegalStateException e) {
@@ -71,26 +63,21 @@ public class NetworkBufferPoolTest {
@Test
public void testDestroyAll() {
try {
- NetworkBufferPool globalPool = new NetworkBufferPool(10, 128, MemoryType.HEAP);
+ NetworkBufferPool globalPool = new NetworkBufferPool(8, 128, MemoryType.HEAP);
- BufferPool fixedPool = globalPool.createBufferPool(2, true);
- BufferPool nonFixedPool = globalPool.createBufferPool(5, false);
+ BufferPool lbp = globalPool.createBufferPool(5);
- assertEquals(2, fixedPool.getNumberOfRequiredMemorySegments());
- assertEquals(5, nonFixedPool.getNumberOfRequiredMemorySegments());
+ assertEquals(5, lbp.getNumberOfRequiredMemorySegments());
Buffer[] buffers = {
- fixedPool.requestBuffer(),
- fixedPool.requestBuffer(),
-
- nonFixedPool.requestBuffer(),
- nonFixedPool.requestBuffer(),
- nonFixedPool.requestBuffer(),
- nonFixedPool.requestBuffer(),
- nonFixedPool.requestBuffer(),
- nonFixedPool.requestBuffer(),
- nonFixedPool.requestBuffer(),
- nonFixedPool.requestBuffer()
+ lbp.requestBuffer(),
+ lbp.requestBuffer(),
+ lbp.requestBuffer(),
+ lbp.requestBuffer(),
+ lbp.requestBuffer(),
+ lbp.requestBuffer(),
+ lbp.requestBuffer(),
+ lbp.requestBuffer()
};
for (Buffer b : buffers) {
@@ -98,16 +85,14 @@ public class NetworkBufferPoolTest {
assertNotNull(b.getMemorySegment());
}
- assertNull(fixedPool.requestBuffer());
- assertNull(nonFixedPool.requestBuffer());
+ assertNull(lbp.requestBuffer());
// destroy all allocated ones
globalPool.destroyAllBufferPools();
// check the destroyed status
assertFalse(globalPool.isDestroyed());
- assertTrue(fixedPool.isDestroyed());
- assertTrue(nonFixedPool.isDestroyed());
+ assertTrue(lbp.isDestroyed());
assertEquals(0, globalPool.getNumberOfRegisteredBufferPools());
@@ -122,15 +107,7 @@ public class NetworkBufferPoolTest {
// can request no more buffers
try {
- fixedPool.requestBuffer();
- fail("Should fail with an IllegalStateException");
- }
- catch (IllegalStateException e) {
- // that's the way we like it, aha, aha
- }
-
- try {
- nonFixedPool.requestBuffer();
+ lbp.requestBuffer();
fail("Should fail with an IllegalStateException");
}
catch (IllegalStateException e) {
@@ -138,7 +115,7 @@ public class NetworkBufferPoolTest {
}
// can create a new pool now
- assertNotNull(globalPool.createBufferPool(10, false));
+ assertNotNull(globalPool.createBufferPool(8));
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
index e05fb56..15ff2da 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
@@ -128,7 +128,7 @@ public class LocalInputChannelTest {
// Create a buffer pool for this partition
partition.registerBufferPool(
- networkBuffers.createBufferPool(producerBufferPoolSize, true));
+ networkBuffers.createBufferPool(producerBufferPoolSize));
// Create the producer
partitionProducers[i] = new TestPartitionProducer(
@@ -162,7 +162,7 @@ public class LocalInputChannelTest {
i,
parallelism,
numberOfBuffersPerChannel,
- networkBuffers.createBufferPool(parallelism, true),
+ networkBuffers.createBufferPool(parallelism),
partitionManager,
new TaskEventDispatcher(),
partitionIds)));
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java
index 74f1adf..9f06c6a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobGraphTest.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.junit.Test;
public class JobGraphTest {
@@ -44,8 +45,8 @@ public class JobGraphTest {
JobVertex source1 = new JobVertex("source1");
JobVertex source2 = new JobVertex("source2");
JobVertex target = new JobVertex("target");
- target.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE);
- target.connectNewDataSetAsInput(source2, DistributionPattern.ALL_TO_ALL);
+ target.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ target.connectNewDataSetAsInput(source2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
jg.addVertex(source1);
jg.addVertex(source2);
@@ -84,11 +85,11 @@ public class JobGraphTest {
JobVertex intermediate1 = new JobVertex("intermediate1");
JobVertex intermediate2 = new JobVertex("intermediate2");
- target1.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE);
- target2.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE);
- target2.connectNewDataSetAsInput(intermediate2, DistributionPattern.POINTWISE);
- intermediate2.connectNewDataSetAsInput(intermediate1, DistributionPattern.POINTWISE);
- intermediate1.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE);
+ target1.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ target2.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ target2.connectNewDataSetAsInput(intermediate2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ intermediate2.connectNewDataSetAsInput(intermediate1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ intermediate1.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
JobGraph graph = new JobGraph("TestGraph",
source1, source2, intermediate1, intermediate2, target1, target2);
@@ -121,19 +122,19 @@ public class JobGraphTest {
JobVertex l13 = new JobVertex("layer 1 - 3");
JobVertex l2 = new JobVertex("layer 2");
- root.connectNewDataSetAsInput(l13, DistributionPattern.POINTWISE);
- root.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE);
- root.connectNewDataSetAsInput(l2, DistributionPattern.POINTWISE);
+ root.connectNewDataSetAsInput(l13, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ root.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ root.connectNewDataSetAsInput(l2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
- l2.connectNewDataSetAsInput(l11, DistributionPattern.POINTWISE);
- l2.connectNewDataSetAsInput(l12, DistributionPattern.POINTWISE);
+ l2.connectNewDataSetAsInput(l11, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ l2.connectNewDataSetAsInput(l12, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
- l11.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE);
+ l11.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
- l12.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE);
- l12.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE);
+ l12.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ l12.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
- l13.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE);
+ l13.connectNewDataSetAsInput(source2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
JobGraph graph = new JobGraph("TestGraph",
source1, source2, root, l11, l13, l12, l2);
@@ -177,10 +178,10 @@ public class JobGraphTest {
JobVertex op2 = new JobVertex("op2");
JobVertex op3 = new JobVertex("op3");
- op1.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE);
- op2.connectNewDataSetAsInput(op1, DistributionPattern.POINTWISE);
- op2.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE);
- op3.connectNewDataSetAsInput(op2, DistributionPattern.POINTWISE);
+ op1.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ op2.connectNewDataSetAsInput(op1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ op2.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ op3.connectNewDataSetAsInput(op2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
JobGraph graph = new JobGraph("TestGraph", source, op1, op2, op3);
List<JobVertex> sorted = graph.getVerticesSortedTopologicallyFromSources();
@@ -206,10 +207,10 @@ public class JobGraphTest {
JobVertex v3 = new JobVertex("3");
JobVertex v4 = new JobVertex("4");
- v1.connectNewDataSetAsInput(v4, DistributionPattern.POINTWISE);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
- v3.connectNewDataSetAsInput(v2, DistributionPattern.POINTWISE);
- v4.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE);
+ v1.connectNewDataSetAsInput(v4, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ v3.connectNewDataSetAsInput(v2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
JobGraph jg = new JobGraph("Cyclic Graph", v1, v2, v3, v4);
try {
@@ -236,12 +237,12 @@ public class JobGraphTest {
JobVertex v4 = new JobVertex("4");
JobVertex target = new JobVertex("target");
- v1.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE);
- v1.connectNewDataSetAsInput(v4, DistributionPattern.POINTWISE);
- v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
- v3.connectNewDataSetAsInput(v2, DistributionPattern.POINTWISE);
- v4.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE);
- target.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE);
+ v1.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ v1.connectNewDataSetAsInput(v4, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ v3.connectNewDataSetAsInput(v2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ v4.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ target.connectNewDataSetAsInput(v3, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
JobGraph jg = new JobGraph("Cyclic Graph", v1, v2, v3, v4, source, target);
try {
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobTaskVertexTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobTaskVertexTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobTaskVertexTest.java
index 48f06b0..d94b93e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobTaskVertexTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/JobTaskVertexTest.java
@@ -27,6 +27,7 @@ import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.java.io.DiscardingOutputFormat;
import org.apache.flink.core.io.GenericInputSplit;
import org.apache.flink.core.io.InputSplit;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.operators.util.TaskConfig;
import org.junit.Test;
@@ -41,7 +42,7 @@ public class JobTaskVertexTest {
public void testConnectDirectly() {
JobVertex source = new JobVertex("source");
JobVertex target = new JobVertex("target");
- target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE);
+ target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
assertTrue(source.isInputVertex());
assertFalse(source.isOutputVertex());
@@ -62,7 +63,7 @@ public class JobTaskVertexTest {
JobVertex source = new JobVertex("source");
JobVertex target1= new JobVertex("target1");
JobVertex target2 = new JobVertex("target2");
- target1.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE);
+ target1.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
target2.connectDataSetAsInput(source.getProducedDataSets().get(0), DistributionPattern.ALL_TO_ALL);
assertTrue(source.isInputVertex());
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java
index d1d5f03..62b9b40 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobgraph/jsonplan/JsonGeneratorTest.java
@@ -64,8 +64,8 @@ public class JsonGeneratorTest {
join2.connectNewDataSetAsInput(join1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
join2.connectNewDataSetAsInput(source3, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
- sink1.connectNewDataSetAsInput(join2, DistributionPattern.POINTWISE);
- sink2.connectNewDataSetAsInput(join1, DistributionPattern.ALL_TO_ALL);
+ sink1.connectNewDataSetAsInput(join2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
+ sink2.connectNewDataSetAsInput(join1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
JobGraph jg = new JobGraph("my job", source1, source2, source3,
intermediate1, intermediate2, join1, join2, sink1, sink2);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
index fe33022..49d0239 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeJobRecoveryTest.java
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.executiongraph.TerminalJobStatusListener;
import org.apache.flink.runtime.instance.ActorGateway;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -129,7 +130,7 @@ public class LeaderChangeJobRecoveryTest extends TestLogger {
sender.setParallelism(parallelism);
receiver.setParallelism(parallelism);
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
sender.setSlotSharingGroup(slotSharingGroup);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java
index 19cc444..7ae9974 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeStateCleanupTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.leaderelection;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.instance.ActorGateway;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -272,7 +273,7 @@ public class LeaderChangeStateCleanupTest extends TestLogger {
sender.setParallelism(parallelism);
receiver.setParallelism(parallelism);
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
sender.setSlotSharingGroup(slotSharingGroup);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java
index 876e908..5a14b40 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.instance.ActorGateway;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -91,7 +92,7 @@ public class TaskCancelAsyncProducerConsumerITCase extends TestLogger {
JobVertex consumer = new JobVertex("AsyncConsumer");
consumer.setParallelism(1);
consumer.setInvokableClass(AsyncConsumer.class);
- consumer.connectNewDataSetAsInput(producer, DistributionPattern.POINTWISE);
+ consumer.connectNewDataSetAsInput(producer, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
SlotSharingGroup slot = new SlotSharingGroup(producer.getID(), consumer.getID());
producer.setSlotSharingGroup(slot);
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala
index 12e2d63..d4b4cbf 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/CoLocationConstraintITCase.scala
@@ -19,18 +19,18 @@
package org.apache.flink.runtime.jobmanager
import akka.actor.ActorSystem
-import akka.actor.Status.Success
import akka.testkit.{ImplicitSender, TestKit}
-import org.apache.flink.api.common.ExecutionConfig
import org.apache.flink.runtime.akka.ListeningBehaviour
-import org.apache.flink.runtime.jobgraph.{JobGraph, DistributionPattern, JobVertex}
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType
+import org.apache.flink.runtime.jobgraph.{DistributionPattern, JobGraph, JobVertex}
import org.apache.flink.runtime.jobmanager.Tasks.{Receiver, Sender}
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup
-import org.apache.flink.runtime.messages.JobManagerMessages.{JobSubmitSuccess, JobResultSuccess, SubmitJob}
+import org.apache.flink.runtime.messages.JobManagerMessages.{JobResultSuccess, JobSubmitSuccess, SubmitJob}
import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingUtils}
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
+
import scala.concurrent.duration._
@RunWith(classOf[JUnitRunner])
@@ -60,7 +60,8 @@ class CoLocationConstraintITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val sharingGroup = new SlotSharingGroup(sender.getID, receiver.getID)
sender.setSlotSharingGroup(sharingGroup)
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
index 31e72dd..5374d01 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
@@ -26,6 +26,7 @@ import org.apache.flink.runtime.akka.ListeningBehaviour
import org.apache.flink.runtime.checkpoint.{CheckpointCoordinator, CompletedCheckpoint}
import org.apache.flink.runtime.client.JobExecutionException
import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType
import org.apache.flink.runtime.jobgraph.tasks.{ExternalizedCheckpointSettings, JobSnapshottingSettings}
import org.apache.flink.runtime.jobgraph.{DistributionPattern, JobGraph, JobVertex, ScheduleMode}
import org.apache.flink.runtime.jobmanager.Tasks._
@@ -180,7 +181,8 @@ class JobManagerITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Pointwise Job", sender, receiver)
@@ -215,7 +217,8 @@ class JobManagerITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Bipartite Job", sender, receiver)
@@ -251,8 +254,10 @@ class JobManagerITCase(_system: ActorSystem)
sender2.setParallelism(2 * num_tasks)
receiver.setParallelism(3 * num_tasks)
- receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE)
- receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL)
+ receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
+ receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2)
@@ -296,8 +301,10 @@ class JobManagerITCase(_system: ActorSystem)
sender2.setParallelism(2 * num_tasks)
receiver.setParallelism(3 * num_tasks)
- receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE)
- receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL)
+ receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
+ receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2)
@@ -338,8 +345,10 @@ class JobManagerITCase(_system: ActorSystem)
forwarder.setSlotSharingGroup(sharingGroup)
receiver.setSlotSharingGroup(sharingGroup)
- forwarder.connectNewDataSetAsInput(sender, DistributionPattern.ALL_TO_ALL)
- receiver.connectNewDataSetAsInput(forwarder, DistributionPattern.ALL_TO_ALL)
+ forwarder.connectNewDataSetAsInput(sender, DistributionPattern.ALL_TO_ALL,
+ ResultPartitionType.PIPELINED)
+ receiver.connectNewDataSetAsInput(forwarder, DistributionPattern.ALL_TO_ALL,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Forwarding Job", sender, forwarder, receiver)
@@ -375,7 +384,8 @@ class JobManagerITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Pointwise Job", sender, receiver)
@@ -423,7 +433,8 @@ class JobManagerITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Pointwise Job", sender, receiver)
@@ -468,7 +479,8 @@ class JobManagerITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Pointwise job", sender, receiver)
@@ -508,7 +520,8 @@ class JobManagerITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Pointwise job", sender, receiver)
@@ -556,7 +569,8 @@ class JobManagerITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Pointwise job", sender, receiver)
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala
index b96369f..f3ab409 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/RecoveryITCase.scala
@@ -18,22 +18,23 @@
package org.apache.flink.runtime.jobmanager
-import akka.actor.{PoisonPill, ActorSystem}
+import akka.actor.{ActorSystem, PoisonPill}
import akka.testkit.{ImplicitSender, TestKit}
import org.apache.flink.api.common.ExecutionConfig
import org.apache.flink.configuration.{ConfigConstants, Configuration}
import org.apache.flink.runtime.akka.ListeningBehaviour
-import org.apache.flink.runtime.jobgraph.{JobStatus, JobGraph, DistributionPattern, JobVertex}
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType
+import org.apache.flink.runtime.jobgraph.{DistributionPattern, JobGraph, JobStatus, JobVertex}
import org.apache.flink.runtime.jobmanager.Tasks.{BlockingOnceReceiver, FailingOnceReceiver}
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup
-import org.apache.flink.runtime.messages.JobManagerMessages.{JobSubmitSuccess, JobResultSuccess, SubmitJob}
+import org.apache.flink.runtime.messages.JobManagerMessages.{JobResultSuccess, JobSubmitSuccess, SubmitJob}
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._
import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingCluster, TestingUtils}
import org.junit.runner.RunWith
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
import org.scalatest.junit.JUnitRunner
-import scala.concurrent.duration._
+import scala.concurrent.duration._
import language.postfixOps
@RunWith(classOf[JUnitRunner])
@@ -81,7 +82,8 @@ class RecoveryITCase(_system: ActorSystem)
sender.setParallelism(NUM_TASKS)
receiver.setParallelism(NUM_TASKS)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val executionConfig = new ExecutionConfig()
executionConfig.setNumberOfExecutionRetries(1);
@@ -125,7 +127,8 @@ class RecoveryITCase(_system: ActorSystem)
sender.setParallelism(NUM_TASKS)
receiver.setParallelism(NUM_TASKS)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val sharingGroup = new SlotSharingGroup
sender.setSlotSharingGroup(sharingGroup)
@@ -173,7 +176,8 @@ class RecoveryITCase(_system: ActorSystem)
sender.setParallelism(NUM_TASKS)
receiver.setParallelism(NUM_TASKS)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val sharingGroup = new SlotSharingGroup
sender.setSlotSharingGroup(sharingGroup)
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala
index f986e73..4fffd68 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/SlotSharingITCase.scala
@@ -19,18 +19,18 @@
package org.apache.flink.runtime.jobmanager
import akka.actor.ActorSystem
-import akka.actor.Status.Success
import akka.testkit.{ImplicitSender, TestKit}
-import org.apache.flink.api.common.ExecutionConfig
import org.apache.flink.runtime.akka.ListeningBehaviour
-import org.apache.flink.runtime.jobgraph.{JobVertex, DistributionPattern, JobGraph}
-import org.apache.flink.runtime.jobmanager.Tasks.{Sender, AgnosticBinaryReceiver, Receiver}
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType
+import org.apache.flink.runtime.jobgraph.{DistributionPattern, JobGraph, JobVertex}
+import org.apache.flink.runtime.jobmanager.Tasks.{AgnosticBinaryReceiver, Receiver, Sender}
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup
-import org.apache.flink.runtime.messages.JobManagerMessages.{JobSubmitSuccess, JobResultSuccess, SubmitJob}
+import org.apache.flink.runtime.messages.JobManagerMessages.{JobResultSuccess, JobSubmitSuccess, SubmitJob}
import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingUtils}
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
+
import scala.concurrent.duration._
@RunWith(classOf[JUnitRunner])
@@ -60,7 +60,8 @@ class SlotSharingITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val sharingGroup = new SlotSharingGroup(sender.getID, receiver.getID)
sender.setSlotSharingGroup(sharingGroup)
@@ -107,8 +108,10 @@ class SlotSharingITCase(_system: ActorSystem)
sender2.setSlotSharingGroup(sharingGroup)
receiver.setSlotSharingGroup(sharingGroup)
- receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE)
- receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL)
+ receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
+ receiver.connectNewDataSetAsInput(sender2, DistributionPattern.ALL_TO_ALL,
+ ResultPartitionType.PIPELINED)
val jobGraph = new JobGraph("Bipartite job", sender1, sender2, receiver)
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala
index d0136f0..9775d33 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/TaskManagerFailsWithSlotSharingITCase.scala
@@ -18,12 +18,13 @@
package org.apache.flink.runtime.jobmanager
-import akka.actor.{Kill, ActorSystem, PoisonPill}
+import akka.actor.{ActorSystem, Kill, PoisonPill}
import akka.testkit.{ImplicitSender, TestKit}
import org.apache.flink.api.common.ExecutionConfig
import org.apache.flink.runtime.akka.ListeningBehaviour
import org.apache.flink.runtime.client.JobExecutionException
-import org.apache.flink.runtime.jobgraph.{JobVertex, DistributionPattern, JobGraph}
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType
+import org.apache.flink.runtime.jobgraph.{DistributionPattern, JobGraph, JobVertex}
import org.apache.flink.runtime.jobmanager.Tasks.{BlockingReceiver, Sender}
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup
import org.apache.flink.runtime.messages.JobManagerMessages.{JobResultFailure, JobSubmitSuccess, SubmitJob}
@@ -32,6 +33,7 @@ import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingUtils}
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
+
import scala.concurrent.duration._
@RunWith(classOf[JUnitRunner])
@@ -61,7 +63,8 @@ class TaskManagerFailsWithSlotSharingITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val sharingGroup = new SlotSharingGroup()
sender.setSlotSharingGroup(sharingGroup)
@@ -110,7 +113,8 @@ class TaskManagerFailsWithSlotSharingITCase(_system: ActorSystem)
sender.setParallelism(num_tasks)
receiver.setParallelism(num_tasks)
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE)
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED)
val sharingGroup = new SlotSharingGroup()
sender.setSlotSharingGroup(sharingGroup)
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java
index 6e088f6..ec55f19 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferMassiveRandomTest.java
@@ -51,8 +51,8 @@ public class BarrierBufferMassiveRandomTest {
try {
ioMan = new IOManagerAsync();
- BufferPool pool1 = new NetworkBufferPool(100, PAGE_SIZE, MemoryType.HEAP).createBufferPool(100, true);
- BufferPool pool2 = new NetworkBufferPool(100, PAGE_SIZE, MemoryType.HEAP).createBufferPool(100, true);
+ BufferPool pool1 = new NetworkBufferPool(100, PAGE_SIZE, MemoryType.HEAP).createBufferPool(100);
+ BufferPool pool2 = new NetworkBufferPool(100, PAGE_SIZE, MemoryType.HEAP).createBufferPool(100);
RandomGeneratingInputGate myIG = new RandomGeneratingInputGate(
new BufferPool[] { pool1, pool2 },
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java
index 3d0e5ab..b38df61 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java
@@ -25,6 +25,7 @@ import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.runtime.io.network.api.reader.RecordReader;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -129,11 +130,14 @@ public class NetworkStackThroughputITCase {
consumer.getConfiguration().setBoolean(IS_SLOW_RECEIVER_CONFIG_KEY, isSlowReceiver);
if (useForwarder) {
- forwarder.connectNewDataSetAsInput(producer, DistributionPattern.ALL_TO_ALL);
- consumer.connectNewDataSetAsInput(forwarder, DistributionPattern.ALL_TO_ALL);
+ forwarder.connectNewDataSetAsInput(producer, DistributionPattern.ALL_TO_ALL,
+ ResultPartitionType.PIPELINED);
+ consumer.connectNewDataSetAsInput(forwarder, DistributionPattern.ALL_TO_ALL,
+ ResultPartitionType.PIPELINED);
}
else {
- consumer.connectNewDataSetAsInput(producer, DistributionPattern.ALL_TO_ALL);
+ consumer.connectNewDataSetAsInput(producer, DistributionPattern.ALL_TO_ALL,
+ ResultPartitionType.PIPELINED);
}
return jobGraph;
http://git-wip-us.apache.org/repos/asf/flink/blob/8b49ee5a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
index eacdeb4..df4f370 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java
@@ -28,6 +28,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.client.JobClient;
import org.apache.flink.runtime.instance.ActorGateway;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -165,7 +166,8 @@ public class ZooKeeperLeaderElectionITCase extends TestLogger {
sender.setParallelism(parallelism);
receiver.setParallelism(parallelism);
- receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
+ receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE,
+ ResultPartitionType.PIPELINED);
SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
sender.setSlotSharingGroup(slotSharingGroup);