You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by uc...@apache.org on 2015/01/12 09:16:16 UTC

[08/18] flink git commit: [FLINK-986] [FLINK-25] [Distributed runtime] Add initial support for intermediate results

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java
index e29b149..bc815dc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java
@@ -31,11 +31,11 @@ import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.core.memory.InputViewDataInputStreamWrapper;
 import org.apache.flink.core.memory.OutputViewDataOutputStreamWrapper;
-import org.apache.flink.runtime.event.task.AbstractTaskEvent;
+import org.apache.flink.runtime.event.task.TaskEvent;
 import org.apache.flink.types.Value;
 import org.apache.flink.util.InstantiationUtil;
 
-public abstract class IterationEventWithAggregators extends AbstractTaskEvent {
+public abstract class IterationEventWithAggregators extends TaskEvent {
 	
 	protected static final String[] NO_STRINGS = new String[0];
 	protected static final Value[] NO_VALUES = new Value[0];

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java
index 6cc1e10..9e74c34 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java
@@ -23,12 +23,12 @@ import java.io.IOException;
 
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.runtime.event.task.AbstractTaskEvent;
+import org.apache.flink.runtime.event.task.TaskEvent;
 
 /**
  * Signals that the iteration is completely executed, participating tasks must terminate now
  */
-public class TerminationEvent extends AbstractTaskEvent {
+public class TerminationEvent extends TaskEvent {
 
 	public static final TerminationEvent INSTANCE = new TerminationEvent();
 	

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
index c73c1ea..3b2eda5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
@@ -26,13 +26,12 @@ import org.apache.flink.util.Collector;
 
 /**
  * A {@link Collector} to update the solution set of a workset iteration.
- * <p/>
+ * <p>
  * The records are written to a HashTable hash table to allow in-memory point updates.
- * <p/>
+ * <p>
  * Assumption for fast updates: the build side iterator of the hash table is already positioned for the update. This
  * is for example the case when a solution set update happens directly after a solution set join. If this assumption
  * doesn't hold, use {@link SolutionSetUpdateOutputCollector}, which probes the hash table before updating.
-
  */
 public class SolutionSetFastUpdateOutputCollector<T> implements Collector<T> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java
index 502dfc9..21a9cc8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java
@@ -24,9 +24,9 @@ import org.apache.flink.util.Collector;
 
 /**
  * A {@link Collector} to update the solution set of a workset iteration.
- * <p/>
+ * <p>
  * The records are written to a HashTable hash table to allow in-memory point updates.
- * <p/>
+ * <p>
  * Records will only be collected, if there is a match after probing the hash table. If the build side iterator is
  * already positioned for the update, use {@link SolutionSetFastUpdateOutputCollector} to the save re-probing.
  * 

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
index 0bc2dd2..f400b7e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
@@ -27,9 +27,9 @@ import org.apache.flink.util.Collector;
 
 /**
  * A {@link Collector} to update the solution set of a workset iteration.
- * <p/>
+ * <p>
  * The records are written to a HashTable hash table to allow in-memory point updates.
- * <p/>
+ * <p>
  * Records will only be collected, if there is a match after probing the hash table. If the build side iterator is
  * already positioned for the update, use {@link SolutionSetFastUpdateOutputCollector} to the save re-probing.
  * 

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java
index 22bc46d..ad1d274 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java
@@ -27,7 +27,7 @@ import org.apache.flink.util.Collector;
 
 /**
  * A {@link Collector} to update the iteration workset (partial solution for bulk iterations).
- * <p/>
+ * <p>
  * The records are written to a {@link DataOutputView} to allow in-memory data exchange.
  */
 public class WorksetUpdateOutputCollector<T> implements Collector<T> {

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
index 1008256..4f10683 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
@@ -29,7 +29,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
 import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.runtime.execution.Environment;
-import org.apache.flink.runtime.io.network.api.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
 import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
 import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannelBroker;
 import org.apache.flink.runtime.iterative.concurrent.Broker;
@@ -164,7 +164,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 	@Override
 	public DistributedRuntimeUDFContext createRuntimeContext(String taskName) {
 		Environment env = getEnvironment();
-		return new IterativeRuntimeUdfContext(taskName, env.getCurrentNumberOfSubtasks(),
+		return new IterativeRuntimeUdfContext(taskName, env.getNumberOfSubtasks(),
 				env.getIndexInSubtaskGroup(), getUserCodeClassLoader());
 	}
 
@@ -227,7 +227,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 		for (int inputNum : this.iterativeInputs) {
 			MutableReader<?> reader = this.inputReaders[inputNum];
 
-			if (!reader.isInputClosed()) {
+			if (!reader.isFinished()) {
 				if (reader.hasReachedEndOfSuperstep()) {
 					reader.startNextSuperstep();
 				}
@@ -238,7 +238,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 					Object o = this.inputSerializers[inputNum].getSerializer().createInstance();
 					while ((o = inIter.next(o)) != null);
 					
-					if (!reader.isInputClosed()) {
+					if (!reader.isFinished()) {
 						// also reset the end-of-superstep state
 						reader.startNextSuperstep();
 					}
@@ -249,7 +249,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 		for (int inputNum : this.iterativeBroadcastInputs) {
 			MutableReader<?> reader = this.broadcastInputReaders[inputNum];
 
-			if (!reader.isInputClosed()) {
+			if (!reader.isFinished()) {
 				
 				// sanity check that the BC input is at the end of the superstep
 				if (!reader.hasReachedEndOfSuperstep()) {
@@ -283,10 +283,10 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 
 	/**
 	 * Creates a new {@link WorksetUpdateOutputCollector}.
-	 * <p/>
+	 * <p>
 	 * This collector is used by {@link IterationIntermediatePactTask} or {@link IterationTailPactTask} to update the
 	 * workset.
-	 * <p/>
+	 * <p>
 	 * If a non-null delegate is given, the new {@link Collector} will write to the solution set and also call
 	 * collect(T) of the delegate.
 	 *
@@ -305,12 +305,12 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 
 	/**
 	 * Creates a new solution set update output collector.
-	 * <p/>
+	 * <p>
 	 * This collector is used by {@link IterationIntermediatePactTask} or {@link IterationTailPactTask} to update the
 	 * solution set of workset iterations. Depending on the task configuration, either a fast (non-probing)
 	 * {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or normal (re-probing)
 	 * {@link SolutionSetUpdateOutputCollector} is created.
-	 * <p/>
+	 * <p>
 	 * If a non-null delegate is given, the new {@link Collector} will write back to the solution set and also call
 	 * collect(T) of the delegate.
 	 *

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
index ea8efb6..b8e4857 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
@@ -22,6 +22,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.flink.runtime.io.network.api.writer.BufferWriter;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,7 +36,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.MemorySegment;
 import org.apache.flink.runtime.io.disk.InputViewIterator;
-import org.apache.flink.runtime.io.network.api.BufferWriter;
 import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
 import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannelBroker;
 import org.apache.flink.runtime.iterative.concurrent.Broker;
@@ -84,7 +85,7 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 
 	private Collector<X> finalOutputCollector;
 
-	private List<BufferWriter> finalOutputWriters;
+	private List<RecordWriter<?>> finalOutputWriters;
 
 	private TypeSerializerFactory<Y> feedbackTypeSerializer;
 
@@ -114,11 +115,11 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 
 		// at this time, the outputs to the step function are created
 		// add the outputs for the final solution
-		this.finalOutputWriters = new ArrayList<BufferWriter>();
+		this.finalOutputWriters = new ArrayList<RecordWriter<?>>();
 		final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
 		final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
 		this.finalOutputCollector = RegularPactTask.getOutputCollector(this, finalOutConfig,
-			userCodeClassLoader, this.finalOutputWriters, finalOutConfig.getNumOutputs());
+			userCodeClassLoader, this.finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());
 
 		// sanity check the setup
 		final int writersIntoStepFunction = this.eventualOutputs.size();
@@ -129,7 +130,7 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 			throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
 		}
 		// now, we can instantiate the sync gate
-		this.toSync = new BufferWriter(this);
+		this.toSync = getEnvironment().getWriter(syncGateIndex);
 	}
 
 	/**
@@ -233,9 +234,6 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 
 	@Override
 	public void run() throws Exception {
-		// initialize the serializers (one per channel) of the record writers
-		RegularPactTask.initOutputWriters(this.finalOutputWriters);
-
 		final String brokerKey = brokerKey();
 		final int workerIndex = getEnvironment().getIndexInSubtaskGroup();
 		
@@ -445,6 +443,6 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 		if (log.isInfoEnabled()) {
 			log.info(formatLogString("sending " + WorkerDoneEvent.class.getSimpleName() + " to sync"));
 		}
-		this.toSync.broadcastEvent(event);
+		this.toSync.writeEventToAllChannels(event);
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
index 4bdb3cf..e7801e4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
@@ -18,26 +18,26 @@
 
 package org.apache.flink.runtime.iterative.task;
 
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.runtime.io.network.api.BufferWriter;
-import org.apache.flink.runtime.io.network.channels.EndOfSuperstepEvent;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.io.network.api.EndOfSuperstepEvent;
 import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
 import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatch;
 import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatchBroker;
 import org.apache.flink.runtime.iterative.event.TerminationEvent;
 import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
 import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
 
 /**
  * An intermediate iteration task, which runs a Driver}inside.
- * <p/>
+ * <p>
  * It will propagate {@link EndOfSuperstepEvent}s and {@link TerminationEvent}s to it's connected tasks. Furthermore
  * intermediate tasks can also update the iteration state, either the workset or the solution set.
- * <p/>
+ * <p>
  * If the iteration state is updated, the output of this task will be send back to the {@link IterationHeadPactTask} via
  * a {@link BlockingBackChannel} for the workset -XOR- a eHashTable for the solution set. In this case
  * this task must be scheduled on the same instance as the head.
@@ -123,7 +123,7 @@ public class IterationIntermediatePactTask<S extends Function, OT> extends Abstr
 	}
 
 	private void sendEndOfSuperstep() throws IOException, InterruptedException {
-		for (BufferWriter eventualOutput : this.eventualOutputs) {
+		for (RecordWriter eventualOutput : this.eventualOutputs) {
 			eventualOutput.sendEndOfSuperstep();
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
index 91ae1e9..6ce615d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
@@ -24,13 +24,13 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.flink.runtime.event.task.TaskEvent;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.flink.api.common.aggregators.Aggregator;
 import org.apache.flink.api.common.aggregators.AggregatorWithName;
 import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
-import org.apache.flink.runtime.event.task.AbstractTaskEvent;
-import org.apache.flink.runtime.io.network.api.MutableRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
 import org.apache.flink.runtime.iterative.event.AllWorkersDoneEvent;
 import org.apache.flink.runtime.iterative.event.TerminationEvent;
 import org.apache.flink.runtime.iterative.event.WorkerDoneEvent;
@@ -73,7 +73,7 @@ public class IterationSynchronizationSinkTask extends AbstractInvokable implemen
 	
 	@Override
 	public void registerInputOutput() {
-		this.headEventReader = new MutableRecordReader<IntegerRecord>(this);
+		this.headEventReader = new MutableRecordReader<IntegerRecord>(getEnvironment().getReader(0));
 	}
 
 	@Override
@@ -99,7 +99,7 @@ public class IterationSynchronizationSinkTask extends AbstractInvokable implemen
 		int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
 		eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
 				getEnvironment().getUserClassLoader());
-		headEventReader.subscribeToEvent(eventHandler, WorkerDoneEvent.class);
+		headEventReader.subscribeToTaskEvent(eventHandler, WorkerDoneEvent.class);
 
 		IntegerRecord dummy = new IntegerRecord();
 		
@@ -199,8 +199,8 @@ public class IterationSynchronizationSinkTask extends AbstractInvokable implemen
 		}
 	}
 
-	private void sendToAllWorkers(AbstractTaskEvent event) throws IOException, InterruptedException {
-		headEventReader.publishEvent(event);
+	private void sendToAllWorkers(TaskEvent event) throws IOException, InterruptedException {
+		headEventReader.sendTaskEvent(event);
 	}
 
 	private String formatLogString(String message) {

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
index f7db9d1..159d3f2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
@@ -30,12 +30,12 @@ import org.apache.flink.util.Collector;
 
 /**
  * An iteration tail, which runs a driver inside.
- * <p/>
+ * <p>
  * If the iteration state is updated, the output of this task will be send back to the {@link IterationHeadPactTask} via
  * a BackChannel for the workset -OR- a HashTable for the solution set. Therefore this
  * task must be scheduled on the same instance as the head. It's also possible for the tail to update *both* the workset
  * and the solution set.
- * <p/>
+ * <p>
  * If there is a separate solution set tail, the iteration head has to make sure to wait for it to finish.
  */
 public class IterationTailPactTask<S extends Function, OT> extends AbstractIterativePactTask<S, OT> {

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java
index 780b34b..8cce6ef 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java
@@ -21,14 +21,14 @@ package org.apache.flink.runtime.iterative.task;
 import java.util.Map;
 
 import org.apache.flink.api.common.aggregators.Aggregator;
-import org.apache.flink.runtime.event.task.AbstractTaskEvent;
-import org.apache.flink.runtime.event.task.EventListener;
+import org.apache.flink.runtime.event.task.TaskEvent;
+import org.apache.flink.runtime.util.event.EventListener;
 import org.apache.flink.runtime.iterative.event.WorkerDoneEvent;
 import org.apache.flink.types.Value;
 
 import com.google.common.base.Preconditions;
 
-public class SyncEventHandler implements EventListener {
+public class SyncEventHandler implements EventListener<TaskEvent> {
 	
 	private final ClassLoader userCodeClassLoader;
 	
@@ -49,7 +49,7 @@ public class SyncEventHandler implements EventListener {
 	}
 
 	@Override
-	public void eventOccurred(AbstractTaskEvent event) {
+	public void onEvent(TaskEvent event) {
 		if (WorkerDoneEvent.class.equals(event.getClass())) {
 			onWorkerDoneEvent((WorkerDoneEvent) event);
 			return;

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java
index 5a9f308..29a6581 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java
@@ -18,19 +18,25 @@
 
 package org.apache.flink.runtime.jobgraph;
 
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+
 /**
- * A distribution pattern determines which subtasks of a producing Nephele task a wired to which
- * subtasks of a consuming subtask.
+ * A distribution pattern determines, which sub tasks of a producing task are connected to which
+ * consuming sub tasks.
  */
 public enum DistributionPattern {
 
 	/**
-	 * Each subtask of the producing Nephele task is wired to each subtask of the consuming Nephele task.
+	 * Each producing sub task is connected to each sub task of the consuming task.
+	 * <p>
+	 * {@link ExecutionVertex#connectAllToAll(org.apache.flink.runtime.executiongraph.IntermediateResultPartition[], int)}
 	 */
-	BIPARTITE,
+	ALL_TO_ALL,
 
 	/**
-	 * The i-th subtask of the producing Nephele task is wired to the i-th subtask of the consuming Nephele task.
+	 * Each producing sub task is connected to one or more subtask(s) of the consuming task.
+	 * <p>
+	 * {@link ExecutionVertex#connectPointwise(org.apache.flink.runtime.executiongraph.IntermediateResultPartition[], int)}
 	 */
 	POINTWISE
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionID.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionID.java
new file mode 100644
index 0000000..190845d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionID.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.flink.runtime.AbstractID;
+
+public class IntermediateResultPartitionID extends AbstractID {
+
+	private static final long serialVersionUID = 1L;
+
+	/**
+	 * Creates an new random intermediate result partition ID.
+	 */
+	public IntermediateResultPartitionID() {
+		super();
+	}
+
+	public IntermediateResultPartitionID(long lowerPart, long upperPart) {
+		super(lowerPart, upperPart);
+	}
+
+	public static IntermediateResultPartitionID fromByteBuf(ByteBuf buf) {
+		long lower = buf.readLong();
+		long upper = buf.readLong();
+		return new IntermediateResultPartitionID(lower, upper);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionType.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionType.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionType.java
new file mode 100644
index 0000000..25d057e
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionType.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph;
+
+public enum IntermediateResultPartitionType {
+
+	BLOCKING(true, false, false),
+
+	PIPELINED(false, true, true),
+	PIPELINED_PERSISTENT(true, true, true);
+
+	private final boolean isPersistent;
+
+	private final boolean isPipelined;
+
+	private final boolean hasBackPressure;
+
+	IntermediateResultPartitionType(boolean isPersistent, boolean isPipelined, boolean hasBackPressure) {
+		this.isPersistent = isPersistent;
+		this.isPipelined = isPipelined;
+		this.hasBackPressure = hasBackPressure;
+	}
+
+	public boolean hasBackPressure() {
+		return hasBackPressure;
+	}
+
+	public boolean isPipelined() {
+		return isPipelined;
+	}
+
+	public boolean isPersistent() {
+		return isPersistent;
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java
index 842fe62..dc5d9ae 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java
@@ -55,12 +55,6 @@ public final class JobID extends AbstractID {
 		return new JobID(lower, upper);
 	}
 
-	public static JobID fromByteBuffer(ByteBuffer buf, int offset) {
-		long lower = buf.getLong(offset);
-		long upper = buf.getLong(offset + 8);
-		return new JobID(lower, upper);
-	}
-
 	public static JobID fromHexString(String hexString) {
 		return new JobID(DatatypeConverter.parseHexBinary(hexString));
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
index 1666788..ff1b272 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
@@ -80,7 +80,7 @@ public abstract class AbstractInvokable {
 	 * @return the current number of subtasks the respective task is split into
 	 */
 	public final int getCurrentNumberOfSubtasks() {
-		return this.environment.getCurrentNumberOfSubtasks();
+		return this.environment.getNumberOfSubtasks();
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java
index 8d48975..103c1be 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java
@@ -18,18 +18,17 @@
 
 package org.apache.flink.runtime.jobmanager.web;
 
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
 import org.apache.flink.runtime.executiongraph.IntermediateResult;
 import org.apache.flink.runtime.instance.AllocatedSlot;
-import org.apache.flink.runtime.io.network.channels.ChannelType;
 import org.apache.flink.util.StringUtils;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 public class JsonFactory {
 
 	public static String toJson(ExecutionVertex vertex) {
@@ -92,8 +91,7 @@ public class JsonFactory {
 			
 			json.append("{");
 			json.append("\"groupvertexid\": \"" + input.getJobVertexId() + "\",");
-			json.append("\"groupvertexname\": \"" +  StringUtils.escapeHtml(jobVertex.getJobVertex().getName()) + "\",");
-			json.append("\"channelType\": \"" +  ChannelType.NETWORK + "\"");
+			json.append("\"groupvertexname\": \"" +  StringUtils.escapeHtml(jobVertex.getJobVertex().getName()) + "\"");
 			json.append("}");
 			
 			// print delimiter

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
index e3ffdba..be4453d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
@@ -16,14 +16,9 @@
  * limitations under the License.
  */
 
-
 package org.apache.flink.runtime.operators;
 
 import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.runtime.plugable.DeserializationDelegate;
-import org.apache.flink.util.InstantiationUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.flink.api.common.io.CleanupWhenUnsuccessful;
 import org.apache.flink.api.common.io.OutputFormat;
 import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
@@ -32,16 +27,20 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.runtime.execution.CancelTaskException;
-import org.apache.flink.runtime.io.network.api.MutableReader;
-import org.apache.flink.runtime.io.network.api.MutableRecordReader;
-import org.apache.flink.runtime.io.network.api.MutableUnionRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.UnionBufferReader;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
 import org.apache.flink.runtime.operators.sort.UnilateralSortMerger;
 import org.apache.flink.runtime.operators.util.CloseableInputProvider;
 import org.apache.flink.runtime.operators.util.ReaderIterator;
 import org.apache.flink.runtime.operators.util.TaskConfig;
+import org.apache.flink.runtime.plugable.DeserializationDelegate;
+import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.MutableObjectIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
@@ -59,13 +58,12 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 	
 	// OutputFormat instance. volatile, because the asynchronous canceller may access it
 	private volatile OutputFormat<IT> format;
-	
+
+	private MutableReader<?> inputReader;
+
 	// input reader
 	private MutableObjectIterator<IT> reader;
-	
-	// input iterator
-	private MutableObjectIterator<IT> input;
-	
+
 	// The serializer for the input type
 	private TypeSerializerFactory<IT> inputTypeSerializerFactory;
 	
@@ -129,11 +127,12 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 		try {
 			
 			// initialize local strategies
+			MutableObjectIterator<IT> input1;
 			switch (this.config.getInputLocalStrategy(0)) {
 			case NONE:
 				// nothing to do
 				localStrategy = null;
-				input = reader;
+				input1 = reader;
 				break;
 			case SORT:
 				// initialize sort local strategy
@@ -154,7 +153,7 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 							this.config.getSpillingThresholdInput(0));
 					
 					this.localStrategy = sorter;
-					this.input = sorter.getIterator();
+					input1 = sorter.getIterator();
 				} catch (Exception e) {
 					throw new RuntimeException("Initializing the input processing failed" +
 						e.getMessage() == null ? "." : ": " + e.getMessage(), e);
@@ -167,7 +166,7 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 			// read the reader and write it to the output
 			
 			final TypeSerializer<IT> serializer = this.inputTypeSerializerFactory.getSerializer();
-			final MutableObjectIterator<IT> input = this.input;
+			final MutableObjectIterator<IT> input = input1;
 			final OutputFormat<IT> format = this.format;
 
 
@@ -181,7 +180,7 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 			}
 
 			// open
-			format.open(this.getEnvironment().getIndexInSubtaskGroup(), this.getEnvironment().getCurrentNumberOfSubtasks());
+			format.open(this.getEnvironment().getIndexInSubtaskGroup(), this.getEnvironment().getNumberOfSubtasks());
 
 			if (objectReuseEnabled) {
 				IT record = serializer.createInstance();
@@ -253,6 +252,8 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 					LOG.error("Error closing local strategy", t);
 				}
 			}
+
+			RegularPactTask.clearReaders(new MutableReader[]{inputReader});
 		}
 
 		if (!this.taskCanceled) {
@@ -337,9 +338,6 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 	 */
 	@SuppressWarnings("unchecked")
 	private void initInputReaders() throws Exception {
-		
-		MutableReader<?> inputReader;
-		
 		int numGates = 0;
 		//  ---------------- create the input readers ---------------------
 		// in case where a logical input unions multiple physical inputs, create a union reader
@@ -347,25 +345,20 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 		numGates += groupSize;
 		if (groupSize == 1) {
 			// non-union case
-			inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(this);
+			inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(getEnvironment().getReader(0));
 		} else if (groupSize > 1){
 			// union case
-			
-			MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
-			for (int j = 0; j < groupSize; ++j) {
-				readers[j] = new MutableRecordReader<IOReadableWritable>(this);
-			}
-			inputReader = new MutableUnionRecordReader<IOReadableWritable>(readers);
+			UnionBufferReader reader = new UnionBufferReader(getEnvironment().getAllReaders());
+			inputReader = new MutableRecordReader<IOReadableWritable>(reader);
 		} else {
 			throw new Exception("Illegal input group size in task configuration: " + groupSize);
 		}
-		
+
 		this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
-		
 		@SuppressWarnings({ "rawtypes" })
 		final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
 		this.reader = (MutableObjectIterator<IT>)iter;
-		
+
 		// final sanity check
 		if (numGates != this.config.getNumInputs()) {
 			throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
index b50bf36..330f438 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
@@ -18,17 +18,7 @@
 
 package org.apache.flink.runtime.operators;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
 import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.util.InstantiationUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.io.InputFormat;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -36,13 +26,23 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.io.InputSplit;
 import org.apache.flink.runtime.execution.CancelTaskException;
-import org.apache.flink.runtime.io.network.api.BufferWriter;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
 import org.apache.flink.runtime.operators.chaining.ChainedDriver;
 import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
 import org.apache.flink.runtime.operators.util.TaskConfig;
 import org.apache.flink.util.Collector;
+import org.apache.flink.util.InstantiationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
 
 /**
  * DataSourceTask which is executed by a task manager. The task reads data and uses an 
@@ -51,11 +51,10 @@ import org.apache.flink.util.Collector;
  * @see org.apache.flink.api.common.io.InputFormat
  */
 public class DataSourceTask<OT> extends AbstractInvokable {
-	
+
 	private static final Logger LOG = LoggerFactory.getLogger(DataSourceTask.class);
 
-	
-	private List<BufferWriter> eventualOutputs;
+	private List<RecordWriter<?>> eventualOutputs;
 
 	// Output collector
 	private Collector<OT> output;
@@ -145,9 +144,6 @@ public class DataSourceTask<OT> extends AbstractInvokable {
 		final TypeSerializer<OT> serializer = this.serializerFactory.getSerializer();
 		
 		try {
-			// initialize the serializers (one per channel) of the record writers
-			RegularPactTask.initOutputWriters(this.eventualOutputs);
-
 			// start all chained tasks
 			RegularPactTask.openChainedTasks(this.chainedTasks, this);
 			
@@ -159,7 +155,7 @@ public class DataSourceTask<OT> extends AbstractInvokable {
 			{
 				// get start and end
 				final InputSplit split = splitIterator.next();
-				
+
 				if (LOG.isDebugEnabled()) {
 					LOG.debug(getLogString("Opening input split " + split.toString()));
 				}
@@ -174,7 +170,6 @@ public class DataSourceTask<OT> extends AbstractInvokable {
 				}
 				
 				try {
-
 					final Collector<OT> output = this.output;
 
 					if (objectReuseEnabled) {
@@ -222,7 +217,7 @@ public class DataSourceTask<OT> extends AbstractInvokable {
 			// close the input, but do not report any exceptions, since we already have another root cause
 			try {
 				this.format.close();
-			} catch (Throwable t) {}
+			} catch (Throwable ignored) {}
 			
 			RegularPactTask.cancelChainedTasks(this.chainedTasks);
 			
@@ -236,6 +231,8 @@ public class DataSourceTask<OT> extends AbstractInvokable {
 				// drop exception, if the task was canceled
 				RegularPactTask.logAndThrowException(ex, this);
 			}
+		} finally {
+			RegularPactTask.clearWriters(eventualOutputs);
 		}
 
 		if (!this.taskCanceled) {
@@ -304,7 +301,7 @@ public class DataSourceTask<OT> extends AbstractInvokable {
 	 */
 	private void initOutputs(ClassLoader cl) throws Exception {
 		this.chainedTasks = new ArrayList<ChainedDriver<?, ?>>();
-		this.eventualOutputs = new ArrayList<BufferWriter>();
+		this.eventualOutputs = new ArrayList<RecordWriter<?>>();
 		this.output = RegularPactTask.initOutputs(this, cl, this.config, this.chainedTasks, this.eventualOutputs, getExecutionConfig());
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
index 34aaced..2d36c1e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
@@ -19,10 +19,7 @@
 package org.apache.flink.runtime.operators;
 
 import akka.actor.ActorRef;
-import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.api.common.ExecutionConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.accumulators.AccumulatorHelper;
 import org.apache.flink.api.common.distributions.DataDistribution;
@@ -40,14 +37,15 @@ import org.apache.flink.runtime.broadcast.BroadcastVariableMaterialization;
 import org.apache.flink.runtime.execution.CancelTaskException;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.network.api.BufferWriter;
-import org.apache.flink.runtime.io.network.api.ChannelSelector;
-import org.apache.flink.runtime.io.network.api.MutableReader;
-import org.apache.flink.runtime.io.network.api.MutableRecordReader;
-import org.apache.flink.runtime.io.network.api.MutableUnionRecordReader;
-import org.apache.flink.runtime.io.network.api.RecordWriter;
+import org.apache.flink.runtime.io.network.api.reader.BufferReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.UnionBufferReader;
+import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.memorymanager.MemoryManager;
+import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.operators.chaining.ChainedDriver;
 import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
 import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator;
@@ -69,6 +67,8 @@ import org.apache.flink.types.Record;
 import org.apache.flink.util.Collector;
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.MutableObjectIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -111,7 +111,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	 * The output writers for the data that this task forwards to the next task. The latest driver (the central, if no chained
 	 * drivers exist, otherwise the last chained driver) produces its output to these writers.
 	 */
-	protected List<BufferWriter> eventualOutputs;
+	protected List<RecordWriter<?>> eventualOutputs;
 
 	/**
 	 * The input readers to this task.
@@ -278,9 +278,6 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 		// so that the thread that creates them catches its own errors that may happen in that process.
 		// this is especially important, since there may be asynchronous closes (such as through canceling).
 		try {
-			// initialize the serializers (one per channel) of the record writers
-			initOutputWriters(this.eventualOutputs);
-
 			// initialize the remaining data structures on the input and trigger the local processing
 			// the local processing includes building the dams / caches
 			try {
@@ -302,7 +299,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 							throw new IllegalArgumentException();
 						}
 						else if (numberOfEventsUntilInterrupt > 0) {
-							this.inputReaders[i].setIterative(numberOfEventsUntilInterrupt);
+							this.inputReaders[i].setIterativeReader();
 							iterativeInputs.add(i);
 				
 							if (LOG.isDebugEnabled()) {
@@ -324,7 +321,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 							throw new IllegalArgumentException();
 						}
 						else if (numberOfEventsUntilInterrupt > 0) {
-							this.broadcastInputReaders[i].setIterative(numberOfEventsUntilInterrupt);
+							this.broadcastInputReaders[i].setIterativeReader();
 							iterativeBcInputs.add(i);
 				
 							if (LOG.isDebugEnabled()) {
@@ -365,6 +362,9 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 		finally {
 			// clean up in any case!
 			closeLocalStrategiesAndCaches();
+
+			clearReaders(inputReaders);
+			clearWriters(eventualOutputs);
 		}
 
 		if (this.running) {
@@ -564,14 +564,12 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	 *          Each chained task might have accumulators which will be merged
 	 *          with the accumulators of the stub.
 	 */
-	protected static void reportAndClearAccumulators(Environment env, Map<String, Accumulator<?, ?>> accumulators,
-			ArrayList<ChainedDriver<?, ?>> chainedTasks) {
+	protected static void reportAndClearAccumulators(
+			Environment env, Map<String, Accumulator<?, ?>> accumulators, ArrayList<ChainedDriver<?, ?>> chainedTasks) {
 
 		// We can merge here the accumulators from the stub and the chained
 		// tasks. Type conflicts can occur here if counters with same name but
 		// different type were used.
-
-
 		for (ChainedDriver<?, ?> chainedTask : chainedTasks) {
 			if (FunctionUtils.getFunctionRuntimeContext(chainedTask.getStub(), null) != null) {
 				Map<String, Accumulator<?, ?>> chainedAccumulators = FunctionUtils.getFunctionRuntimeContext(chainedTask.getStub(), null).getAllAccumulators();
@@ -587,7 +585,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 		// Report accumulators to JobManager
 		JobManagerMessages.ReportAccumulatorResult accResult = new JobManagerMessages.ReportAccumulatorResult(new
 				AccumulatorEvent(env.getJobID(), AccumulatorHelper.copy(accumulators)));
-		env.getAccumulator().tell(accResult, ActorRef.noSender());
+		env.getJobManager().tell(accResult, ActorRef.noSender());
 
 		// We also clear the accumulators, since stub instances might be reused
 		// (e.g. in iterations) and we don't want to count twice. This may not be
@@ -664,7 +662,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 
 	/**
 	 * Sets the last output {@link Collector} of the collector chain of this {@link RegularPactTask}.
-	 * <p/>
+	 * <p>
 	 * In case of chained tasks, the output collector of the last {@link ChainedDriver} is set. Otherwise it is the
 	 * single collector of the {@link RegularPactTask}.
 	 *
@@ -713,31 +711,34 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 		final int numInputs = getNumTaskInputs();
 		final MutableReader<?>[] inputReaders = new MutableReader[numInputs];
 
-		int numGates = 0;
+		int currentReaderOffset = 0;
 
 		for (int i = 0; i < numInputs; i++) {
 			//  ---------------- create the input readers ---------------------
 			// in case where a logical input unions multiple physical inputs, create a union reader
 			final int groupSize = this.config.getGroupSize(i);
-			numGates += groupSize;
+
 			if (groupSize == 1) {
 				// non-union case
-				inputReaders[i] = new MutableRecordReader<IOReadableWritable>(this);
+				inputReaders[i] = new MutableRecordReader<IOReadableWritable>(getEnvironment().getReader(currentReaderOffset));
 			} else if (groupSize > 1){
 				// union case
-				MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
+				BufferReader[] readers = new BufferReader[groupSize];
 				for (int j = 0; j < groupSize; ++j) {
-					readers[j] = new MutableRecordReader<IOReadableWritable>(this);
+					readers[j] = getEnvironment().getReader(currentReaderOffset + j);
 				}
-				inputReaders[i] = new MutableUnionRecordReader<IOReadableWritable>(readers);
+				UnionBufferReader reader = new UnionBufferReader(readers);
+				inputReaders[i] = new MutableRecordReader<IOReadableWritable>(reader);
 			} else {
 				throw new Exception("Illegal input group size in task configuration: " + groupSize);
 			}
+
+			currentReaderOffset += groupSize;
 		}
 		this.inputReaders = inputReaders;
 
 		// final sanity check
-		if (numGates != this.config.getNumInputs()) {
+		if (currentReaderOffset != this.config.getNumInputs()) {
 			throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
 		}
 	}
@@ -751,24 +752,29 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	protected void initBroadcastInputReaders() throws Exception {
 		final int numBroadcastInputs = this.config.getNumBroadcastInputs();
 		final MutableReader<?>[] broadcastInputReaders = new MutableReader[numBroadcastInputs];
-		
+
+		int currentReaderOffset = config.getNumInputs();
+
 		for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
 			//  ---------------- create the input readers ---------------------
 			// in case where a logical input unions multiple physical inputs, create a union reader
 			final int groupSize = this.config.getBroadcastGroupSize(i);
 			if (groupSize == 1) {
 				// non-union case
-				broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(this);
+				broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(getEnvironment().getReader(currentReaderOffset));
 			} else if (groupSize > 1){
 				// union case
-				MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
+				BufferReader[] readers = new BufferReader[groupSize];
 				for (int j = 0; j < groupSize; ++j) {
-					readers[j] = new MutableRecordReader<IOReadableWritable>(this);
+					readers[j] = getEnvironment().getReader(currentReaderOffset + j);
 				}
-				broadcastInputReaders[i] = new MutableUnionRecordReader<IOReadableWritable>(readers);
+				UnionBufferReader reader = new UnionBufferReader(readers);
+				broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(reader);
 			} else {
 				throw new Exception("Illegal input group size in task configuration: " + groupSize);
 			}
+
+			currentReaderOffset += groupSize;
 		}
 		this.broadcastInputReaders = broadcastInputReaders;
 	}
@@ -853,8 +859,8 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 
 		for (int i = 0; i < numInputs; i++) {
 			final int memoryPages;
-			final boolean cached =  this.config.isInputCached(i);
 			final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
+			final boolean cached =  this.config.isInputCached(i);
 
 			this.inputIsAsyncMaterialized[i] = async;
 			this.inputIsCached[i] = cached;
@@ -886,16 +892,16 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	}
 
 	protected void resetAllInputs() throws Exception {
-		
+
 		// first we need to make sure that caches consume remaining data
 		// NOTE: we need to do this before closing the local strategies
 		for (int i = 0; i < this.inputs.length; i++) {
-			
+
 			if (this.inputIsCached[i] && this.resettableInputs[i] != null) {
 				this.resettableInputs[i].consumeAndCacheRemainingData();
 			}
 		}
-		
+
 		// close all local-strategies. they will either get re-initialized, or we have
 		// read them now and their data is cached
 		for (int i = 0; i < this.localStrategies.length; i++) {
@@ -1054,7 +1060,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	 */
 	protected void initOutputs() throws Exception {
 		this.chainedTasks = new ArrayList<ChainedDriver<?, ?>>();
-		this.eventualOutputs = new ArrayList<BufferWriter>();
+		this.eventualOutputs = new ArrayList<RecordWriter<?>>();
 
 		ClassLoader userCodeClassLoader = getUserCodeClassLoader();
 
@@ -1063,7 +1069,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 
 	public DistributedRuntimeUDFContext createRuntimeContext(String taskName) {
 		Environment env = getEnvironment();
-		return new DistributedRuntimeUDFContext(taskName, env.getCurrentNumberOfSubtasks(),
+		return new DistributedRuntimeUDFContext(taskName, env.getNumberOfSubtasks(),
 				env.getIndexInSubtaskGroup(), getUserCodeClassLoader(), env.getCopyTask());
 	}
 
@@ -1208,7 +1214,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	 */
 	public static String constructLogString(String message, String taskName, AbstractInvokable parent) {
 		return message + ":  " + taskName + " (" + (parent.getEnvironment().getIndexInSubtaskGroup() + 1) +
-				'/' + parent.getEnvironment().getCurrentNumberOfSubtasks() + ')';
+				'/' + parent.getEnvironment().getNumberOfSubtasks() + ')';
 	}
 
 	/**
@@ -1255,7 +1261,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	 *
 	 * @return The OutputCollector that data produced in this task is submitted to.
 	 */
-	public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl, List<BufferWriter> eventualOutputs, int numOutputs)
+	public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl, List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs)
 			throws Exception
 	{
 		if (numOutputs == 0) {
@@ -1289,7 +1295,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 					oe = new RecordOutputEmitter(strategy, comparator, partitioner, distribution);
 				}
 
-				writers.add(new RecordWriter<Record>(task, oe));
+				writers.add(new RecordWriter<Record>(task.getEnvironment().getWriter(outputOffset + i), oe));
 			}
 			if (eventualOutputs != null) {
 				eventualOutputs.addAll(writers);
@@ -1321,7 +1327,8 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 					final TypeComparator<T> comparator = compFactory.createComparator();
 					oe = new OutputEmitter<T>(strategy, comparator, partitioner, dataDist);
 				}
-				writers.add(new RecordWriter<SerializationDelegate<T>>(task, oe));
+
+				writers.add(new RecordWriter<SerializationDelegate<T>>(task.getEnvironment().getWriter(outputOffset + i), oe));
 			}
 			if (eventualOutputs != null) {
 				eventualOutputs.addAll(writers);
@@ -1336,7 +1343,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 	 */
 	@SuppressWarnings("unchecked")
 	public static <T> Collector<T> initOutputs(AbstractInvokable nepheleTask, ClassLoader cl, TaskConfig config,
-					List<ChainedDriver<?, ?>> chainedTasksTarget, List<BufferWriter> eventualOutputs, ExecutionConfig executionConfig)
+					List<ChainedDriver<?, ?>> chainedTasksTarget, List<RecordWriter<?>> eventualOutputs, ExecutionConfig executionConfig)
 	throws Exception
 	{
 		final int numOutputs = config.getNumOutputs();
@@ -1370,7 +1377,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 
 				if (i == numChained -1) {
 					// last in chain, instantiate the output collector for this task
-					previous = getOutputCollector(nepheleTask, chainedStubConf, cl, eventualOutputs, chainedStubConf.getNumOutputs());
+					previous = getOutputCollector(nepheleTask, chainedStubConf, cl, eventualOutputs, 0, chainedStubConf.getNumOutputs());
 				}
 
 				ct.setup(chainedStubConf, taskName, previous, nepheleTask, cl, executionConfig);
@@ -1384,13 +1391,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 		// else
 
 		// instantiate the output collector the default way from this configuration
-		return getOutputCollector(nepheleTask , config, cl, eventualOutputs, numOutputs);
-	}
-
-	public static void initOutputWriters(List<BufferWriter> writers) {
-		for (BufferWriter writer : writers) {
-			((RecordWriter<?>) writer).initializeSerializers();
-		}
+		return getOutputCollector(nepheleTask , config, cl, eventualOutputs, 0, numOutputs);
 	}
 	
 	// --------------------------------------------------------------------------------------------
@@ -1528,4 +1529,16 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
 		}
 		return a;
 	}
+
+	public static void clearWriters(List<RecordWriter<?>> writers) {
+		for (RecordWriter<?> writer : writers) {
+			writer.clearBuffers();
+		}
+	}
+
+	public static void clearReaders(MutableReader<?>[] readers) {
+		for (MutableReader<?> reader : readers) {
+			reader.clearBuffers();
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
index c36e742..838286e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
@@ -61,7 +61,7 @@ public abstract class ChainedDriver<IT, OT> implements Collector<IT> {
 			this.udfContext = ((RegularPactTask<?, ?>) parent).createRuntimeContext(taskName);
 		} else {
 			Environment env = parent.getEnvironment();
-			this.udfContext = new DistributedRuntimeUDFContext(taskName, env.getCurrentNumberOfSubtasks(), 
+			this.udfContext = new DistributedRuntimeUDFContext(taskName, env.getNumberOfSubtasks(),
 					env.getIndexInSubtaskGroup(), userCodeClassLoader, env.getCopyTask());
 		}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java
index 5b0cdf1..56723df 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java
@@ -25,7 +25,7 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.io.network.api.RecordWriter;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 import org.apache.flink.util.Collector;
 
@@ -103,8 +103,6 @@ public class OutputCollector<T> implements Collector<T>
 				writer.flush();
 			} catch (IOException e) {
 				throw new RuntimeException(e.getMessage(), e);
-			} catch (InterruptedException e) {
-				throw new RuntimeException(e.getMessage(), e);
 			}
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
index ec92e3f..ff1a148 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.shipping;
 import org.apache.flink.api.common.distributions.DataDistribution;
 import org.apache.flink.api.common.functions.Partitioner;
 import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.runtime.io.network.api.ChannelSelector;
+import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
 import org.apache.flink.runtime.plugable.SerializationDelegate;
 
 public class OutputEmitter<T> implements ChannelSelector<SerializationDelegate<T>> {

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java
index 845591b..dfd58dc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java
@@ -24,7 +24,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.flink.runtime.io.network.api.RecordWriter;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.types.Record;
 import org.apache.flink.util.Collector;
 
@@ -97,8 +97,6 @@ public class RecordOutputCollector implements Collector<Record>
 				writer.flush();
 			} catch (IOException e) {
 				throw new RuntimeException(e.getMessage(), e);
-			} catch (InterruptedException e) {
-				throw new RuntimeException(e.getMessage(), e);
 			}
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java
index 455ebbf..300d73f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.shipping;
 import org.apache.flink.api.common.distributions.DataDistribution;
 import org.apache.flink.api.common.functions.Partitioner;
 import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.runtime.io.network.api.ChannelSelector;
+import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
 import org.apache.flink.types.Key;
 import org.apache.flink.types.Record;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java
index 85e36a4..7c00c53 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators.util;
 import java.io.IOException;
 
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.io.network.api.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
 import org.apache.flink.runtime.plugable.DeserializationDelegate;
 import org.apache.flink.runtime.plugable.NonReusingDeserializationDelegate;
 import org.apache.flink.runtime.plugable.ReusingDeserializationDelegate;

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java
index 226abf0..e4a131d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java
@@ -16,17 +16,14 @@
  * limitations under the License.
  */
 
-
 package org.apache.flink.runtime.operators.util;
 
 import java.io.IOException;
 
 import org.apache.flink.util.MutableObjectIterator;
 
-
 /**
  * A simple iterator provider that returns a supplied iterator and does nothing when closed.
- *
  */
 public class SimpleCloseableInputProvider<E> implements CloseableInputProvider<E>
 {
@@ -34,8 +31,7 @@ public class SimpleCloseableInputProvider<E> implements CloseableInputProvider<E
 	 * The iterator returned by this class.
 	 */
 	private final MutableObjectIterator<E> iterator;
-	
-	
+
 	/**
 	 * Creates a new simple input provider that will return the given iterator.
 	 * 
@@ -44,15 +40,12 @@ public class SimpleCloseableInputProvider<E> implements CloseableInputProvider<E
 	public SimpleCloseableInputProvider(MutableObjectIterator<E> iterator) {
 		this.iterator = iterator;
 	}
-	
-	
 
 	@Override
 	public void close() throws IOException {
 		// do nothing
 	}
 
-
 	@Override
 	public MutableObjectIterator<E> getIterator() {
 		return this.iterator;

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 8570992..5b6259a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -18,46 +18,48 @@
 
 package org.apache.flink.runtime.taskmanager;
 
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-
 import akka.actor.ActorRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.execution.RuntimeEnvironment;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.io.network.api.reader.BufferReader;
+import org.apache.flink.runtime.io.network.api.writer.BufferWriter;
+import org.apache.flink.runtime.io.network.partition.IntermediateResultPartition;
 import org.apache.flink.runtime.jobgraph.JobID;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.memorymanager.MemoryManager;
 import org.apache.flink.runtime.messages.ExecutionGraphMessages;
 import org.apache.flink.runtime.profiling.TaskManagerProfiler;
 import org.apache.flink.util.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 
 public final class Task {
 
 	/** For atomic state updates */
-	private static final AtomicReferenceFieldUpdater<Task, ExecutionState> STATE_UPDATER = 
+	private static final AtomicReferenceFieldUpdater<Task, ExecutionState> STATE_UPDATER =
 			AtomicReferenceFieldUpdater.newUpdater(Task.class, ExecutionState.class, "executionState");
-			
+
 	/** The log object used for debugging. */
 	private static final Logger LOG = LoggerFactory.getLogger(Task.class);
 
 	// --------------------------------------------------------------------------------------------
-	
+
 	private final JobID jobId;
-	
+
 	private final JobVertexID vertexId;
-	
+
 	private final int subtaskIndex;
-	
+
 	private final int numberOfSubtasks;
-	
+
 	private final ExecutionAttemptID executionId;
-	
+
 	private final String taskName;
 
 	private final TaskManager taskManager;
@@ -66,15 +68,15 @@ public final class Task {
 
 	/** The environment (with the invokable) executed by this task */
 	private volatile RuntimeEnvironment environment;
-	
+
 	/** The current execution state of the task */
 	private volatile ExecutionState executionState = ExecutionState.DEPLOYING;
 
 	// --------------------------------------------------------------------------------------------	
-	
-	public Task(JobID jobId, JobVertexID vertexId, int taskIndex, int parallelism, 
-			ExecutionAttemptID executionId, String taskName, TaskManager taskManager)
-	{
+
+	public Task(JobID jobId, JobVertexID vertexId, int taskIndex, int parallelism,
+				ExecutionAttemptID executionId, String taskName, TaskManager taskManager) {
+
 		this.jobId = jobId;
 		this.vertexId = vertexId;
 		this.subtaskIndex = taskIndex;
@@ -84,11 +86,8 @@ public final class Task {
 		this.taskManager = taskManager;
 	}
 
-
 	/**
 	 * Returns the ID of the job this task belongs to.
-	 * 
-	 * @return the ID of the job this task belongs to
 	 */
 	public JobID getJobID() {
 		return this.jobId;
@@ -96,8 +95,6 @@ public final class Task {
 
 	/**
 	 * Returns the ID of this task vertex.
-	 * 
-	 * @return the ID of this task vertex.
 	 */
 	public JobVertexID getVertexID() {
 		return this.vertexId;
@@ -105,71 +102,63 @@ public final class Task {
 
 	/**
 	 * Gets the index of the parallel subtask [0, parallelism).
-	 * 
-	 * @return The task index of the parallel subtask.
 	 */
 	public int getSubtaskIndex() {
 		return subtaskIndex;
 	}
-	
+
 	/**
 	 * Gets the total number of subtasks of the task that this subtask belongs to.
-	 * 
-	 * @return The total number of this task's subtasks.
 	 */
 	public int getNumberOfSubtasks() {
 		return numberOfSubtasks;
 	}
-	
+
 	/**
 	 * Gets the ID of the execution attempt.
-	 * 
-	 * @return The ID of the execution attempt.
 	 */
 	public ExecutionAttemptID getExecutionId() {
 		return executionId;
 	}
-	
+
 	/**
 	 * Returns the current execution state of the task.
-	 * 
-	 * @return the current execution state of the task
 	 */
 	public ExecutionState getExecutionState() {
 		return this.executionState;
 	}
-	
+
 	public void setEnvironment(RuntimeEnvironment environment) {
 		this.environment = environment;
 	}
-	
+
 	public RuntimeEnvironment getEnvironment() {
 		return environment;
 	}
-	
+
 	public boolean isCanceledOrFailed() {
 		return executionState == ExecutionState.CANCELING ||
 				executionState == ExecutionState.CANCELED ||
 				executionState == ExecutionState.FAILED;
 	}
-	
+
 	public String getTaskName() {
 		return taskName;
 	}
-	
+
 	public String getTaskNameWithSubtasks() {
 		return this.taskName + " (" + (this.subtaskIndex + 1) + "/" + this.numberOfSubtasks + ")";
 	}
-	
+
 	// ----------------------------------------------------------------------------------------------------------------
 	//  States and Transitions
 	// ----------------------------------------------------------------------------------------------------------------
-	
+
 	/**
 	 * Marks the task as finished. This succeeds, if the task was previously in the state
 	 * "RUNNING", otherwise it fails. Failure indicates that the task was either
 	 * canceled, or set to failed.
-	 * 
+	 *
 	 * @return True, if the task correctly enters the state FINISHED.
 	 */
 	public boolean markAsFinished() {
@@ -177,20 +166,21 @@ public final class Task {
 			notifyObservers(ExecutionState.FINISHED, null);
 			taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.FINISHED, null);
 			return true;
-		} else {
+		}
+		else {
 			return false;
 		}
 	}
-	
+
 	public void markFailed(Throwable error) {
 		while (true) {
 			ExecutionState current = this.executionState;
-			
+
 			// if canceled, fine. we are done, and the jobmanager has been told
 			if (current == ExecutionState.CANCELED) {
 				return;
 			}
-			
+
 			// if canceling, we are done, but we cannot be sure that the jobmanager has been told.
 			// after all, we may have recognized our failure state before the cancelling and never sent a canceled
 			// message back
@@ -201,23 +191,22 @@ public final class Task {
 			}
 		}
 	}
-	
+
 	public void cancelExecution() {
 		while (true) {
 			ExecutionState current = this.executionState;
-			
+
 			// if the task is already canceled (or canceling) or finished or failed,
 			// then we need not do anything
 			if (current == ExecutionState.FINISHED || current == ExecutionState.CANCELED ||
-					current == ExecutionState.CANCELING || current == ExecutionState.FAILED)
-			{
+					current == ExecutionState.CANCELING || current == ExecutionState.FAILED) {
 				return;
 			}
-			
+
 			if (current == ExecutionState.DEPLOYING) {
 				// directly set to canceled
 				if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELED)) {
-					
+
 					notifyObservers(ExecutionState.CANCELED, null);
 					taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.CANCELED, null);
 					return;
@@ -226,14 +215,15 @@ public final class Task {
 			else if (current == ExecutionState.RUNNING) {
 				// go to canceling and perform the actual task canceling
 				if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELING)) {
-					
+
 					notifyObservers(ExecutionState.CANCELING, null);
 					try {
 						this.environment.cancelExecution();
-					} catch (Throwable e) {
+					}
+					catch (Throwable e) {
 						LOG.error("Error while cancelling the task.", e);
 					}
-					
+
 					return;
 				}
 			}
@@ -242,30 +232,33 @@ public final class Task {
 			}
 		}
 	}
-	
+
 	/**
 	 * Sets the tasks to be cancelled and reports a failure back to the master.
-	 * This method is important if a failure needs to be reported to the master, because
-	 * a simple canceled m
-	 * 
-	 * @param cause The exception to report in the error message
 	 */
 	public void failExternally(Throwable cause) {
 		while (true) {
 			ExecutionState current = this.executionState;
-			
+
 			// if the task is already canceled (or canceling) or finished or failed,
 			// then we need not do anything
-			if (current == ExecutionState.FINISHED || current == ExecutionState.CANCELED ||
-					current == ExecutionState.CANCELING || current == ExecutionState.FAILED)
-			{
+			if (current == ExecutionState.CANCELED || current == ExecutionState.CANCELING || current == ExecutionState.FAILED) {
 				return;
 			}
-			
+
+			if (current == ExecutionState.FINISHED) {
+				// Set state to failed in order to correctly unregister task from network environment
+				if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
+					notifyObservers(ExecutionState.FAILED, null);
+
+					return;
+				}
+			}
+
 			if (current == ExecutionState.DEPLOYING) {
 				// directly set to canceled
 				if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
-					
+
 					notifyObservers(ExecutionState.FAILED, null);
 					taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.FAILED, cause);
 					return;
@@ -276,13 +269,14 @@ public final class Task {
 				if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
 					try {
 						this.environment.cancelExecution();
-					} catch (Throwable e) {
+					}
+					catch (Throwable e) {
 						LOG.error("Error while cancelling the task.", e);
 					}
-					
+
 					notifyObservers(ExecutionState.FAILED, null);
 					taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.FAILED, cause);
-					
+
 					return;
 				}
 			}
@@ -291,18 +285,18 @@ public final class Task {
 			}
 		}
 	}
-	
+
 	public void cancelingDone() {
 		while (true) {
 			ExecutionState current = this.executionState;
-			
+
 			if (current == ExecutionState.CANCELED || current == ExecutionState.FAILED) {
 				return;
 			}
 			if (!(current == ExecutionState.RUNNING || current == ExecutionState.CANCELING)) {
 				LOG.error(String.format("Unexpected state transition in Task: %s -> %s", current, ExecutionState.CANCELED));
 			}
-			
+
 			if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELED)) {
 				notifyObservers(ExecutionState.CANCELED, null);
 				taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.CANCELED, null);
@@ -319,16 +313,14 @@ public final class Task {
 			final Thread thread = this.environment.getExecutingThread();
 			thread.start();
 			return true;
-		} else {
+		}
+		else {
 			return false;
 		}
 	}
-	
+
 	/**
 	 * Unregisters the task from the central memory manager.
-	 * 
-	 * @param memoryManager
-	 *        the central memory manager
 	 */
 	public void unregisterMemoryManager(MemoryManager memoryManager) {
 		RuntimeEnvironment env = this.environment;
@@ -336,18 +328,13 @@ public final class Task {
 			memoryManager.releaseAll(env.getInvokable());
 		}
 	}
-	
+
 	// -----------------------------------------------------------------------------------------------------------------
 	//                                        Task Profiling
 	// -----------------------------------------------------------------------------------------------------------------
 
 	/**
 	 * Registers the task manager profiler with the task.
-	 * 
-	 * @param taskManagerProfiler
-	 *        the task manager profiler
-	 * @param jobConfiguration
-	 *        the configuration attached to the job
 	 */
 	public void registerProfiler(TaskManagerProfiler taskManagerProfiler, Configuration jobConfiguration) {
 		taskManagerProfiler.registerTask(this, jobConfiguration);
@@ -355,45 +342,58 @@ public final class Task {
 
 	/**
 	 * Unregisters the task from the task manager profiler.
-	 * 
-	 * @param taskManagerProfiler
-	 *        the task manager profiler
 	 */
 	public void unregisterProfiler(TaskManagerProfiler taskManagerProfiler) {
 		if (taskManagerProfiler != null) {
 			taskManagerProfiler.unregisterTask(this.executionId);
 		}
 	}
-	
+
+	// ------------------------------------------------------------------------
+	// Intermediate result partitions
+	// ------------------------------------------------------------------------
+
+	public BufferReader[] getReaders() {
+		return environment != null ? environment.getAllReaders() : null;
+	}
+
+	public BufferWriter[] getWriters() {
+		return environment != null ? environment.getAllWriters() : null;
+	}
+
+	public IntermediateResultPartition[] getProducedPartitions() {
+		return environment != null ? environment.getProducedPartitions() : null;
+	}
+
 	// --------------------------------------------------------------------------------------------
 	//                                     State Listeners
 	// --------------------------------------------------------------------------------------------
-	
-	public void registerExecutionListener(ActorRef listener){
+
+	public void registerExecutionListener(ActorRef listener) {
 		executionListenerActors.add(listener);
 	}
 
-	public void unregisterExecutionListener(ActorRef listener){
+	public void unregisterExecutionListener(ActorRef listener) {
 		executionListenerActors.remove(listener);
 	}
-	
+
 	private void notifyObservers(ExecutionState newState, String message) {
 		if (LOG.isInfoEnabled()) {
 			LOG.info(getTaskNameWithSubtasks() + " switched to " + newState + (message == null ? "" : " : " + message));
 		}
-		
-		for(ActorRef listener: executionListenerActors){
+
+		for (ActorRef listener : executionListenerActors) {
 			listener.tell(new ExecutionGraphMessages.ExecutionStateChanged(
 							jobId, vertexId, taskName, numberOfSubtasks, subtaskIndex,
-							executionId, newState,System.currentTimeMillis(), message),
+							executionId, newState, System.currentTimeMillis(), message),
 					ActorRef.noSender());
 		}
 	}
-	
+
 	// --------------------------------------------------------------------------------------------
 	//                                       Utilities
 	// --------------------------------------------------------------------------------------------
-	
+
 	@Override
 	public String toString() {
 		return getTaskNameWithSubtasks() + " [" + executionState + ']';

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java
index b0f1089..8ca1544 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java
@@ -68,4 +68,4 @@ public class AtomicDisposableReferenceCounter {
 			return isDisposed;
 		}
 	}
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/util/BufferPoolConnector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/BufferPoolConnector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/BufferPoolConnector.java
deleted file mode 100644
index 87d030e..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/BufferPoolConnector.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.util;
-
-import java.util.Queue;
-
-import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.runtime.io.network.BufferRecycler;
-
-public class BufferPoolConnector implements BufferRecycler {
-
-	/**
-	 * Reference to the memory pool the byte buffer was originally taken from.
-	 */
-	private final Queue<MemorySegment> memoryPool;
-
-	/**
-	 * Constructs a new buffer pool connector
-	 *
-	 * @param bufferPool
-	 *        a reference to the memory pool the byte buffer was originally taken from
-	 */
-	public BufferPoolConnector(final Queue<MemorySegment> bufferPool) {
-		this.memoryPool = bufferPool;
-	}
-
-	@Override
-	public void recycle(final MemorySegment buffer) {
-		synchronized (this.memoryPool) {
-			this.memoryPool.add(buffer);
-			this.memoryPool.notify();
-		}
-	}
-}