You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2014/09/21 04:12:36 UTC

[12/63] [abbrv] Refactor job graph construction to incremental attachment based

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphIterator.java
deleted file mode 100644
index f4efc88..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphIterator.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.Stack;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class provides an implementation of the {@link Iterator} interface which allows to
- * traverse an execution graph and visit every reachable vertex exactly once. The order
- * in which the vertices are visited corresponds to the order of their discovery in a depth first
- * search.
- * <p>
- * This class is not thread-safe.
- * 
- */
-public class ExecutionGraphIterator implements Iterator<ExecutionVertex> {
-
-	/**
-	 * The log object used for debugging.
-	 */
-	private static final Logger LOG = LoggerFactory.getLogger(ExecutionGraphIterator.class);
-
-	/**
-	 * The execution this iterator traverses.
-	 */
-	private final ExecutionGraph executionGraph;
-
-	/**
-	 * Stores whether the graph is traversed starting from the input or the output vertices.
-	 */
-	private final boolean forward;
-
-	/**
-	 * The stage that should be traversed by this iterator.
-	 */
-	private final int startStage;
-
-	/**
-	 * Stores whether the iterator is confined to the start stage or not.
-	 */
-	private final boolean confinedToStage;
-
-	/**
-	 * The number of visited vertices from the entry set (either input or output vertices).
-	 */
-	private int numVisitedEntryVertices = 0;
-
-	/**
-	 * Stack used for the depth first search.
-	 */
-	private final Stack<TraversalEntry> traversalStack = new Stack<TraversalEntry>();
-
-	/**
-	 * Set of already visited vertices during traversal.
-	 */
-	private final Set<ExecutionVertex> alreadyVisited = new HashSet<ExecutionVertex>();
-
-	/**
-	 * Auxiliary class which stores which vertices have already been visited.
-	 * 
-	 */
-	private static class TraversalEntry {
-
-		/**
-		 * Execution vertex this entry has been created for.
-		 */
-		private final ExecutionVertex executionVertex;
-
-		/**
-		 * Next gate to traverse.
-		 */
-		private int currentGate;
-
-		/**
-		 * Next channel to traverse.
-		 */
-		private int currentChannel;
-
-		/**
-		 * Constructs a new traversal entry.
-		 * 
-		 * @param executionVertex
-		 *        the execution vertex this entry belongs to
-		 * @param currentGate
-		 *        the gate index to use to visit the next vertex
-		 * @param currentChannel
-		 *        the channel index to use to visit the next vertex
-		 */
-		public TraversalEntry(final ExecutionVertex executionVertex, final int currentGate, final int currentChannel) {
-			this.executionVertex = executionVertex;
-			this.currentGate = currentGate;
-			this.currentChannel = currentChannel;
-		}
-
-		/**
-		 * Returns the execution vertex this entry belongs to.
-		 * 
-		 * @return the execution vertex this entry belongs to
-		 */
-		public ExecutionVertex getExecutionVertex() {
-			return this.executionVertex;
-		}
-
-		/**
-		 * Returns the gate index to use to visit the next vertex.
-		 * 
-		 * @return the gate index to use to visit the next vertex
-		 */
-		public int getCurrentGate() {
-			return this.currentGate;
-		}
-
-		/**
-		 * Returns the channel index to use to visit the next vertex.
-		 * 
-		 * @return the channel index to use to visit the next vertex
-		 */
-		public int getCurrentChannel() {
-			return this.currentChannel;
-		}
-
-		/**
-		 * Increases the channel index by one.
-		 */
-		public void increaseCurrentChannel() {
-			this.currentChannel++;
-		}
-
-		/**
-		 * Increases the gate index by one.
-		 */
-		public void increaseCurrentGate() {
-			this.currentGate++;
-		}
-
-		/**
-		 * Resets the channel index.
-		 */
-		public void resetCurrentChannel() {
-			this.currentChannel = 0;
-		}
-
-	}
-
-	/**
-	 * Creates a new execution graph iterator.
-	 * 
-	 * @param executionGraph
-	 *        the execution graph that should be traversed
-	 * @param forward
-	 *        <code>true</code> if the graph should be traversed in correct order, <code>false</code> to traverse it in
-	 *        reverse order
-	 */
-	public ExecutionGraphIterator(final ExecutionGraph executionGraph, final boolean forward) {
-		this(executionGraph, forward ? 0 : (executionGraph.getNumberOfStages() - 1), false, forward);
-	}
-
-	/**
-	 * Creates a new execution graph iterator.
-	 * 
-	 * @param executionGraph
-	 *        the execution graph that should be traversed
-	 * @param startStage
-	 *        the index of the stage of the graph where the traversal is supposed to begin
-	 * @param confinedToStage
-	 *        <code>false</code> if the graph iterator is allowed to traverse to upper (in case of reverse order
-	 *        traversal lower) stages, <code>true</code> otherwise.
-	 * @param forward
-	 *        <code>true</code> if the graph should be traversed in correct order, <code>false</code> to traverse it in
-	 *        reverse order
-	 */
-	public ExecutionGraphIterator(final ExecutionGraph executionGraph, final int startStage,
-			final boolean confinedToStage, final boolean forward) {
-
-		this.executionGraph = executionGraph;
-		this.forward = forward;
-		this.startStage = startStage;
-		this.confinedToStage = confinedToStage;
-
-		if (startStage >= this.executionGraph.getNumberOfStages()) {
-			return;
-		}
-
-		if (forward) {
-			if (executionGraph.getNumberOfInputVertices(startStage) > 0) {
-
-				final TraversalEntry te = new TraversalEntry(executionGraph.getInputVertex(startStage, 0), 0, 0);
-				this.traversalStack.push(te);
-				this.alreadyVisited.add(te.getExecutionVertex());
-
-			}
-		} else {
-			if (executionGraph.getNumberOfOutputVertices(startStage) > 0) {
-
-				final TraversalEntry te = new TraversalEntry(executionGraph.getOutputVertex(startStage, 0), 0, 0);
-				this.traversalStack.push(te);
-				this.alreadyVisited.add(te.getExecutionVertex());
-			}
-
-		}
-	}
-
-	/**
-	 * Creates a new execution graph iterator. This constructor can be used to
-	 * traverse only specific parts of the graph starting at <code>startVertex</code>.
-	 * The iterator will not switch to the next input/output vertex of an output/input vertex
-	 * has been reached.
-	 * 
-	 * @param executionGraph
-	 *        the execution graph that should be traversed
-	 * @param startVertex
-	 *        the vertex to start the traversal from
-	 * @param forward
-	 *        <code>true</code> if the graph should be traversed in correct order, <code>false</code> to reverse it in
-	 *        reverse order
-	 */
-	public ExecutionGraphIterator(final ExecutionGraph executionGraph, final ExecutionVertex startVertex,
-			final boolean forward) {
-
-		this.executionGraph = executionGraph;
-		this.forward = forward;
-		this.numVisitedEntryVertices = -1;
-		this.startStage = 0;
-		this.confinedToStage = false;
-
-		final TraversalEntry te = new TraversalEntry(startVertex, 0, 0);
-		this.traversalStack.push(te);
-		this.alreadyVisited.add(startVertex);
-	}
-
-
-	@Override
-	public boolean hasNext() {
-
-		if (this.traversalStack.isEmpty()) {
-
-			if (this.numVisitedEntryVertices < 0) {
-				// User chose a specific starting vertex
-				return false;
-			}
-
-			++this.numVisitedEntryVertices;
-
-			if (this.forward) {
-				if (this.executionGraph.getNumberOfInputVertices(this.startStage) <= this.numVisitedEntryVertices) {
-					return false;
-				}
-			} else {
-				if (this.executionGraph.getNumberOfOutputVertices(this.startStage) <= this.numVisitedEntryVertices) {
-					return false;
-				}
-			}
-		}
-
-		return true;
-	}
-
-
-	@Override
-	public ExecutionVertex next() {
-
-		if (this.traversalStack.isEmpty()) {
-
-			if (this.numVisitedEntryVertices < 0) {
-				// User chose a specific entry vertex
-				return null;
-			}
-
-			TraversalEntry newentry;
-
-			if (this.forward) {
-				newentry = new TraversalEntry(this.executionGraph.getInputVertex(this.startStage,
-					this.numVisitedEntryVertices),
-					0, 0);
-			} else {
-				newentry = new TraversalEntry(this.executionGraph.getOutputVertex(this.startStage,
-					this.numVisitedEntryVertices),
-					0, 0);
-			}
-
-			this.traversalStack.push(newentry);
-			this.alreadyVisited.add(newentry.getExecutionVertex());
-		}
-
-		final ExecutionVertex returnVertex = this.traversalStack.peek().getExecutionVertex();
-
-		// Propose vertex to be visited next
-		do {
-
-			final TraversalEntry te = this.traversalStack.peek();
-
-			// Check if we can traverse deeper into the graph
-			final ExecutionVertex candidateVertex = getCandidateVertex(te, forward);
-			if (candidateVertex == null) {
-				// Pop it from the stack
-				this.traversalStack.pop();
-			} else {
-				// Create new entry and put it on the stack
-				final TraversalEntry newte = new TraversalEntry(candidateVertex, 0, 0);
-				this.traversalStack.push(newte);
-				this.alreadyVisited.add(candidateVertex);
-				break;
-			}
-
-		} while (!this.traversalStack.isEmpty());
-
-		return returnVertex;
-	}
-
-	/**
-	 * Returns a candidate vertex which could potentially be visited next because it is reachable from the
-	 * currently considered vertex.
-	 * 
-	 * @param te
-	 *        the traversal entry for the current source vertex
-	 * @param forward
-	 *        <code>true</code> if the graph should be traversed in correct order, <code>false</code> to traverse it in
-	 *        reverse order
-	 * @return a candidate vertex which could potentially be visited next
-	 */
-	private ExecutionVertex getCandidateVertex(final TraversalEntry te, final boolean forward) {
-
-		if (forward) {
-
-			while (true) {
-
-				if (this.confinedToStage && te.getCurrentChannel() == 0) {
-					while (currentGateLeadsToOtherStage(te, this.forward)) {
-						te.increaseCurrentGate();
-					}
-				}
-
-				// No more outgoing edges to consider
-				if (te.getCurrentGate() >= te.getExecutionVertex().getNumberOfOutputGates()) {
-					break;
-				}
-
-				if (te.getCurrentChannel() >= te.getExecutionVertex().getOutputGate(te.getCurrentGate())
-					.getNumberOfEdges()) {
-					te.increaseCurrentGate();
-					te.resetCurrentChannel();
-				} else {
-					final ExecutionEdge outputChannel = te.getExecutionVertex().getOutputGate(te.getCurrentGate())
-						.getEdge(te.getCurrentChannel());
-					final ExecutionVertex tmp = outputChannel.getInputGate().getVertex();
-					if (tmp == null) {
-						LOG.error("Inconsistency in vertex map found (forward)!");
-					}
-					te.increaseCurrentChannel();
-					if (!this.alreadyVisited.contains(tmp)) {
-						return tmp;
-					}
-				}
-			}
-		} else {
-
-			while (true) {
-
-				// No more outgoing edges to consider
-				if (te.getCurrentGate() >= te.getExecutionVertex().getNumberOfInputGates()) {
-					break;
-				}
-
-				if (te.getCurrentChannel() >= te.getExecutionVertex().getInputGate(te.getCurrentGate())
-					.getNumberOfEdges()) {
-					te.increaseCurrentGate();
-					te.resetCurrentChannel();
-				} else {
-					final ExecutionEdge inputChannel = te.getExecutionVertex().getInputGate(te.getCurrentGate()).getEdge(te.getCurrentChannel());
-					final ExecutionVertex tmp = inputChannel.getOutputGate().getVertex();
-					if (tmp == null) {
-						LOG.error("Inconsistency in vertex map found (backward)!");
-					}
-					te.increaseCurrentChannel();
-					if (!this.alreadyVisited.contains(tmp)) {
-						return tmp;
-					}
-				}
-			}
-		}
-
-		return null;
-	}
-
-	private boolean currentGateLeadsToOtherStage(final TraversalEntry te, final boolean forward) {
-
-		final ExecutionGroupVertex groupVertex = te.getExecutionVertex().getGroupVertex();
-
-		if (forward) {
-
-			if (te.getCurrentGate() >= groupVertex.getNumberOfForwardLinks()) {
-				return false;
-			}
-
-			final ExecutionGroupEdge edge = groupVertex.getForwardEdge(te.getCurrentGate());
-			if (edge.getTargetVertex().getStageNumber() == groupVertex.getStageNumber()) {
-				return false;
-			}
-
-		} else {
-			if (te.getCurrentGate() >= groupVertex.getNumberOfBackwardLinks()) {
-				return false;
-			}
-
-			final ExecutionGroupEdge edge = groupVertex.getBackwardEdge(te.getCurrentGate());
-			if (edge.getSourceVertex().getStageNumber() == groupVertex.getStageNumber()) {
-				return false;
-			}
-		}
-
-		return true;
-	}
-
-
-	@Override
-	public void remove() {
-
-		throw new UnsupportedOperationException("The method remove is not implemented for this type of iterator");
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGroupVertexIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGroupVertexIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGroupVertexIterator.java
deleted file mode 100644
index 1095925..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGroupVertexIterator.java
+++ /dev/null
@@ -1,331 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.Stack;
-
-/**
- * This class provides an implementation of the {@link Iterator} interface which allows to
- * traverse an execution graph and visit every reachable group vertex exactly once. The order
- * in which the group vertices are visited corresponds to the order of their discovery in a depth first
- * search.
- * <p>
- * This class is not thread-safe.
- * 
- */
-public class ExecutionGroupVertexIterator implements Iterator<ExecutionGroupVertex> {
-
-	/**
-	 * Stores whether the group graph is traversed starting from the input or the output vertices.
-	 */
-	private final boolean forward;
-
-	/**
-	 * The stage to traverse, -1 to traverse all stages of the graph.
-	 */
-	private final int stage;
-
-	/**
-	 * List of entry vertices for the traversal (either input or output vertices).
-	 */
-	private final List<ExecutionGroupVertex> entryVertices = new ArrayList<ExecutionGroupVertex>();
-
-	/**
-	 * Number of already visited entry vertices.
-	 */
-	private int numVisitedEntryVertices = 0;
-
-	/**
-	 * Stack used for the traversal.
-	 */
-	private final Stack<TraversalEntry> traversalStack = new Stack<TraversalEntry>();
-
-	/**
-	 * Set storing the vertices already visited during traversal.
-	 */
-	private final Set<ExecutionGroupVertex> alreadyVisited = new HashSet<ExecutionGroupVertex>();
-
-	/**
-	 * Auxiliary class which stores which vertices have already been visited.
-	 * 
-	 */
-	private static class TraversalEntry {
-
-		/**
-		 * The group vertex this entry is created for.
-		 */
-		private final ExecutionGroupVertex groupVertex;
-
-		/**
-		 * The current outgoing link of the group vertex.
-		 */
-		private int currentLink = 0;
-
-		/**
-		 * Constructs a new traversal entry.
-		 * 
-		 * @param groupVertex
-		 *        the group vertex this traversal entry belongs to
-		 * @param currentLink
-		 *        the link index to use to visit the next group vertex
-		 */
-		public TraversalEntry(final ExecutionGroupVertex groupVertex, final int currentLink) {
-			this.groupVertex = groupVertex;
-			this.currentLink = currentLink;
-		}
-
-		/**
-		 * Returns the group vertex this traversal entry belongs to.
-		 * 
-		 * @return the group vertex this traversal entry belongs to
-		 */
-		public ExecutionGroupVertex getGroupVertex() {
-			return this.groupVertex;
-		}
-
-		/**
-		 * Returns the link index to use to visit the next group vertex.
-		 * 
-		 * @return the link index to use to visit the next group vertex
-		 */
-		public int getCurrentLink() {
-			return this.currentLink;
-		}
-
-		/**
-		 * Increases the link index by one.
-		 */
-		public void increaseCurrentLink() {
-			this.currentLink++;
-		}
-	}
-
-	/**
-	 * Creates a new execution group vertex iterator.
-	 * 
-	 * @param executionGraph
-	 *        the execution graph that should be traversed
-	 * @param forward
-	 *        <code>true</code> if the graph should be traversed in correct order, <code>false</code> to reverse it in
-	 *        reverse order
-	 * @param stage
-	 *        the number of the stage whose vertices should be traversed or -1 if all stages should be included in the
-	 *        traversal
-	 */
-	public ExecutionGroupVertexIterator(final ExecutionGraph executionGraph, final boolean forward, final int stage) {
-
-		this.forward = forward;
-		this.stage = stage;
-
-		// Collect start vertices
-		if (stage < 0) {
-
-			for (int i = 0; i < executionGraph.getNumberOfStages(); i++) {
-				collectStartVertices(executionGraph.getStage(i));
-			}
-		} else {
-			if (stage < executionGraph.getNumberOfStages()) {
-				collectStartVertices(executionGraph.getStage(stage));
-			}
-		}
-
-		if (this.entryVertices.size() > 0) {
-			final TraversalEntry te = new TraversalEntry(this.entryVertices.get(0), 0);
-			this.traversalStack.push(te);
-			this.alreadyVisited.add(te.getGroupVertex());
-		}
-	}
-
-	/**
-	 * Collects all input group vertices (i.e. vertices with no incoming link or incoming links from other stages) in
-	 * the given stage and adds them to an internal list.
-	 * 
-	 * @param stage
-	 *        the number of the stage whose input vertices should be collected
-	 */
-	private void collectStartVertices(final ExecutionStage stage) {
-
-		for (int i = 0; i < stage.getNumberOfStageMembers(); i++) {
-
-			final ExecutionGroupVertex groupVertex = stage.getStageMember(i);
-
-			if (forward) {
-				if ((groupVertex.getNumberOfBackwardLinks() == 0)
-					|| ((this.stage >= 0) && allConnectionsFromOtherStage(groupVertex, true))) {
-					this.entryVertices.add(groupVertex);
-				}
-			} else {
-				if ((groupVertex.getNumberOfForwardLinks() == 0)
-					|| ((this.stage >= 0) && allConnectionsFromOtherStage(groupVertex, false))) {
-					this.entryVertices.add(groupVertex);
-				}
-			}
-		}
-	}
-
-	/**
-	 * Checks if for the given group vertex all incoming (if forward is <code>true</code>) or outgoing links (if forward
-	 * is <code>false</code>) come from
-	 * other stages than the one the given vertex is in.
-	 * 
-	 * @param groupVertex
-	 *        the group vertex to check for
-	 * @param forward
-	 *        <code>true</code> if incoming links should be considered, <code>false</code> for outgoing links
-	 * @return <code>true</code> if all incoming or outgoing links (depends on the forward switch) come from other
-	 *         stages, <code>false</code> otherwise
-	 */
-	private boolean allConnectionsFromOtherStage(final ExecutionGroupVertex groupVertex, final boolean forward) {
-
-		if (forward) {
-			for (int i = 0; i < groupVertex.getNumberOfBackwardLinks(); i++) {
-				if (this.stage == groupVertex.getBackwardEdge(i).getSourceVertex().getStageNumber()) {
-					return false;
-				}
-			}
-		} else {
-			for (int i = 0; i < groupVertex.getNumberOfForwardLinks(); i++) {
-				if (this.stage == groupVertex.getForwardEdge(i).getTargetVertex().getStageNumber()) {
-					return false;
-				}
-			}
-		}
-
-		return true;
-	}
-
-
-	@Override
-	public boolean hasNext() {
-
-		if (this.traversalStack.isEmpty()) {
-
-			++this.numVisitedEntryVertices;
-			if (this.entryVertices.size() <= this.numVisitedEntryVertices) {
-				return false;
-			}
-		}
-
-		return true;
-
-	}
-
-
-	@Override
-	public ExecutionGroupVertex next() {
-
-		if (this.traversalStack.isEmpty()) {
-
-			final TraversalEntry newentry = new TraversalEntry(this.entryVertices.get(this.numVisitedEntryVertices), 0);
-			this.traversalStack.push(newentry);
-			this.alreadyVisited.add(newentry.getGroupVertex());
-		}
-
-		final ExecutionGroupVertex returnVertex = this.traversalStack.peek().getGroupVertex();
-
-		// Propose vertex to be visited next
-		do {
-
-			final TraversalEntry te = this.traversalStack.peek();
-
-			// Check if we can traverse deeper into the graph
-			final ExecutionGroupVertex candidateVertex = getCandidateVertex(te, forward);
-			if (candidateVertex == null) {
-				// Pop it from the stack
-				traversalStack.pop();
-			} else {
-				// Create new entry and put it on the stack
-				final TraversalEntry newte = new TraversalEntry(candidateVertex, 0);
-				this.traversalStack.push(newte);
-				this.alreadyVisited.add(newte.getGroupVertex());
-				break;
-			}
-
-		} while (!this.traversalStack.isEmpty());
-
-		return returnVertex;
-
-	}
-
-	/**
-	 * Returns a candidate group vertex which could potentially be visited next because it is reachable from the
-	 * currently considered group vertex.
-	 * 
-	 * @param te
-	 *        the traversal entry for the current source group vertex
-	 * @param forward
-	 *        <code>true</code> if the graph should be traversed in correct order, <code>false</code> to traverse it in
-	 *        reverse order
-	 * @return a candidate group vertex which could potentially be visited next
-	 */
-	private ExecutionGroupVertex getCandidateVertex(final TraversalEntry te, final boolean forward) {
-
-		while (true) {
-
-			if (forward) {
-				// No more outgoing links to consider
-				if (te.getCurrentLink() >= te.getGroupVertex().getNumberOfForwardLinks()) {
-					break;
-				}
-			} else {
-				// No more outgoing links to consider
-				if (te.getCurrentLink() >= te.getGroupVertex().getNumberOfBackwardLinks()) {
-					break;
-				}
-			}
-
-			ExecutionGroupVertex tmp = null;
-			if (forward) {
-				tmp = te.getGroupVertex().getForwardEdge(te.getCurrentLink()).getTargetVertex();
-			} else {
-				tmp = te.getGroupVertex().getBackwardEdge(te.getCurrentLink()).getSourceVertex();
-			}
-
-			// Increase the current link index by one
-			te.increaseCurrentLink();
-
-			// If stage >= 0, tmp must be in the same stage as te.getGroupVertex()
-			if (this.stage >= 0) {
-				if (tmp.getStageNumber() != this.stage) {
-					continue;
-				}
-			}
-
-			if (!this.alreadyVisited.contains(tmp)) {
-				return tmp;
-			}
-		}
-
-		return null;
-	}
-
-
-	@Override
-	public void remove() {
-		// According to the documentation this method is optional and does not need to be implemented
-
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
new file mode 100644
index 0000000..72f03b4
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.flink.core.io.InputSplit;
+import org.apache.flink.core.io.InputSplitAssigner;
+import org.apache.flink.core.io.InputSplitSource;
+import org.apache.flink.runtime.JobException;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSet;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
+import org.apache.flink.runtime.jobgraph.JobEdge;
+import org.apache.flink.runtime.jobgraph.JobID;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+
+
+public class ExecutionJobVertex {
+	
+	/** Use the same log for all ExecutionGraph classes */
+	private static final Log LOG = ExecutionGraph.LOG;
+	
+	
+	private final ExecutionGraph graph;
+	
+	private final AbstractJobVertex jobVertex;
+	
+	private final ExecutionVertex2[] taskVertices;
+
+	private final IntermediateResult[] producedDataSets;
+	
+	private final InputSplitAssigner splitAssigner;
+	
+	private final int parallelism;
+	
+	
+	private final AtomicInteger numRunningTasks = new AtomicInteger(0);
+	
+	private final AtomicInteger numFinishedTasks = new AtomicInteger(0);
+	
+	private final AtomicInteger numCancelledOrFailedTasks = new AtomicInteger(0);
+	
+	
+	private SlotSharingGroup slotSharingGroup;
+	
+	
+	public ExecutionJobVertex(ExecutionGraph graph, AbstractJobVertex jobVertex, int defaultParallelism) throws JobException {
+		if (graph == null || jobVertex == null) {
+			throw new NullPointerException();
+		}
+		
+		this.graph = graph;
+		this.jobVertex = jobVertex;
+		
+		int vertexParallelism = jobVertex.getParallelism();
+		int numTaskVertices = vertexParallelism > 0 ? vertexParallelism : defaultParallelism;
+		
+		this.parallelism = numTaskVertices;
+		this.taskVertices = new ExecutionVertex2[numTaskVertices];
+		
+		// create the intermediate results
+		this.producedDataSets = new IntermediateResult[jobVertex.getNumberOfProducedIntermediateDataSets()];
+		for (int i = 0; i < jobVertex.getProducedDataSets().size(); i++) {
+			IntermediateDataSet set = jobVertex.getProducedDataSets().get(i);
+			this.producedDataSets[i] = new IntermediateResult(set.getId(), this, numTaskVertices);
+		}
+		
+		// create all task vertices
+		for (int i = 0; i < numTaskVertices; i++) {
+			ExecutionVertex2 vertex = new ExecutionVertex2(this, i, this.producedDataSets);
+			this.taskVertices[i] = vertex;
+		}
+		
+		// take the sharing group
+		this.slotSharingGroup = jobVertex.getSlotSharingGroup();
+		
+		// set up the input splits, if the vertex has any
+		try {
+			@SuppressWarnings("unchecked")
+			InputSplitSource<InputSplit> splitSource = (InputSplitSource<InputSplit>) jobVertex.getInputSplitSource();
+			if (splitSource != null) {
+				InputSplit[] splits = splitSource.createInputSplits(numTaskVertices);
+				this.splitAssigner = splitSource.getInputSplitAssigner(splits);
+			} else {
+				this.splitAssigner = null;
+			}
+		}
+		catch (Throwable t) {
+			throw new JobException("Creating the input splits caused an error: " + t.getMessage(), t);
+		}
+	}
+
+	public ExecutionGraph getGraph() {
+		return this.graph;
+	}
+	
+	public AbstractJobVertex getJobVertex() {
+		return this.jobVertex;
+	}
+
+	public int getParallelism() {
+		return this.parallelism;
+	}
+	
+	public JobID getJobId() {
+		return this.graph.getJobID();
+	}
+	
+	public JobVertexID getJobVertexId() {
+		return this.jobVertex.getID();
+	}
+	
+	public ExecutionVertex2[] getTaskVertices() {
+		return taskVertices;
+	}
+	
+	public IntermediateResult[] getProducedDataSets() {
+		return producedDataSets;
+	}
+	
+	public InputSplitAssigner getSplitAssigner() {
+		return splitAssigner;
+	}
+	
+	public void setSlotSharingGroup(SlotSharingGroup slotSharingGroup) {
+		this.slotSharingGroup = slotSharingGroup;
+	}
+	
+	public SlotSharingGroup getSlotSharingGroup() {
+		return slotSharingGroup;
+	}
+	
+	//---------------------------------------------------------------------------------------------
+	
+	public void connectToPredecessors(Map<IntermediateDataSetID, IntermediateResult> intermediateDataSets) throws JobException {
+		
+		List<JobEdge> inputs = jobVertex.getInputs();
+		
+		if (LOG.isDebugEnabled()) {
+			LOG.debug(String.format("Connecting ExecutionJobVertex %s (%s) to %d predecessors.", jobVertex.getID(), jobVertex.getName(), inputs.size()));
+		}
+		
+		for (int num = 0; num < inputs.size(); num++) {
+			JobEdge edge = inputs.get(num);
+			
+			if (LOG.isDebugEnabled()) {
+				if (edge.getSource() == null) {
+					LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s.", 
+							num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId()));
+				} else {
+					LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).",
+							num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName()));
+				}
+			}
+			
+			// fetch the intermediate result via ID. if it does not exist, then it either has not been created, or the order
+			// in which this method is called for the job vertices is not a topological order
+			IntermediateResult ires = intermediateDataSets.get(edge.getSourceId());
+			if (ires == null) {
+				throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID "
+						+ edge.getSourceId());
+			}
+			
+			int consumerIndex = ires.registerConsumer();
+			
+			for (int i = 0; i < parallelism; i++) {
+				ExecutionVertex2 ev = taskVertices[i];
+				ev.connectSource(num, ires, edge, consumerIndex);
+			}
+			
+			
+		}
+	}
+	
+	//---------------------------------------------------------------------------------------------
+	//  State, deployment, and recovery logic 
+	//---------------------------------------------------------------------------------------------
+	
+	void vertexSwitchedToRunning(int subtask) {
+		this.numRunningTasks.incrementAndGet();
+	}
+	
+	void vertexFailed(int subtask) {
+		
+	}
+	
+	void vertexCancelled(int subtask) {
+		
+	}
+	
+	// --------------------------------------------------------------------------------------------
+	//  Miscellaneous
+	// --------------------------------------------------------------------------------------------
+	
+	public void execute(Runnable action) {
+		this.graph.execute(action);
+	}
+	
+}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionPipeline.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionPipeline.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionPipeline.java
deleted file mode 100644
index d998f60..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionPipeline.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.util.Iterator;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import org.apache.flink.runtime.execution.ExecutionState;
-import org.apache.flink.runtime.instance.AllocatedResource;
-import org.apache.flink.runtime.util.UnmodifiableIterator;
-
-/**
- * An execution pipeline is a scheduling abstraction which defines a set of {@link ExecutionVertex} objects which must
- * be deployed together. An {@link ExecutionVertex} always belongs to exactly one execution pipeline.
- * <p>
- * This class is thread-safe.
- * 
- */
-public final class ExecutionPipeline {
-
-	/**
-	 * The set of vertices belonging to this execution pipeline.
-	 */
-	private final CopyOnWriteArrayList<ExecutionVertex> vertices = new CopyOnWriteArrayList<ExecutionVertex>();
-
-	/**
-	 * Adds the given {@link ExecutionVertex} to this pipeline.
-	 * 
-	 * @param vertex
-	 *        the vertex to be added to this pipeline
-	 */
-	void addToPipeline(final ExecutionVertex vertex) {
-
-		if (!this.vertices.addIfAbsent(vertex)) {
-			throw new IllegalStateException("Vertex " + vertex + " has already been added to pipeline " + this);
-		}
-	}
-
-	/**
-	 * Removes the given {@link ExecutionVertex} from this pipeline.
-	 * 
-	 * @param vertex
-	 *        the vertex to be removed from this pipeline.
-	 */
-	void removeFromPipeline(final ExecutionVertex vertex) {
-
-		if (!this.vertices.remove(vertex)) {
-			throw new IllegalStateException("Vertex " + vertex + " was not part of the pipeline " + this);
-		}
-	}
-
-	/**
-	 * Returns an {@link Iterator} to the vertices which are part of this pipeline.
-	 * 
-	 * @return an iterator to the vertices of this pipeline
-	 */
-	public Iterator<ExecutionVertex> iterator() {
-
-		return new UnmodifiableIterator<ExecutionVertex>(this.vertices.iterator());
-	}
-
-	/**
-	 * Checks if the pipeline is currently finishing its execution, i.e. all vertices contained in the pipeline have
-	 * switched to the <code>FINISHING</code> or <code>FINISHED</code> state.
-	 * 
-	 * @return <code>true</code> if the pipeline is currently finishing, <code>false</code> otherwise
-	 */
-	public boolean isFinishing() {
-
-		final Iterator<ExecutionVertex> it = this.vertices.iterator();
-		while (it.hasNext()) {
-
-			final ExecutionState state = it.next().getExecutionState();
-			if (state != ExecutionState.FINISHING && state != ExecutionState.FINISHED) {
-				return false;
-			}
-		}
-
-		return true;
-	}
-
-	/**
-	 * Sets the given allocated resource for all vertices included in this pipeline.
-	 * 
-	 * @param resource
-	 *        the allocated resource to set for all vertices included in this pipeline
-	 */
-	public void setAllocatedResource(final AllocatedResource resource) {
-
-		final Iterator<ExecutionVertex> it = this.vertices.iterator();
-		while (it.hasNext()) {
-			final ExecutionVertex vertex = it.next();
-			vertex.setAllocatedResource(resource);
-		}
-	}
-
-	/**
-	 * Updates the execution state for all vertices included in this pipeline.
-	 * 
-	 * @param executionState
-	 *        the execution state to set for all vertices included in this pipeline
-	 */
-	public void updateExecutionState(final ExecutionState executionState) {
-
-		final Iterator<ExecutionVertex> it = this.vertices.iterator();
-		while (it.hasNext()) {
-			final ExecutionVertex vertex = it.next();
-			vertex.updateExecutionState(executionState);
-		}
-
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionSignature.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionSignature.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionSignature.java
deleted file mode 100644
index 40f2468..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionSignature.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager;
-import org.apache.flink.runtime.jobgraph.JobID;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-
-/**
- * An execution signature allows to uniquely identify a job vertex. The signature involves the name of the
- * class to be invoked at runtime as well as a cryptographic hash of all the JAR files which are required to
- * instantiate the class and run it. The execution is the basis for feedback learning as it enables the profiler
- * to recognize particular parts of a job. Execution signature objects are immutable and, consequently, thread-safe.
- * <p>
- * This class is thread-safe.
- * 
- */
-public final class ExecutionSignature {
-
-	/**
-	 * The log object used for debugging.
-	 */
-	private static final Logger LOG = LoggerFactory.getLogger(ExecutionSignature.class);
-
-	/**
-	 * The name of the hashing algorithm to be used.
-	 */
-	private static final String HASHINGALGORITHM = "SHA-1";
-
-	/**
-	 * The message digest object used to calculate the signature.
-	 */
-	private static MessageDigest messageDigest = null;
-
-	/**
-	 * The buffer storing the signature.
-	 */
-	private final byte[] signature;
-
-	/**
-	 * Constructs a new execution signature object and passes the signature buffer.
-	 * 
-	 * @param signature
-	 *        the byte buffer containing the signature.
-	 */
-	private ExecutionSignature(final byte[] signature) {
-		this.signature = signature;
-	}
-
-	/**
-	 * Calculates the execution signature from the given class name and job ID.
-	 * 
-	 * @param invokableClass
-	 *        the name of the class to contain the task program
-	 * @param jobID
-	 *        the ID of the job
-	 * @return the cryptographic signature of this vertex
-	 */
-	public static synchronized ExecutionSignature createSignature(
-			final Class<? extends AbstractInvokable> invokableClass, final JobID jobID) {
-
-		// First, try to load message digest algorithm, if necessary
-		if (messageDigest == null) {
-			try {
-				messageDigest = MessageDigest.getInstance(HASHINGALGORITHM);
-			} catch (NoSuchAlgorithmException e) {
-				LOG.error("Unable to load message digest algorithm " + HASHINGALGORITHM);
-				return null;
-			}
-		}
-
-		// Reset digest buffer and add the name of the invokable class to the message digest buffer
-		messageDigest.reset();
-		messageDigest.update(invokableClass.getName().getBytes());
-
-		String[] requiredJarFiles;
-		// Next, retrieve the JAR-files associated with this job
-		try {
-			requiredJarFiles = LibraryCacheManager.getRequiredJarFiles(jobID);
-		} catch (IOException ioe) {
-			// Output an error message and return
-			LOG.error("Cannot access library cache manager for job ID " + jobID);
-			return null;
-		}
-
-		// Now, sort the list of JAR-files in order to always calculate the signature in the same manner
-		Arrays.sort(requiredJarFiles);
-
-		// Finally, add the names of the JAR-files to the hash calculation
-		for (int i = 0; i < requiredJarFiles.length; i++) {
-			messageDigest.update(requiredJarFiles[i].getBytes());
-		}
-
-		return new ExecutionSignature(messageDigest.digest());
-	}
-
-
-	@Override
-	public boolean equals(final Object obj) {
-
-		if (obj instanceof ExecutionSignature) {
-
-			final ExecutionSignature executionSignature = (ExecutionSignature) obj;
-			return Arrays.equals(this.signature, executionSignature.signature);
-		}
-
-		return false;
-	}
-
-
-	@Override
-	public int hashCode() {
-
-		int hashCode = 0;
-
-		for (int i = 0; i < this.signature.length; i++) {
-			hashCode += this.signature[i];
-		}
-
-		return hashCode;
-	}
-
-
-	@Override
-	public String toString() {
-
-		final StringBuffer stringBuffer = new StringBuffer();
-		for (int i = 0; i < this.signature.length; i++) {
-			stringBuffer.append(Integer.toHexString(0xFF & this.signature[i]));
-		}
-
-		return stringBuffer.toString();
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStage.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStage.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStage.java
deleted file mode 100644
index 54e7b12..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStage.java
+++ /dev/null
@@ -1,411 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import org.apache.flink.runtime.instance.Instance;
-import org.apache.flink.runtime.io.network.channels.ChannelType;
-
-/**
- * An execution stage contains all execution group vertices (and as a result all execution vertices) which
- * must run at the same time. The execution of a job progresses in terms of stages, i.e. the next stage of a
- * job can only start to execute if the execution of its preceding stage is complete.
- * <p>
- * This class is thread-safe.
- */
-public final class ExecutionStage {
-
-	/**
-	 * The execution graph that this stage belongs to.
-	 */
-	private final ExecutionGraph executionGraph;
-
-	/**
-	 * List of group vertices which are assigned to this stage.
-	 */
-	private final CopyOnWriteArrayList<ExecutionGroupVertex> stageMembers = new CopyOnWriteArrayList<ExecutionGroupVertex>();
-
-	/**
-	 * Number of the stage.
-	 */
-	private volatile int stageNum = -1;
-
-	/**
-	 * Constructs a new execution stage and assigns the given stage number to it.
-	 * 
-	 * @param executionGraph
-	 *        the executionGraph that this stage belongs to
-	 * @param stageNum
-	 *        the number of this execution stage
-	 */
-	public ExecutionStage(final ExecutionGraph executionGraph, final int stageNum) {
-		this.executionGraph = executionGraph;
-		this.stageNum = stageNum;
-	}
-
-	/**
-	 * Sets the number of this execution stage.
-	 * 
-	 * @param stageNum
-	 *        the new number of this execution stage
-	 */
-	public void setStageNumber(final int stageNum) {
-		this.stageNum = stageNum;
-	}
-
-	/**
-	 * Returns the number of this execution stage.
-	 * 
-	 * @return the number of this execution stage
-	 */
-	public int getStageNumber() {
-
-		return this.stageNum;
-	}
-
-	/**
-	 * Adds a new execution group vertex to this stage if it is not already included.
-	 * 
-	 * @param groupVertex
-	 *        the new execution group vertex to include
-	 */
-	public void addStageMember(final ExecutionGroupVertex groupVertex) {
-
-		if (this.stageMembers.addIfAbsent(groupVertex)) {
-			groupVertex.setExecutionStage(this);
-		}
-	}
-
-	/**
-	 * Removes the specified group vertex from the execution stage.
-	 * 
-	 * @param groupVertex
-	 *        the group vertex to remove from the stage
-	 */
-	public void removeStageMember(ExecutionGroupVertex groupVertex) {
-
-		this.stageMembers.remove(groupVertex);
-	}
-
-	/**
-	 * Returns the number of group vertices this execution stage includes.
-	 * 
-	 * @return the number of group vertices this execution stage includes
-	 */
-	public int getNumberOfStageMembers() {
-
-		return this.stageMembers.size();
-	}
-
-	/**
-	 * Returns the stage member internally stored at index <code>index</code>.
-	 * 
-	 * @param index
-	 *        the index of the group vertex to return
-	 * @return the stage member internally stored at the specified index or <code>null</code> if no group vertex exists
-	 *         with such an index
-	 */
-	public ExecutionGroupVertex getStageMember(final int index) {
-
-		try {
-			return this.stageMembers.get(index);
-		} catch (ArrayIndexOutOfBoundsException e) {
-			return null;
-		}
-	}
-
-	/**
-	 * Returns the number of input execution vertices in this stage, i.e. the number
-	 * of execution vertices which are connected to vertices in a lower stage
-	 * or have no input channels.
-	 * 
-	 * @return the number of input vertices in this stage
-	 */
-	public int getNumberOfInputExecutionVertices() {
-
-		int retVal = 0;
-
-		final Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
-		while (it.hasNext()) {
-
-			final ExecutionGroupVertex groupVertex = it.next();
-			if (groupVertex.isInputVertex()) {
-				retVal += groupVertex.getCurrentNumberOfGroupMembers();
-			}
-		}
-
-		return retVal;
-	}
-
-	/**
-	 * Returns the number of output execution vertices in this stage, i.e. the number
-	 * of execution vertices which are connected to vertices in a higher stage
-	 * or have no output channels.
-	 * 
-	 * @return the number of output vertices in this stage
-	 */
-	public int getNumberOfOutputExecutionVertices() {
-
-		int retVal = 0;
-
-		final Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
-		while (it.hasNext()) {
-
-			final ExecutionGroupVertex groupVertex = it.next();
-			if (groupVertex.isOutputVertex()) {
-				retVal += groupVertex.getCurrentNumberOfGroupMembers();
-			}
-		}
-
-		return retVal;
-	}
-
-	/**
-	 * Returns the output execution vertex with the given index or <code>null</code> if no such vertex exists.
-	 * 
-	 * @param index
-	 *        the index of the vertex to be selected.
-	 * @return the output execution vertex with the given index or <code>null</code> if no such vertex exists
-	 */
-	public ExecutionVertex getInputExecutionVertex(int index) {
-
-		final Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
-		while (it.hasNext()) {
-
-			final ExecutionGroupVertex groupVertex = it.next();
-			if (groupVertex.isInputVertex()) {
-				final int numberOfMembers = groupVertex.getCurrentNumberOfGroupMembers();
-				if (index >= numberOfMembers) {
-					index -= numberOfMembers;
-				} else {
-					return groupVertex.getGroupMember(index);
-				}
-			}
-		}
-
-		return null;
-	}
-
-	/**
-	 * Returns the input execution vertex with the given index or <code>null</code> if no such vertex exists.
-	 * 
-	 * @param index
-	 *        the index of the vertex to be selected.
-	 * @return the input execution vertex with the given index or <code>null</code> if no such vertex exists
-	 */
-	public ExecutionVertex getOutputExecutionVertex(int index) {
-
-		final Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
-		while (it.hasNext()) {
-
-			final ExecutionGroupVertex groupVertex = it.next();
-			if (groupVertex.isOutputVertex()) {
-				final int numberOfMembers = groupVertex.getCurrentNumberOfGroupMembers();
-				if (index >= numberOfMembers) {
-					index -= numberOfMembers;
-				} else {
-					return groupVertex.getGroupMember(index);
-				}
-			}
-		}
-
-		return null;
-	}
-
-	/**
-	 * Returns the execution graph that this stage belongs to.
-	 * 
-	 * @return the execution graph that this stage belongs to
-	 */
-	public ExecutionGraph getExecutionGraph() {
-
-		return this.executionGraph;
-	}
-
-	/**
-	 * Reconstructs the execution pipelines for this execution stage.
-	 */
-	void reconstructExecutionPipelines() {
-
-		Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
-		final Set<ExecutionVertex> alreadyVisited = new HashSet<ExecutionVertex>();
-
-		while (it.hasNext()) {
-
-			final ExecutionGroupVertex groupVertex = it.next();
-
-			// We only look at input vertices first
-			if (!groupVertex.isInputVertex()) {
-				continue;
-			}
-
-			final Iterator<ExecutionVertex> vertexIt = groupVertex.iterator();
-			while (vertexIt.hasNext()) {
-
-				final ExecutionVertex vertex = vertexIt.next();
-				reconstructExecutionPipeline(vertex, true, alreadyVisited);
-			}
-		}
-
-		it = this.stageMembers.iterator();
-		alreadyVisited.clear();
-
-		while (it.hasNext()) {
-
-			final ExecutionGroupVertex groupVertex = it.next();
-
-			// We only look at input vertices first
-			if (!groupVertex.isOutputVertex()) {
-				continue;
-			}
-
-			final Iterator<ExecutionVertex> vertexIt = groupVertex.iterator();
-			while (vertexIt.hasNext()) {
-
-				final ExecutionVertex vertex = vertexIt.next();
-				reconstructExecutionPipeline(vertex, false, alreadyVisited);
-			}
-		}
-	}
-
-	/**
-	 * Reconstructs the execution pipeline starting at the given vertex by conducting a depth-first search.
-	 * 
-	 * @param vertex
-	 *        the vertex to start the depth-first search from
-	 * @param forward
-	 *        <code>true</code> to traverse the graph according to the original direction of the edges or
-	 *        <code>false</code> for the opposite direction
-	 * @param alreadyVisited
-	 *        a set of vertices that have already been visited in the depth-first search
-	 */
-	private void reconstructExecutionPipeline(final ExecutionVertex vertex, final boolean forward,
-			final Set<ExecutionVertex> alreadyVisited) {
-
-		ExecutionPipeline pipeline = vertex.getExecutionPipeline();
-		if (pipeline == null) {
-			pipeline = new ExecutionPipeline();
-			vertex.setExecutionPipeline(pipeline);
-		}
-
-		alreadyVisited.add(vertex);
-
-		if (forward) {
-
-			final int numberOfOutputGates = vertex.getNumberOfOutputGates();
-			for (int i = 0; i < numberOfOutputGates; ++i) {
-
-				final ExecutionGate outputGate = vertex.getOutputGate(i);
-				final ChannelType channelType = outputGate.getChannelType();
-				final int numberOfOutputChannels = outputGate.getNumberOfEdges();
-				for (int j = 0; j < numberOfOutputChannels; ++j) {
-
-					final ExecutionEdge outputChannel = outputGate.getEdge(j);
-					final ExecutionVertex connectedVertex = outputChannel.getInputGate().getVertex();
-
-					boolean recurse = false;
-
-					if (!alreadyVisited.contains(connectedVertex)) {
-						recurse = true;
-					}
-
-					if (channelType == ChannelType.IN_MEMORY
-						&& !pipeline.equals(connectedVertex.getExecutionPipeline())) {
-
-						connectedVertex.setExecutionPipeline(pipeline);
-						recurse = true;
-					}
-
-					if (recurse) {
-						reconstructExecutionPipeline(connectedVertex, true, alreadyVisited);
-					}
-				}
-			}
-		} else {
-
-			final int numberOfInputGates = vertex.getNumberOfInputGates();
-			for (int i = 0; i < numberOfInputGates; ++i) {
-
-				final ExecutionGate inputGate = vertex.getInputGate(i);
-				final ChannelType channelType = inputGate.getChannelType();
-				final int numberOfInputChannels = inputGate.getNumberOfEdges();
-				for (int j = 0; j < numberOfInputChannels; ++j) {
-
-					final ExecutionEdge inputChannel = inputGate.getEdge(j);
-					final ExecutionVertex connectedVertex = inputChannel.getOutputGate().getVertex();
-
-					boolean recurse = false;
-
-					if (!alreadyVisited.contains(connectedVertex)) {
-						recurse = true;
-					}
-
-					if (channelType == ChannelType.IN_MEMORY
-						&& !pipeline.equals(connectedVertex.getExecutionPipeline())) {
-
-						connectedVertex.setExecutionPipeline(pipeline);
-						recurse = true;
-					}
-
-					if (recurse) {
-						reconstructExecutionPipeline(connectedVertex, false, alreadyVisited);
-					}
-				}
-			}
-		}
-	}
-
-	public int getMaxNumberSubtasks(){
-		int maxDegree = 0;
-
-		for(int i =0; i < this.getNumberOfStageMembers(); i++){
-			final ExecutionGroupVertex groupVertex = this.getStageMember(i);
-
-			if(groupVertex.getCurrentNumberOfGroupMembers() > maxDegree){
-				maxDegree = groupVertex.getCurrentNumberOfGroupMembers();
-			}
-		}
-
-		return maxDegree;
-	}
-
-	public int getRequiredSlots(){
-		Set<Instance> instanceSet = new HashSet<Instance>();
-
-		for(int i=0; i< this.getNumberOfStageMembers(); i++){
-			final ExecutionGroupVertex groupVertex = this.getStageMember(i);
-
-			final Iterator<ExecutionVertex> vertexIterator = groupVertex.iterator();
-
-			while(vertexIterator.hasNext()){
-				final ExecutionVertex vertex = vertexIterator.next();
-
-				instanceSet.add(vertex.getAllocatedResource().getInstance());
-			}
-
-		}
-
-		return instanceSet.size();
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStageListener.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStageListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStageListener.java
deleted file mode 100644
index 16c5478..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStageListener.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import org.apache.flink.runtime.jobgraph.JobID;
-
-/**
- * This interface must be implemented in order to receive a notification whenever the job has entered a new execution
- * stage.
- * 
- */
-public interface ExecutionStageListener {
-
-	/**
-	 * This method is called to indicate that the job with the given {@link JobID} has finished its previous
-	 * {@link ExecutionStage} and has entered the next {@link ExecutionStage}. Note that a notification is not sent when
-	 * the job has entered its initial execution stage.
-	 * 
-	 * @param jobID
-	 *        the ID of the job the notification belongs to
-	 * @param executionStage
-	 *        the next execution stage that has just been entered
-	 */
-	void nextExecutionStageEntered(JobID jobID, ExecutionStage executionStage);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionState.java
new file mode 100644
index 0000000..31108a4
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionState.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+public enum ExecutionState {
+
+	/** The job is scheduled, but nothing is running yet */
+	CREATED,
+	
+	/** At least some tasks of the job are running */
+	RUNNING,
+	
+	/** The execution has finished, no further tasks may be added to the job */
+	FINISHED,
+	
+	/** The execution has failed */
+	FAILED
+}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
index d1ee262..adc0f09 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
@@ -41,11 +41,11 @@ import org.apache.flink.runtime.execution.ExecutionStateTransition;
 import org.apache.flink.runtime.instance.AllocatedResource;
 import org.apache.flink.runtime.instance.AllocationID;
 import org.apache.flink.runtime.io.network.gates.GateID;
-import org.apache.flink.runtime.taskmanager.AbstractTaskResult;
+import org.apache.flink.runtime.taskmanager.TaskOperationResult;
 import org.apache.flink.runtime.taskmanager.TaskCancelResult;
 import org.apache.flink.runtime.taskmanager.TaskKillResult;
 import org.apache.flink.runtime.taskmanager.TaskSubmissionResult;
-import org.apache.flink.runtime.taskmanager.AbstractTaskResult.ReturnCode;
+import org.apache.flink.runtime.taskmanager.TaskOperationResult.ReturnCode;
 import org.apache.flink.runtime.util.AtomicEnum;
 import org.apache.flink.runtime.util.SerializableArrayList;
 import org.apache.flink.util.StringUtils;
@@ -432,8 +432,8 @@ public final class ExecutionVertex {
 
 		if (this.cancelRequested.compareAndSet(true, false)) {
 			final TaskCancelResult tsr = cancelTask();
-			if (tsr.getReturnCode() != AbstractTaskResult.ReturnCode.SUCCESS
-				&& tsr.getReturnCode() != AbstractTaskResult.ReturnCode.TASK_NOT_FOUND) {
+			if (tsr.getReturnCode() != AbstractTaskResult.TaskOperationResult.SUCCESS
+				&& tsr.getReturnCode() != AbstractTaskResult.TaskOperationResult.TASK_NOT_FOUND) {
 				LOG.error("Unable to cancel vertex " + this + ": " + tsr.getReturnCode().toString()
 					+ ((tsr.getDescription() != null) ? (" (" + tsr.getDescription() + ")") : ""));
 			}
@@ -675,7 +675,7 @@ public final class ExecutionVertex {
 
 		if (ar == null) {
 			final TaskSubmissionResult result = new TaskSubmissionResult(getID(),
-				AbstractTaskResult.ReturnCode.NO_INSTANCE);
+				AbstractTaskResult.TaskOperationResult.NO_INSTANCE);
 			result.setDescription("Assigned instance of vertex " + this.toString() + " is null!");
 			return result;
 		}
@@ -690,7 +690,7 @@ public final class ExecutionVertex {
 
 		} catch (IOException e) {
 			final TaskSubmissionResult result = new TaskSubmissionResult(getID(),
-				AbstractTaskResult.ReturnCode.IPC_ERROR);
+				AbstractTaskResult.TaskOperationResult.IPC_ERROR);
 			result.setDescription(StringUtils.stringifyException(e));
 			return result;
 		}
@@ -709,7 +709,7 @@ public final class ExecutionVertex {
 		final ExecutionState state = this.executionState.get();
 
 		if (state != ExecutionState.RUNNING) {
-			final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.ReturnCode.ILLEGAL_STATE);
+			final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.TaskOperationResult.ILLEGAL_STATE);
 			result.setDescription("Vertex " + this.toString() + " is in state " + state);
 			return result;
 		}
@@ -717,7 +717,7 @@ public final class ExecutionVertex {
 		final AllocatedResource ar = this.allocatedResource.get();
 
 		if (ar == null) {
-			final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.ReturnCode.NO_INSTANCE);
+			final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.TaskOperationResult.NO_INSTANCE);
 			result.setDescription("Assigned instance of vertex " + this.toString() + " is null!");
 			return result;
 		}
@@ -725,7 +725,7 @@ public final class ExecutionVertex {
 		try {
 			return ar.getInstance().killTask(this.vertexID);
 		} catch (IOException e) {
-			final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.ReturnCode.IPC_ERROR);
+			final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.TaskOperationResult.IPC_ERROR);
 			result.setDescription(StringUtils.stringifyException(e));
 			return result;
 		}
@@ -746,15 +746,15 @@ public final class ExecutionVertex {
 			final ExecutionState previousState = this.executionState.get();
 
 			if (previousState == ExecutionState.CANCELED) {
-				return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+				return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
 			}
 
 			if (previousState == ExecutionState.FAILED) {
-				return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+				return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
 			}
 
 			if (previousState == ExecutionState.FINISHED) {
-				return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+				return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
 			}
 
 			// The vertex has already received a cancel request
@@ -774,7 +774,7 @@ public final class ExecutionVertex {
 					continue;
 				}
 
-				return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+				return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
 			}
 
 			// Check if we had a race. If state change is accepted, send cancel request
@@ -783,20 +783,20 @@ public final class ExecutionVertex {
 				if (this.groupVertex.getStageNumber() != this.executionGraph.getIndexOfCurrentExecutionStage()) {
 					// Set to canceled directly
 					updateExecutionState(ExecutionState.CANCELED, null);
-					return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+					return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
 				}
 
 				if (previousState != ExecutionState.RUNNING && previousState != ExecutionState.FINISHING) {
 					// Set to canceled directly
 					updateExecutionState(ExecutionState.CANCELED, null);
-					return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+					return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
 				}
 
 				final AllocatedResource ar = this.allocatedResource.get();
 
 				if (ar == null) {
 					final TaskCancelResult result = new TaskCancelResult(getID(),
-						AbstractTaskResult.ReturnCode.NO_INSTANCE);
+						AbstractTaskResult.TaskOperationResult.NO_INSTANCE);
 					result.setDescription("Assigned instance of vertex " + this.toString() + " is null!");
 					return result;
 				}
@@ -806,7 +806,7 @@ public final class ExecutionVertex {
 
 				} catch (IOException e) {
 					final TaskCancelResult result = new TaskCancelResult(getID(),
-						AbstractTaskResult.ReturnCode.IPC_ERROR);
+						AbstractTaskResult.TaskOperationResult.IPC_ERROR);
 					result.setDescription(StringUtils.stringifyException(e));
 					return result;
 				}