You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2014/09/21 04:12:36 UTC
[12/63] [abbrv] Refactor job graph construction to incremental
attachment based
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphIterator.java
deleted file mode 100644
index f4efc88..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphIterator.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.Stack;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class provides an implementation of the {@link Iterator} interface which allows to
- * traverse an execution graph and visit every reachable vertex exactly once. The order
- * in which the vertices are visited corresponds to the order of their discovery in a depth first
- * search.
- * <p>
- * This class is not thread-safe.
- *
- */
-public class ExecutionGraphIterator implements Iterator<ExecutionVertex> {
-
- /**
- * The log object used for debugging.
- */
- private static final Logger LOG = LoggerFactory.getLogger(ExecutionGraphIterator.class);
-
- /**
- * The execution this iterator traverses.
- */
- private final ExecutionGraph executionGraph;
-
- /**
- * Stores whether the graph is traversed starting from the input or the output vertices.
- */
- private final boolean forward;
-
- /**
- * The stage that should be traversed by this iterator.
- */
- private final int startStage;
-
- /**
- * Stores whether the iterator is confined to the start stage or not.
- */
- private final boolean confinedToStage;
-
- /**
- * The number of visited vertices from the entry set (either input or output vertices).
- */
- private int numVisitedEntryVertices = 0;
-
- /**
- * Stack used for the depth first search.
- */
- private final Stack<TraversalEntry> traversalStack = new Stack<TraversalEntry>();
-
- /**
- * Set of already visited vertices during traversal.
- */
- private final Set<ExecutionVertex> alreadyVisited = new HashSet<ExecutionVertex>();
-
- /**
- * Auxiliary class which stores which vertices have already been visited.
- *
- */
- private static class TraversalEntry {
-
- /**
- * Execution vertex this entry has been created for.
- */
- private final ExecutionVertex executionVertex;
-
- /**
- * Next gate to traverse.
- */
- private int currentGate;
-
- /**
- * Next channel to traverse.
- */
- private int currentChannel;
-
- /**
- * Constructs a new traversal entry.
- *
- * @param executionVertex
- * the execution vertex this entry belongs to
- * @param currentGate
- * the gate index to use to visit the next vertex
- * @param currentChannel
- * the channel index to use to visit the next vertex
- */
- public TraversalEntry(final ExecutionVertex executionVertex, final int currentGate, final int currentChannel) {
- this.executionVertex = executionVertex;
- this.currentGate = currentGate;
- this.currentChannel = currentChannel;
- }
-
- /**
- * Returns the execution vertex this entry belongs to.
- *
- * @return the execution vertex this entry belongs to
- */
- public ExecutionVertex getExecutionVertex() {
- return this.executionVertex;
- }
-
- /**
- * Returns the gate index to use to visit the next vertex.
- *
- * @return the gate index to use to visit the next vertex
- */
- public int getCurrentGate() {
- return this.currentGate;
- }
-
- /**
- * Returns the channel index to use to visit the next vertex.
- *
- * @return the channel index to use to visit the next vertex
- */
- public int getCurrentChannel() {
- return this.currentChannel;
- }
-
- /**
- * Increases the channel index by one.
- */
- public void increaseCurrentChannel() {
- this.currentChannel++;
- }
-
- /**
- * Increases the gate index by one.
- */
- public void increaseCurrentGate() {
- this.currentGate++;
- }
-
- /**
- * Resets the channel index.
- */
- public void resetCurrentChannel() {
- this.currentChannel = 0;
- }
-
- }
-
- /**
- * Creates a new execution graph iterator.
- *
- * @param executionGraph
- * the execution graph that should be traversed
- * @param forward
- * <code>true</code> if the graph should be traversed in correct order, <code>false</code> to traverse it in
- * reverse order
- */
- public ExecutionGraphIterator(final ExecutionGraph executionGraph, final boolean forward) {
- this(executionGraph, forward ? 0 : (executionGraph.getNumberOfStages() - 1), false, forward);
- }
-
- /**
- * Creates a new execution graph iterator.
- *
- * @param executionGraph
- * the execution graph that should be traversed
- * @param startStage
- * the index of the stage of the graph where the traversal is supposed to begin
- * @param confinedToStage
- * <code>false</code> if the graph iterator is allowed to traverse to upper (in case of reverse order
- * traversal lower) stages, <code>true</code> otherwise.
- * @param forward
- * <code>true</code> if the graph should be traversed in correct order, <code>false</code> to traverse it in
- * reverse order
- */
- public ExecutionGraphIterator(final ExecutionGraph executionGraph, final int startStage,
- final boolean confinedToStage, final boolean forward) {
-
- this.executionGraph = executionGraph;
- this.forward = forward;
- this.startStage = startStage;
- this.confinedToStage = confinedToStage;
-
- if (startStage >= this.executionGraph.getNumberOfStages()) {
- return;
- }
-
- if (forward) {
- if (executionGraph.getNumberOfInputVertices(startStage) > 0) {
-
- final TraversalEntry te = new TraversalEntry(executionGraph.getInputVertex(startStage, 0), 0, 0);
- this.traversalStack.push(te);
- this.alreadyVisited.add(te.getExecutionVertex());
-
- }
- } else {
- if (executionGraph.getNumberOfOutputVertices(startStage) > 0) {
-
- final TraversalEntry te = new TraversalEntry(executionGraph.getOutputVertex(startStage, 0), 0, 0);
- this.traversalStack.push(te);
- this.alreadyVisited.add(te.getExecutionVertex());
- }
-
- }
- }
-
- /**
- * Creates a new execution graph iterator. This constructor can be used to
- * traverse only specific parts of the graph starting at <code>startVertex</code>.
- * The iterator will not switch to the next input/output vertex of an output/input vertex
- * has been reached.
- *
- * @param executionGraph
- * the execution graph that should be traversed
- * @param startVertex
- * the vertex to start the traversal from
- * @param forward
- * <code>true</code> if the graph should be traversed in correct order, <code>false</code> to reverse it in
- * reverse order
- */
- public ExecutionGraphIterator(final ExecutionGraph executionGraph, final ExecutionVertex startVertex,
- final boolean forward) {
-
- this.executionGraph = executionGraph;
- this.forward = forward;
- this.numVisitedEntryVertices = -1;
- this.startStage = 0;
- this.confinedToStage = false;
-
- final TraversalEntry te = new TraversalEntry(startVertex, 0, 0);
- this.traversalStack.push(te);
- this.alreadyVisited.add(startVertex);
- }
-
-
- @Override
- public boolean hasNext() {
-
- if (this.traversalStack.isEmpty()) {
-
- if (this.numVisitedEntryVertices < 0) {
- // User chose a specific starting vertex
- return false;
- }
-
- ++this.numVisitedEntryVertices;
-
- if (this.forward) {
- if (this.executionGraph.getNumberOfInputVertices(this.startStage) <= this.numVisitedEntryVertices) {
- return false;
- }
- } else {
- if (this.executionGraph.getNumberOfOutputVertices(this.startStage) <= this.numVisitedEntryVertices) {
- return false;
- }
- }
- }
-
- return true;
- }
-
-
- @Override
- public ExecutionVertex next() {
-
- if (this.traversalStack.isEmpty()) {
-
- if (this.numVisitedEntryVertices < 0) {
- // User chose a specific entry vertex
- return null;
- }
-
- TraversalEntry newentry;
-
- if (this.forward) {
- newentry = new TraversalEntry(this.executionGraph.getInputVertex(this.startStage,
- this.numVisitedEntryVertices),
- 0, 0);
- } else {
- newentry = new TraversalEntry(this.executionGraph.getOutputVertex(this.startStage,
- this.numVisitedEntryVertices),
- 0, 0);
- }
-
- this.traversalStack.push(newentry);
- this.alreadyVisited.add(newentry.getExecutionVertex());
- }
-
- final ExecutionVertex returnVertex = this.traversalStack.peek().getExecutionVertex();
-
- // Propose vertex to be visited next
- do {
-
- final TraversalEntry te = this.traversalStack.peek();
-
- // Check if we can traverse deeper into the graph
- final ExecutionVertex candidateVertex = getCandidateVertex(te, forward);
- if (candidateVertex == null) {
- // Pop it from the stack
- this.traversalStack.pop();
- } else {
- // Create new entry and put it on the stack
- final TraversalEntry newte = new TraversalEntry(candidateVertex, 0, 0);
- this.traversalStack.push(newte);
- this.alreadyVisited.add(candidateVertex);
- break;
- }
-
- } while (!this.traversalStack.isEmpty());
-
- return returnVertex;
- }
-
- /**
- * Returns a candidate vertex which could potentially be visited next because it is reachable from the
- * currently considered vertex.
- *
- * @param te
- * the traversal entry for the current source vertex
- * @param forward
- * <code>true</code> if the graph should be traversed in correct order, <code>false</code> to traverse it in
- * reverse order
- * @return a candidate vertex which could potentially be visited next
- */
- private ExecutionVertex getCandidateVertex(final TraversalEntry te, final boolean forward) {
-
- if (forward) {
-
- while (true) {
-
- if (this.confinedToStage && te.getCurrentChannel() == 0) {
- while (currentGateLeadsToOtherStage(te, this.forward)) {
- te.increaseCurrentGate();
- }
- }
-
- // No more outgoing edges to consider
- if (te.getCurrentGate() >= te.getExecutionVertex().getNumberOfOutputGates()) {
- break;
- }
-
- if (te.getCurrentChannel() >= te.getExecutionVertex().getOutputGate(te.getCurrentGate())
- .getNumberOfEdges()) {
- te.increaseCurrentGate();
- te.resetCurrentChannel();
- } else {
- final ExecutionEdge outputChannel = te.getExecutionVertex().getOutputGate(te.getCurrentGate())
- .getEdge(te.getCurrentChannel());
- final ExecutionVertex tmp = outputChannel.getInputGate().getVertex();
- if (tmp == null) {
- LOG.error("Inconsistency in vertex map found (forward)!");
- }
- te.increaseCurrentChannel();
- if (!this.alreadyVisited.contains(tmp)) {
- return tmp;
- }
- }
- }
- } else {
-
- while (true) {
-
- // No more outgoing edges to consider
- if (te.getCurrentGate() >= te.getExecutionVertex().getNumberOfInputGates()) {
- break;
- }
-
- if (te.getCurrentChannel() >= te.getExecutionVertex().getInputGate(te.getCurrentGate())
- .getNumberOfEdges()) {
- te.increaseCurrentGate();
- te.resetCurrentChannel();
- } else {
- final ExecutionEdge inputChannel = te.getExecutionVertex().getInputGate(te.getCurrentGate()).getEdge(te.getCurrentChannel());
- final ExecutionVertex tmp = inputChannel.getOutputGate().getVertex();
- if (tmp == null) {
- LOG.error("Inconsistency in vertex map found (backward)!");
- }
- te.increaseCurrentChannel();
- if (!this.alreadyVisited.contains(tmp)) {
- return tmp;
- }
- }
- }
- }
-
- return null;
- }
-
- private boolean currentGateLeadsToOtherStage(final TraversalEntry te, final boolean forward) {
-
- final ExecutionGroupVertex groupVertex = te.getExecutionVertex().getGroupVertex();
-
- if (forward) {
-
- if (te.getCurrentGate() >= groupVertex.getNumberOfForwardLinks()) {
- return false;
- }
-
- final ExecutionGroupEdge edge = groupVertex.getForwardEdge(te.getCurrentGate());
- if (edge.getTargetVertex().getStageNumber() == groupVertex.getStageNumber()) {
- return false;
- }
-
- } else {
- if (te.getCurrentGate() >= groupVertex.getNumberOfBackwardLinks()) {
- return false;
- }
-
- final ExecutionGroupEdge edge = groupVertex.getBackwardEdge(te.getCurrentGate());
- if (edge.getSourceVertex().getStageNumber() == groupVertex.getStageNumber()) {
- return false;
- }
- }
-
- return true;
- }
-
-
- @Override
- public void remove() {
-
- throw new UnsupportedOperationException("The method remove is not implemented for this type of iterator");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGroupVertexIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGroupVertexIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGroupVertexIterator.java
deleted file mode 100644
index 1095925..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGroupVertexIterator.java
+++ /dev/null
@@ -1,331 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.Stack;
-
-/**
- * This class provides an implementation of the {@link Iterator} interface which allows to
- * traverse an execution graph and visit every reachable group vertex exactly once. The order
- * in which the group vertices are visited corresponds to the order of their discovery in a depth first
- * search.
- * <p>
- * This class is not thread-safe.
- *
- */
-public class ExecutionGroupVertexIterator implements Iterator<ExecutionGroupVertex> {
-
- /**
- * Stores whether the group graph is traversed starting from the input or the output vertices.
- */
- private final boolean forward;
-
- /**
- * The stage to traverse, -1 to traverse all stages of the graph.
- */
- private final int stage;
-
- /**
- * List of entry vertices for the traversal (either input or output vertices).
- */
- private final List<ExecutionGroupVertex> entryVertices = new ArrayList<ExecutionGroupVertex>();
-
- /**
- * Number of already visited entry vertices.
- */
- private int numVisitedEntryVertices = 0;
-
- /**
- * Stack used for the traversal.
- */
- private final Stack<TraversalEntry> traversalStack = new Stack<TraversalEntry>();
-
- /**
- * Set storing the vertices already visited during traversal.
- */
- private final Set<ExecutionGroupVertex> alreadyVisited = new HashSet<ExecutionGroupVertex>();
-
- /**
- * Auxiliary class which stores which vertices have already been visited.
- *
- */
- private static class TraversalEntry {
-
- /**
- * The group vertex this entry is created for.
- */
- private final ExecutionGroupVertex groupVertex;
-
- /**
- * The current outgoing link of the group vertex.
- */
- private int currentLink = 0;
-
- /**
- * Constructs a new traversal entry.
- *
- * @param groupVertex
- * the group vertex this traversal entry belongs to
- * @param currentLink
- * the link index to use to visit the next group vertex
- */
- public TraversalEntry(final ExecutionGroupVertex groupVertex, final int currentLink) {
- this.groupVertex = groupVertex;
- this.currentLink = currentLink;
- }
-
- /**
- * Returns the group vertex this traversal entry belongs to.
- *
- * @return the group vertex this traversal entry belongs to
- */
- public ExecutionGroupVertex getGroupVertex() {
- return this.groupVertex;
- }
-
- /**
- * Returns the link index to use to visit the next group vertex.
- *
- * @return the link index to use to visit the next group vertex
- */
- public int getCurrentLink() {
- return this.currentLink;
- }
-
- /**
- * Increases the link index by one.
- */
- public void increaseCurrentLink() {
- this.currentLink++;
- }
- }
-
- /**
- * Creates a new execution group vertex iterator.
- *
- * @param executionGraph
- * the execution graph that should be traversed
- * @param forward
- * <code>true</code> if the graph should be traversed in correct order, <code>false</code> to reverse it in
- * reverse order
- * @param stage
- * the number of the stage whose vertices should be traversed or -1 if all stages should be included in the
- * traversal
- */
- public ExecutionGroupVertexIterator(final ExecutionGraph executionGraph, final boolean forward, final int stage) {
-
- this.forward = forward;
- this.stage = stage;
-
- // Collect start vertices
- if (stage < 0) {
-
- for (int i = 0; i < executionGraph.getNumberOfStages(); i++) {
- collectStartVertices(executionGraph.getStage(i));
- }
- } else {
- if (stage < executionGraph.getNumberOfStages()) {
- collectStartVertices(executionGraph.getStage(stage));
- }
- }
-
- if (this.entryVertices.size() > 0) {
- final TraversalEntry te = new TraversalEntry(this.entryVertices.get(0), 0);
- this.traversalStack.push(te);
- this.alreadyVisited.add(te.getGroupVertex());
- }
- }
-
- /**
- * Collects all input group vertices (i.e. vertices with no incoming link or incoming links from other stages) in
- * the given stage and adds them to an internal list.
- *
- * @param stage
- * the number of the stage whose input vertices should be collected
- */
- private void collectStartVertices(final ExecutionStage stage) {
-
- for (int i = 0; i < stage.getNumberOfStageMembers(); i++) {
-
- final ExecutionGroupVertex groupVertex = stage.getStageMember(i);
-
- if (forward) {
- if ((groupVertex.getNumberOfBackwardLinks() == 0)
- || ((this.stage >= 0) && allConnectionsFromOtherStage(groupVertex, true))) {
- this.entryVertices.add(groupVertex);
- }
- } else {
- if ((groupVertex.getNumberOfForwardLinks() == 0)
- || ((this.stage >= 0) && allConnectionsFromOtherStage(groupVertex, false))) {
- this.entryVertices.add(groupVertex);
- }
- }
- }
- }
-
- /**
- * Checks if for the given group vertex all incoming (if forward is <code>true</code>) or outgoing links (if forward
- * is <code>false</code>) come from
- * other stages than the one the given vertex is in.
- *
- * @param groupVertex
- * the group vertex to check for
- * @param forward
- * <code>true</code> if incoming links should be considered, <code>false</code> for outgoing links
- * @return <code>true</code> if all incoming or outgoing links (depends on the forward switch) come from other
- * stages, <code>false</code> otherwise
- */
- private boolean allConnectionsFromOtherStage(final ExecutionGroupVertex groupVertex, final boolean forward) {
-
- if (forward) {
- for (int i = 0; i < groupVertex.getNumberOfBackwardLinks(); i++) {
- if (this.stage == groupVertex.getBackwardEdge(i).getSourceVertex().getStageNumber()) {
- return false;
- }
- }
- } else {
- for (int i = 0; i < groupVertex.getNumberOfForwardLinks(); i++) {
- if (this.stage == groupVertex.getForwardEdge(i).getTargetVertex().getStageNumber()) {
- return false;
- }
- }
- }
-
- return true;
- }
-
-
- @Override
- public boolean hasNext() {
-
- if (this.traversalStack.isEmpty()) {
-
- ++this.numVisitedEntryVertices;
- if (this.entryVertices.size() <= this.numVisitedEntryVertices) {
- return false;
- }
- }
-
- return true;
-
- }
-
-
- @Override
- public ExecutionGroupVertex next() {
-
- if (this.traversalStack.isEmpty()) {
-
- final TraversalEntry newentry = new TraversalEntry(this.entryVertices.get(this.numVisitedEntryVertices), 0);
- this.traversalStack.push(newentry);
- this.alreadyVisited.add(newentry.getGroupVertex());
- }
-
- final ExecutionGroupVertex returnVertex = this.traversalStack.peek().getGroupVertex();
-
- // Propose vertex to be visited next
- do {
-
- final TraversalEntry te = this.traversalStack.peek();
-
- // Check if we can traverse deeper into the graph
- final ExecutionGroupVertex candidateVertex = getCandidateVertex(te, forward);
- if (candidateVertex == null) {
- // Pop it from the stack
- traversalStack.pop();
- } else {
- // Create new entry and put it on the stack
- final TraversalEntry newte = new TraversalEntry(candidateVertex, 0);
- this.traversalStack.push(newte);
- this.alreadyVisited.add(newte.getGroupVertex());
- break;
- }
-
- } while (!this.traversalStack.isEmpty());
-
- return returnVertex;
-
- }
-
- /**
- * Returns a candidate group vertex which could potentially be visited next because it is reachable from the
- * currently considered group vertex.
- *
- * @param te
- * the traversal entry for the current source group vertex
- * @param forward
- * <code>true</code> if the graph should be traversed in correct order, <code>false</code> to traverse it in
- * reverse order
- * @return a candidate group vertex which could potentially be visited next
- */
- private ExecutionGroupVertex getCandidateVertex(final TraversalEntry te, final boolean forward) {
-
- while (true) {
-
- if (forward) {
- // No more outgoing links to consider
- if (te.getCurrentLink() >= te.getGroupVertex().getNumberOfForwardLinks()) {
- break;
- }
- } else {
- // No more outgoing links to consider
- if (te.getCurrentLink() >= te.getGroupVertex().getNumberOfBackwardLinks()) {
- break;
- }
- }
-
- ExecutionGroupVertex tmp = null;
- if (forward) {
- tmp = te.getGroupVertex().getForwardEdge(te.getCurrentLink()).getTargetVertex();
- } else {
- tmp = te.getGroupVertex().getBackwardEdge(te.getCurrentLink()).getSourceVertex();
- }
-
- // Increase the current link index by one
- te.increaseCurrentLink();
-
- // If stage >= 0, tmp must be in the same stage as te.getGroupVertex()
- if (this.stage >= 0) {
- if (tmp.getStageNumber() != this.stage) {
- continue;
- }
- }
-
- if (!this.alreadyVisited.contains(tmp)) {
- return tmp;
- }
- }
-
- return null;
- }
-
-
- @Override
- public void remove() {
- // According to the documentation this method is optional and does not need to be implemented
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
new file mode 100644
index 0000000..72f03b4
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.flink.core.io.InputSplit;
+import org.apache.flink.core.io.InputSplitAssigner;
+import org.apache.flink.core.io.InputSplitSource;
+import org.apache.flink.runtime.JobException;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSet;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
+import org.apache.flink.runtime.jobgraph.JobEdge;
+import org.apache.flink.runtime.jobgraph.JobID;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+
+
+public class ExecutionJobVertex {
+
+ /** Use the same log for all ExecutionGraph classes */
+ private static final Log LOG = ExecutionGraph.LOG;
+
+
+ private final ExecutionGraph graph;
+
+ private final AbstractJobVertex jobVertex;
+
+ private final ExecutionVertex2[] taskVertices;
+
+ private final IntermediateResult[] producedDataSets;
+
+ private final InputSplitAssigner splitAssigner;
+
+ private final int parallelism;
+
+
+ private final AtomicInteger numRunningTasks = new AtomicInteger(0);
+
+ private final AtomicInteger numFinishedTasks = new AtomicInteger(0);
+
+ private final AtomicInteger numCancelledOrFailedTasks = new AtomicInteger(0);
+
+
+ private SlotSharingGroup slotSharingGroup;
+
+
+ public ExecutionJobVertex(ExecutionGraph graph, AbstractJobVertex jobVertex, int defaultParallelism) throws JobException {
+ if (graph == null || jobVertex == null) {
+ throw new NullPointerException();
+ }
+
+ this.graph = graph;
+ this.jobVertex = jobVertex;
+
+ int vertexParallelism = jobVertex.getParallelism();
+ int numTaskVertices = vertexParallelism > 0 ? vertexParallelism : defaultParallelism;
+
+ this.parallelism = numTaskVertices;
+ this.taskVertices = new ExecutionVertex2[numTaskVertices];
+
+ // create the intermediate results
+ this.producedDataSets = new IntermediateResult[jobVertex.getNumberOfProducedIntermediateDataSets()];
+ for (int i = 0; i < jobVertex.getProducedDataSets().size(); i++) {
+ IntermediateDataSet set = jobVertex.getProducedDataSets().get(i);
+ this.producedDataSets[i] = new IntermediateResult(set.getId(), this, numTaskVertices);
+ }
+
+ // create all task vertices
+ for (int i = 0; i < numTaskVertices; i++) {
+ ExecutionVertex2 vertex = new ExecutionVertex2(this, i, this.producedDataSets);
+ this.taskVertices[i] = vertex;
+ }
+
+ // take the sharing group
+ this.slotSharingGroup = jobVertex.getSlotSharingGroup();
+
+ // set up the input splits, if the vertex has any
+ try {
+ @SuppressWarnings("unchecked")
+ InputSplitSource<InputSplit> splitSource = (InputSplitSource<InputSplit>) jobVertex.getInputSplitSource();
+ if (splitSource != null) {
+ InputSplit[] splits = splitSource.createInputSplits(numTaskVertices);
+ this.splitAssigner = splitSource.getInputSplitAssigner(splits);
+ } else {
+ this.splitAssigner = null;
+ }
+ }
+ catch (Throwable t) {
+ throw new JobException("Creating the input splits caused an error: " + t.getMessage(), t);
+ }
+ }
+
+ public ExecutionGraph getGraph() {
+ return this.graph;
+ }
+
+ public AbstractJobVertex getJobVertex() {
+ return this.jobVertex;
+ }
+
+ public int getParallelism() {
+ return this.parallelism;
+ }
+
+ public JobID getJobId() {
+ return this.graph.getJobID();
+ }
+
+ public JobVertexID getJobVertexId() {
+ return this.jobVertex.getID();
+ }
+
+ public ExecutionVertex2[] getTaskVertices() {
+ return taskVertices;
+ }
+
+ public IntermediateResult[] getProducedDataSets() {
+ return producedDataSets;
+ }
+
+ public InputSplitAssigner getSplitAssigner() {
+ return splitAssigner;
+ }
+
+ public void setSlotSharingGroup(SlotSharingGroup slotSharingGroup) {
+ this.slotSharingGroup = slotSharingGroup;
+ }
+
+ public SlotSharingGroup getSlotSharingGroup() {
+ return slotSharingGroup;
+ }
+
+ //---------------------------------------------------------------------------------------------
+
+ public void connectToPredecessors(Map<IntermediateDataSetID, IntermediateResult> intermediateDataSets) throws JobException {
+
+ List<JobEdge> inputs = jobVertex.getInputs();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Connecting ExecutionJobVertex %s (%s) to %d predecessors.", jobVertex.getID(), jobVertex.getName(), inputs.size()));
+ }
+
+ for (int num = 0; num < inputs.size(); num++) {
+ JobEdge edge = inputs.get(num);
+
+ if (LOG.isDebugEnabled()) {
+ if (edge.getSource() == null) {
+ LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s.",
+ num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId()));
+ } else {
+ LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).",
+ num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName()));
+ }
+ }
+
+ // fetch the intermediate result via ID. if it does not exist, then it either has not been created, or the order
+ // in which this method is called for the job vertices is not a topological order
+ IntermediateResult ires = intermediateDataSets.get(edge.getSourceId());
+ if (ires == null) {
+ throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID "
+ + edge.getSourceId());
+ }
+
+ int consumerIndex = ires.registerConsumer();
+
+ for (int i = 0; i < parallelism; i++) {
+ ExecutionVertex2 ev = taskVertices[i];
+ ev.connectSource(num, ires, edge, consumerIndex);
+ }
+
+
+ }
+ }
+
+ //---------------------------------------------------------------------------------------------
+ // State, deployment, and recovery logic
+ //---------------------------------------------------------------------------------------------
+
+ void vertexSwitchedToRunning(int subtask) {
+ this.numRunningTasks.incrementAndGet();
+ }
+
+ void vertexFailed(int subtask) {
+
+ }
+
+ void vertexCancelled(int subtask) {
+
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Miscellaneous
+ // --------------------------------------------------------------------------------------------
+
+ public void execute(Runnable action) {
+ this.graph.execute(action);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionPipeline.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionPipeline.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionPipeline.java
deleted file mode 100644
index d998f60..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionPipeline.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.util.Iterator;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import org.apache.flink.runtime.execution.ExecutionState;
-import org.apache.flink.runtime.instance.AllocatedResource;
-import org.apache.flink.runtime.util.UnmodifiableIterator;
-
-/**
- * An execution pipeline is a scheduling abstraction which defines a set of {@link ExecutionVertex} objects which must
- * be deployed together. An {@link ExecutionVertex} always belongs to exactly one execution pipeline.
- * <p>
- * This class is thread-safe.
- *
- */
-public final class ExecutionPipeline {
-
- /**
- * The set of vertices belonging to this execution pipeline.
- */
- private final CopyOnWriteArrayList<ExecutionVertex> vertices = new CopyOnWriteArrayList<ExecutionVertex>();
-
- /**
- * Adds the given {@link ExecutionVertex} to this pipeline.
- *
- * @param vertex
- * the vertex to be added to this pipeline
- */
- void addToPipeline(final ExecutionVertex vertex) {
-
- if (!this.vertices.addIfAbsent(vertex)) {
- throw new IllegalStateException("Vertex " + vertex + " has already been added to pipeline " + this);
- }
- }
-
- /**
- * Removes the given {@link ExecutionVertex} from this pipeline.
- *
- * @param vertex
- * the vertex to be removed from this pipeline.
- */
- void removeFromPipeline(final ExecutionVertex vertex) {
-
- if (!this.vertices.remove(vertex)) {
- throw new IllegalStateException("Vertex " + vertex + " was not part of the pipeline " + this);
- }
- }
-
- /**
- * Returns an {@link Iterator} to the vertices which are part of this pipeline.
- *
- * @return an iterator to the vertices of this pipeline
- */
- public Iterator<ExecutionVertex> iterator() {
-
- return new UnmodifiableIterator<ExecutionVertex>(this.vertices.iterator());
- }
-
- /**
- * Checks if the pipeline is currently finishing its execution, i.e. all vertices contained in the pipeline have
- * switched to the <code>FINISHING</code> or <code>FINISHED</code> state.
- *
- * @return <code>true</code> if the pipeline is currently finishing, <code>false</code> otherwise
- */
- public boolean isFinishing() {
-
- final Iterator<ExecutionVertex> it = this.vertices.iterator();
- while (it.hasNext()) {
-
- final ExecutionState state = it.next().getExecutionState();
- if (state != ExecutionState.FINISHING && state != ExecutionState.FINISHED) {
- return false;
- }
- }
-
- return true;
- }
-
- /**
- * Sets the given allocated resource for all vertices included in this pipeline.
- *
- * @param resource
- * the allocated resource to set for all vertices included in this pipeline
- */
- public void setAllocatedResource(final AllocatedResource resource) {
-
- final Iterator<ExecutionVertex> it = this.vertices.iterator();
- while (it.hasNext()) {
- final ExecutionVertex vertex = it.next();
- vertex.setAllocatedResource(resource);
- }
- }
-
- /**
- * Updates the execution state for all vertices included in this pipeline.
- *
- * @param executionState
- * the execution state to set for all vertices included in this pipeline
- */
- public void updateExecutionState(final ExecutionState executionState) {
-
- final Iterator<ExecutionVertex> it = this.vertices.iterator();
- while (it.hasNext()) {
- final ExecutionVertex vertex = it.next();
- vertex.updateExecutionState(executionState);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionSignature.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionSignature.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionSignature.java
deleted file mode 100644
index 40f2468..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionSignature.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.io.IOException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager;
-import org.apache.flink.runtime.jobgraph.JobID;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-
-/**
- * An execution signature allows to uniquely identify a job vertex. The signature involves the name of the
- * class to be invoked at runtime as well as a cryptographic hash of all the JAR files which are required to
- * instantiate the class and run it. The execution is the basis for feedback learning as it enables the profiler
- * to recognize particular parts of a job. Execution signature objects are immutable and, consequently, thread-safe.
- * <p>
- * This class is thread-safe.
- *
- */
-public final class ExecutionSignature {
-
- /**
- * The log object used for debugging.
- */
- private static final Logger LOG = LoggerFactory.getLogger(ExecutionSignature.class);
-
- /**
- * The name of the hashing algorithm to be used.
- */
- private static final String HASHINGALGORITHM = "SHA-1";
-
- /**
- * The message digest object used to calculate the signature.
- */
- private static MessageDigest messageDigest = null;
-
- /**
- * The buffer storing the signature.
- */
- private final byte[] signature;
-
- /**
- * Constructs a new execution signature object and passes the signature buffer.
- *
- * @param signature
- * the byte buffer containing the signature.
- */
- private ExecutionSignature(final byte[] signature) {
- this.signature = signature;
- }
-
- /**
- * Calculates the execution signature from the given class name and job ID.
- *
- * @param invokableClass
- * the name of the class to contain the task program
- * @param jobID
- * the ID of the job
- * @return the cryptographic signature of this vertex
- */
- public static synchronized ExecutionSignature createSignature(
- final Class<? extends AbstractInvokable> invokableClass, final JobID jobID) {
-
- // First, try to load message digest algorithm, if necessary
- if (messageDigest == null) {
- try {
- messageDigest = MessageDigest.getInstance(HASHINGALGORITHM);
- } catch (NoSuchAlgorithmException e) {
- LOG.error("Unable to load message digest algorithm " + HASHINGALGORITHM);
- return null;
- }
- }
-
- // Reset digest buffer and add the name of the invokable class to the message digest buffer
- messageDigest.reset();
- messageDigest.update(invokableClass.getName().getBytes());
-
- String[] requiredJarFiles;
- // Next, retrieve the JAR-files associated with this job
- try {
- requiredJarFiles = LibraryCacheManager.getRequiredJarFiles(jobID);
- } catch (IOException ioe) {
- // Output an error message and return
- LOG.error("Cannot access library cache manager for job ID " + jobID);
- return null;
- }
-
- // Now, sort the list of JAR-files in order to always calculate the signature in the same manner
- Arrays.sort(requiredJarFiles);
-
- // Finally, add the names of the JAR-files to the hash calculation
- for (int i = 0; i < requiredJarFiles.length; i++) {
- messageDigest.update(requiredJarFiles[i].getBytes());
- }
-
- return new ExecutionSignature(messageDigest.digest());
- }
-
-
- @Override
- public boolean equals(final Object obj) {
-
- if (obj instanceof ExecutionSignature) {
-
- final ExecutionSignature executionSignature = (ExecutionSignature) obj;
- return Arrays.equals(this.signature, executionSignature.signature);
- }
-
- return false;
- }
-
-
- @Override
- public int hashCode() {
-
- int hashCode = 0;
-
- for (int i = 0; i < this.signature.length; i++) {
- hashCode += this.signature[i];
- }
-
- return hashCode;
- }
-
-
- @Override
- public String toString() {
-
- final StringBuffer stringBuffer = new StringBuffer();
- for (int i = 0; i < this.signature.length; i++) {
- stringBuffer.append(Integer.toHexString(0xFF & this.signature[i]));
- }
-
- return stringBuffer.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStage.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStage.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStage.java
deleted file mode 100644
index 54e7b12..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStage.java
+++ /dev/null
@@ -1,411 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import org.apache.flink.runtime.instance.Instance;
-import org.apache.flink.runtime.io.network.channels.ChannelType;
-
-/**
- * An execution stage contains all execution group vertices (and as a result all execution vertices) which
- * must run at the same time. The execution of a job progresses in terms of stages, i.e. the next stage of a
- * job can only start to execute if the execution of its preceding stage is complete.
- * <p>
- * This class is thread-safe.
- */
-public final class ExecutionStage {
-
- /**
- * The execution graph that this stage belongs to.
- */
- private final ExecutionGraph executionGraph;
-
- /**
- * List of group vertices which are assigned to this stage.
- */
- private final CopyOnWriteArrayList<ExecutionGroupVertex> stageMembers = new CopyOnWriteArrayList<ExecutionGroupVertex>();
-
- /**
- * Number of the stage.
- */
- private volatile int stageNum = -1;
-
- /**
- * Constructs a new execution stage and assigns the given stage number to it.
- *
- * @param executionGraph
- * the executionGraph that this stage belongs to
- * @param stageNum
- * the number of this execution stage
- */
- public ExecutionStage(final ExecutionGraph executionGraph, final int stageNum) {
- this.executionGraph = executionGraph;
- this.stageNum = stageNum;
- }
-
- /**
- * Sets the number of this execution stage.
- *
- * @param stageNum
- * the new number of this execution stage
- */
- public void setStageNumber(final int stageNum) {
- this.stageNum = stageNum;
- }
-
- /**
- * Returns the number of this execution stage.
- *
- * @return the number of this execution stage
- */
- public int getStageNumber() {
-
- return this.stageNum;
- }
-
- /**
- * Adds a new execution group vertex to this stage if it is not already included.
- *
- * @param groupVertex
- * the new execution group vertex to include
- */
- public void addStageMember(final ExecutionGroupVertex groupVertex) {
-
- if (this.stageMembers.addIfAbsent(groupVertex)) {
- groupVertex.setExecutionStage(this);
- }
- }
-
- /**
- * Removes the specified group vertex from the execution stage.
- *
- * @param groupVertex
- * the group vertex to remove from the stage
- */
- public void removeStageMember(ExecutionGroupVertex groupVertex) {
-
- this.stageMembers.remove(groupVertex);
- }
-
- /**
- * Returns the number of group vertices this execution stage includes.
- *
- * @return the number of group vertices this execution stage includes
- */
- public int getNumberOfStageMembers() {
-
- return this.stageMembers.size();
- }
-
- /**
- * Returns the stage member internally stored at index <code>index</code>.
- *
- * @param index
- * the index of the group vertex to return
- * @return the stage member internally stored at the specified index or <code>null</code> if no group vertex exists
- * with such an index
- */
- public ExecutionGroupVertex getStageMember(final int index) {
-
- try {
- return this.stageMembers.get(index);
- } catch (ArrayIndexOutOfBoundsException e) {
- return null;
- }
- }
-
- /**
- * Returns the number of input execution vertices in this stage, i.e. the number
- * of execution vertices which are connected to vertices in a lower stage
- * or have no input channels.
- *
- * @return the number of input vertices in this stage
- */
- public int getNumberOfInputExecutionVertices() {
-
- int retVal = 0;
-
- final Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
- while (it.hasNext()) {
-
- final ExecutionGroupVertex groupVertex = it.next();
- if (groupVertex.isInputVertex()) {
- retVal += groupVertex.getCurrentNumberOfGroupMembers();
- }
- }
-
- return retVal;
- }
-
- /**
- * Returns the number of output execution vertices in this stage, i.e. the number
- * of execution vertices which are connected to vertices in a higher stage
- * or have no output channels.
- *
- * @return the number of output vertices in this stage
- */
- public int getNumberOfOutputExecutionVertices() {
-
- int retVal = 0;
-
- final Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
- while (it.hasNext()) {
-
- final ExecutionGroupVertex groupVertex = it.next();
- if (groupVertex.isOutputVertex()) {
- retVal += groupVertex.getCurrentNumberOfGroupMembers();
- }
- }
-
- return retVal;
- }
-
- /**
- * Returns the output execution vertex with the given index or <code>null</code> if no such vertex exists.
- *
- * @param index
- * the index of the vertex to be selected.
- * @return the output execution vertex with the given index or <code>null</code> if no such vertex exists
- */
- public ExecutionVertex getInputExecutionVertex(int index) {
-
- final Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
- while (it.hasNext()) {
-
- final ExecutionGroupVertex groupVertex = it.next();
- if (groupVertex.isInputVertex()) {
- final int numberOfMembers = groupVertex.getCurrentNumberOfGroupMembers();
- if (index >= numberOfMembers) {
- index -= numberOfMembers;
- } else {
- return groupVertex.getGroupMember(index);
- }
- }
- }
-
- return null;
- }
-
- /**
- * Returns the input execution vertex with the given index or <code>null</code> if no such vertex exists.
- *
- * @param index
- * the index of the vertex to be selected.
- * @return the input execution vertex with the given index or <code>null</code> if no such vertex exists
- */
- public ExecutionVertex getOutputExecutionVertex(int index) {
-
- final Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
- while (it.hasNext()) {
-
- final ExecutionGroupVertex groupVertex = it.next();
- if (groupVertex.isOutputVertex()) {
- final int numberOfMembers = groupVertex.getCurrentNumberOfGroupMembers();
- if (index >= numberOfMembers) {
- index -= numberOfMembers;
- } else {
- return groupVertex.getGroupMember(index);
- }
- }
- }
-
- return null;
- }
-
- /**
- * Returns the execution graph that this stage belongs to.
- *
- * @return the execution graph that this stage belongs to
- */
- public ExecutionGraph getExecutionGraph() {
-
- return this.executionGraph;
- }
-
- /**
- * Reconstructs the execution pipelines for this execution stage.
- */
- void reconstructExecutionPipelines() {
-
- Iterator<ExecutionGroupVertex> it = this.stageMembers.iterator();
- final Set<ExecutionVertex> alreadyVisited = new HashSet<ExecutionVertex>();
-
- while (it.hasNext()) {
-
- final ExecutionGroupVertex groupVertex = it.next();
-
- // We only look at input vertices first
- if (!groupVertex.isInputVertex()) {
- continue;
- }
-
- final Iterator<ExecutionVertex> vertexIt = groupVertex.iterator();
- while (vertexIt.hasNext()) {
-
- final ExecutionVertex vertex = vertexIt.next();
- reconstructExecutionPipeline(vertex, true, alreadyVisited);
- }
- }
-
- it = this.stageMembers.iterator();
- alreadyVisited.clear();
-
- while (it.hasNext()) {
-
- final ExecutionGroupVertex groupVertex = it.next();
-
- // We only look at input vertices first
- if (!groupVertex.isOutputVertex()) {
- continue;
- }
-
- final Iterator<ExecutionVertex> vertexIt = groupVertex.iterator();
- while (vertexIt.hasNext()) {
-
- final ExecutionVertex vertex = vertexIt.next();
- reconstructExecutionPipeline(vertex, false, alreadyVisited);
- }
- }
- }
-
- /**
- * Reconstructs the execution pipeline starting at the given vertex by conducting a depth-first search.
- *
- * @param vertex
- * the vertex to start the depth-first search from
- * @param forward
- * <code>true</code> to traverse the graph according to the original direction of the edges or
- * <code>false</code> for the opposite direction
- * @param alreadyVisited
- * a set of vertices that have already been visited in the depth-first search
- */
- private void reconstructExecutionPipeline(final ExecutionVertex vertex, final boolean forward,
- final Set<ExecutionVertex> alreadyVisited) {
-
- ExecutionPipeline pipeline = vertex.getExecutionPipeline();
- if (pipeline == null) {
- pipeline = new ExecutionPipeline();
- vertex.setExecutionPipeline(pipeline);
- }
-
- alreadyVisited.add(vertex);
-
- if (forward) {
-
- final int numberOfOutputGates = vertex.getNumberOfOutputGates();
- for (int i = 0; i < numberOfOutputGates; ++i) {
-
- final ExecutionGate outputGate = vertex.getOutputGate(i);
- final ChannelType channelType = outputGate.getChannelType();
- final int numberOfOutputChannels = outputGate.getNumberOfEdges();
- for (int j = 0; j < numberOfOutputChannels; ++j) {
-
- final ExecutionEdge outputChannel = outputGate.getEdge(j);
- final ExecutionVertex connectedVertex = outputChannel.getInputGate().getVertex();
-
- boolean recurse = false;
-
- if (!alreadyVisited.contains(connectedVertex)) {
- recurse = true;
- }
-
- if (channelType == ChannelType.IN_MEMORY
- && !pipeline.equals(connectedVertex.getExecutionPipeline())) {
-
- connectedVertex.setExecutionPipeline(pipeline);
- recurse = true;
- }
-
- if (recurse) {
- reconstructExecutionPipeline(connectedVertex, true, alreadyVisited);
- }
- }
- }
- } else {
-
- final int numberOfInputGates = vertex.getNumberOfInputGates();
- for (int i = 0; i < numberOfInputGates; ++i) {
-
- final ExecutionGate inputGate = vertex.getInputGate(i);
- final ChannelType channelType = inputGate.getChannelType();
- final int numberOfInputChannels = inputGate.getNumberOfEdges();
- for (int j = 0; j < numberOfInputChannels; ++j) {
-
- final ExecutionEdge inputChannel = inputGate.getEdge(j);
- final ExecutionVertex connectedVertex = inputChannel.getOutputGate().getVertex();
-
- boolean recurse = false;
-
- if (!alreadyVisited.contains(connectedVertex)) {
- recurse = true;
- }
-
- if (channelType == ChannelType.IN_MEMORY
- && !pipeline.equals(connectedVertex.getExecutionPipeline())) {
-
- connectedVertex.setExecutionPipeline(pipeline);
- recurse = true;
- }
-
- if (recurse) {
- reconstructExecutionPipeline(connectedVertex, false, alreadyVisited);
- }
- }
- }
- }
- }
-
- public int getMaxNumberSubtasks(){
- int maxDegree = 0;
-
- for(int i =0; i < this.getNumberOfStageMembers(); i++){
- final ExecutionGroupVertex groupVertex = this.getStageMember(i);
-
- if(groupVertex.getCurrentNumberOfGroupMembers() > maxDegree){
- maxDegree = groupVertex.getCurrentNumberOfGroupMembers();
- }
- }
-
- return maxDegree;
- }
-
- public int getRequiredSlots(){
- Set<Instance> instanceSet = new HashSet<Instance>();
-
- for(int i=0; i< this.getNumberOfStageMembers(); i++){
- final ExecutionGroupVertex groupVertex = this.getStageMember(i);
-
- final Iterator<ExecutionVertex> vertexIterator = groupVertex.iterator();
-
- while(vertexIterator.hasNext()){
- final ExecutionVertex vertex = vertexIterator.next();
-
- instanceSet.add(vertex.getAllocatedResource().getInstance());
- }
-
- }
-
- return instanceSet.size();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStageListener.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStageListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStageListener.java
deleted file mode 100644
index 16c5478..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionStageListener.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.executiongraph;
-
-import org.apache.flink.runtime.jobgraph.JobID;
-
-/**
- * This interface must be implemented in order to receive a notification whenever the job has entered a new execution
- * stage.
- *
- */
-public interface ExecutionStageListener {
-
- /**
- * This method is called to indicate that the job with the given {@link JobID} has finished its previous
- * {@link ExecutionStage} and has entered the next {@link ExecutionStage}. Note that a notification is not sent when
- * the job has entered its initial execution stage.
- *
- * @param jobID
- * the ID of the job the notification belongs to
- * @param executionStage
- * the next execution stage that has just been entered
- */
- void nextExecutionStageEntered(JobID jobID, ExecutionStage executionStage);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionState.java
new file mode 100644
index 0000000..31108a4
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionState.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+public enum ExecutionState {
+
+ /** The job is scheduled, but nothing is running yet */
+ CREATED,
+
+ /** At least some tasks of the job are running */
+ RUNNING,
+
+ /** The execution has finished, no further tasks may be added to the job */
+ FINISHED,
+
+ /** The execution has failed */
+ FAILED
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b32e77a2/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
index d1ee262..adc0f09 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
@@ -41,11 +41,11 @@ import org.apache.flink.runtime.execution.ExecutionStateTransition;
import org.apache.flink.runtime.instance.AllocatedResource;
import org.apache.flink.runtime.instance.AllocationID;
import org.apache.flink.runtime.io.network.gates.GateID;
-import org.apache.flink.runtime.taskmanager.AbstractTaskResult;
+import org.apache.flink.runtime.taskmanager.TaskOperationResult;
import org.apache.flink.runtime.taskmanager.TaskCancelResult;
import org.apache.flink.runtime.taskmanager.TaskKillResult;
import org.apache.flink.runtime.taskmanager.TaskSubmissionResult;
-import org.apache.flink.runtime.taskmanager.AbstractTaskResult.ReturnCode;
+import org.apache.flink.runtime.taskmanager.TaskOperationResult.ReturnCode;
import org.apache.flink.runtime.util.AtomicEnum;
import org.apache.flink.runtime.util.SerializableArrayList;
import org.apache.flink.util.StringUtils;
@@ -432,8 +432,8 @@ public final class ExecutionVertex {
if (this.cancelRequested.compareAndSet(true, false)) {
final TaskCancelResult tsr = cancelTask();
- if (tsr.getReturnCode() != AbstractTaskResult.ReturnCode.SUCCESS
- && tsr.getReturnCode() != AbstractTaskResult.ReturnCode.TASK_NOT_FOUND) {
+ if (tsr.getReturnCode() != AbstractTaskResult.TaskOperationResult.SUCCESS
+ && tsr.getReturnCode() != AbstractTaskResult.TaskOperationResult.TASK_NOT_FOUND) {
LOG.error("Unable to cancel vertex " + this + ": " + tsr.getReturnCode().toString()
+ ((tsr.getDescription() != null) ? (" (" + tsr.getDescription() + ")") : ""));
}
@@ -675,7 +675,7 @@ public final class ExecutionVertex {
if (ar == null) {
final TaskSubmissionResult result = new TaskSubmissionResult(getID(),
- AbstractTaskResult.ReturnCode.NO_INSTANCE);
+ AbstractTaskResult.TaskOperationResult.NO_INSTANCE);
result.setDescription("Assigned instance of vertex " + this.toString() + " is null!");
return result;
}
@@ -690,7 +690,7 @@ public final class ExecutionVertex {
} catch (IOException e) {
final TaskSubmissionResult result = new TaskSubmissionResult(getID(),
- AbstractTaskResult.ReturnCode.IPC_ERROR);
+ AbstractTaskResult.TaskOperationResult.IPC_ERROR);
result.setDescription(StringUtils.stringifyException(e));
return result;
}
@@ -709,7 +709,7 @@ public final class ExecutionVertex {
final ExecutionState state = this.executionState.get();
if (state != ExecutionState.RUNNING) {
- final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.ReturnCode.ILLEGAL_STATE);
+ final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.TaskOperationResult.ILLEGAL_STATE);
result.setDescription("Vertex " + this.toString() + " is in state " + state);
return result;
}
@@ -717,7 +717,7 @@ public final class ExecutionVertex {
final AllocatedResource ar = this.allocatedResource.get();
if (ar == null) {
- final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.ReturnCode.NO_INSTANCE);
+ final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.TaskOperationResult.NO_INSTANCE);
result.setDescription("Assigned instance of vertex " + this.toString() + " is null!");
return result;
}
@@ -725,7 +725,7 @@ public final class ExecutionVertex {
try {
return ar.getInstance().killTask(this.vertexID);
} catch (IOException e) {
- final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.ReturnCode.IPC_ERROR);
+ final TaskKillResult result = new TaskKillResult(getID(), AbstractTaskResult.TaskOperationResult.IPC_ERROR);
result.setDescription(StringUtils.stringifyException(e));
return result;
}
@@ -746,15 +746,15 @@ public final class ExecutionVertex {
final ExecutionState previousState = this.executionState.get();
if (previousState == ExecutionState.CANCELED) {
- return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+ return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
}
if (previousState == ExecutionState.FAILED) {
- return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+ return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
}
if (previousState == ExecutionState.FINISHED) {
- return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+ return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
}
// The vertex has already received a cancel request
@@ -774,7 +774,7 @@ public final class ExecutionVertex {
continue;
}
- return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+ return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
}
// Check if we had a race. If state change is accepted, send cancel request
@@ -783,20 +783,20 @@ public final class ExecutionVertex {
if (this.groupVertex.getStageNumber() != this.executionGraph.getIndexOfCurrentExecutionStage()) {
// Set to canceled directly
updateExecutionState(ExecutionState.CANCELED, null);
- return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+ return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
}
if (previousState != ExecutionState.RUNNING && previousState != ExecutionState.FINISHING) {
// Set to canceled directly
updateExecutionState(ExecutionState.CANCELED, null);
- return new TaskCancelResult(getID(), AbstractTaskResult.ReturnCode.SUCCESS);
+ return new TaskCancelResult(getID(), AbstractTaskResult.TaskOperationResult.SUCCESS);
}
final AllocatedResource ar = this.allocatedResource.get();
if (ar == null) {
final TaskCancelResult result = new TaskCancelResult(getID(),
- AbstractTaskResult.ReturnCode.NO_INSTANCE);
+ AbstractTaskResult.TaskOperationResult.NO_INSTANCE);
result.setDescription("Assigned instance of vertex " + this.toString() + " is null!");
return result;
}
@@ -806,7 +806,7 @@ public final class ExecutionVertex {
} catch (IOException e) {
final TaskCancelResult result = new TaskCancelResult(getID(),
- AbstractTaskResult.ReturnCode.IPC_ERROR);
+ AbstractTaskResult.TaskOperationResult.IPC_ERROR);
result.setDescription(StringUtils.stringifyException(e));
return result;
}