You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/06/05 23:24:35 UTC

flink git commit: [FLINK-2133] [jobmanager] Fix possible deadlock when vertices transition to final state.

Repository: flink
Updated Branches:
  refs/heads/master 8a5207610 -> 2298cfedb


[FLINK-2133] [jobmanager] Fix possible deadlock when vertices transition to final state.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2298cfed
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2298cfed
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2298cfed

Branch: refs/heads/master
Commit: 2298cfedbf880b3a6065a307224c5f3e9e326a0b
Parents: 8a52076
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Jun 5 22:39:29 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Jun 5 22:47:04 2015 +0200

----------------------------------------------------------------------
 .../flink/runtime/executiongraph/Execution.java |   2 +-
 .../runtime/executiongraph/ExecutionGraph.java  | 116 +++++-----
 .../executiongraph/ExecutionJobVertex.java      |   2 +-
 .../TerminalStateDeadlockTest.java              | 209 +++++++++++++++++++
 .../operators/testutils/DummyInvokable.java     |   2 +-
 5 files changed, 262 insertions(+), 69 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/2298cfed/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 7972d43..76a58e8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -284,7 +284,7 @@ public class Execution implements Serializable {
 			return true;
 		}
 		else {
-			// call race, already deployed
+			// call race, already deployed, or already done
 			return false;
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/2298cfed/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 73e827f..705496d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -179,10 +179,8 @@ public class ExecutionGraph implements Serializable {
 	 * that was not recoverable and triggered job failure */
 	private volatile Throwable failureCause;
 
-	/** The position of the vertex that is next expected to finish.
-	 * This is an index into the "verticesInCreationOrder" collection.
-	 * Once this value has reached the number of vertices, the job is done. */
-	private volatile int nextVertexToFinish;
+	/** The number of job vertices that have reached a terminal state */
+	private volatile int numFinishedJobVertices;
 	
 	
 	// ------ Fields that are relevant to the execution and need to be cleared before archiving  -------
@@ -200,7 +198,7 @@ public class ExecutionGraph implements Serializable {
 	private CheckpointCoordinator checkpointCoordinator;
 
 	// ------ Fields that are only relevant for archived execution graphs ------------
-	private ExecutionConfig executionConfig = null;
+	private ExecutionConfig executionConfig;
 
 	// --------------------------------------------------------------------------------------------
 	//   Constructors
@@ -598,7 +596,7 @@ public class ExecutionGraph implements Serializable {
 				for (int i = 0; i < stateTimestamps.length; i++) {
 					stateTimestamps[i] = 0;
 				}
-				nextVertexToFinish = 0;
+				numFinishedJobVertices = 0;
 				transitionState(JobStatus.RESTARTING, JobStatus.CREATED);
 				
 				// if we have checkpointed state, reload it into the executions
@@ -653,7 +651,7 @@ public class ExecutionGraph implements Serializable {
 	 */
 	public void waitUntilFinished() throws InterruptedException {
 		synchronized (progressLock) {
-			while (nextVertexToFinish < verticesInCreationOrder.size()) {
+			while (!state.isTerminalState()) {
 				progressLock.wait();
 			}
 		}
@@ -680,76 +678,62 @@ public class ExecutionGraph implements Serializable {
 
 	void jobVertexInFinalState(ExecutionJobVertex ev) {
 		synchronized (progressLock) {
-			int nextPos = nextVertexToFinish;
-			if (nextPos >= verticesInCreationOrder.size()) {
-				// already done, and we still get a report?
-				// this can happen when:
-				// - two job vertices finish almost simultaneously
-				// - The first one advances the position for the second as well (second is in final state)
-				// - the second (after it could grab the lock) tries to advance the position again
-				return;
+			if (numFinishedJobVertices >= verticesInCreationOrder.size()) {
+				throw new IllegalStateException("All vertices are already finished, cannot transition vertex to finished.");
 			}
+
+			numFinishedJobVertices++;
 			
-			// see if we are the next to finish and then progress until the next unfinished one
-			if (verticesInCreationOrder.get(nextPos) == ev) {
-				do {
-					nextPos++;
-				}
-				while (nextPos < verticesInCreationOrder.size() && verticesInCreationOrder.get(nextPos).isInFinalState());
+			if (numFinishedJobVertices == verticesInCreationOrder.size()) {
 				
-				nextVertexToFinish = nextPos;
-				
-				if (nextPos == verticesInCreationOrder.size()) {
+				// we are done, transition to the final state
+				JobStatus current;
+				while (true) {
+					current = this.state;
 					
-					// we are done, transition to the final state
-					JobStatus current;
-					while (true) {
-						current = this.state;
-						
-						if (current == JobStatus.RUNNING) {
-							if (transitionState(current, JobStatus.FINISHED)) {
-								postRunCleanup();
-								break;
-							}
+					if (current == JobStatus.RUNNING) {
+						if (transitionState(current, JobStatus.FINISHED)) {
+							postRunCleanup();
+							break;
 						}
-						else if (current == JobStatus.CANCELLING) {
-							if (transitionState(current, JobStatus.CANCELED)) {
-								postRunCleanup();
-								break;
-							}
+					}
+					else if (current == JobStatus.CANCELLING) {
+						if (transitionState(current, JobStatus.CANCELED)) {
+							postRunCleanup();
+							break;
 						}
-						else if (current == JobStatus.FAILING) {
-							if (numberOfRetriesLeft > 0 && transitionState(current, JobStatus.RESTARTING)) {
-								numberOfRetriesLeft--;
-								future(new Callable<Object>() {
-									@Override
-									public Object call() throws Exception {
-										try {
-											Thread.sleep(delayBeforeRetrying);
-										}
-										catch(InterruptedException e){
-											// should only happen on shutdown
-										}
-										restart();
-										return null;
+					}
+					else if (current == JobStatus.FAILING) {
+						if (numberOfRetriesLeft > 0 && transitionState(current, JobStatus.RESTARTING)) {
+							numberOfRetriesLeft--;
+							future(new Callable<Object>() {
+								@Override
+								public Object call() throws Exception {
+									try {
+										Thread.sleep(delayBeforeRetrying);
+									}
+									catch(InterruptedException e){
+										// should only happen on shutdown
 									}
-								}, AkkaUtils.globalExecutionContext());
-								break;
-							}
-							else if (numberOfRetriesLeft <= 0 && transitionState(current, JobStatus.FAILED, failureCause)) {
-								postRunCleanup();
-								break;
-							}
+									restart();
+									return null;
+								}
+							}, AkkaUtils.globalExecutionContext());
+							break;
 						}
-						else {
-							fail(new Exception("ExecutionGraph went into final state from state " + current));
+						else if (numberOfRetriesLeft <= 0 && transitionState(current, JobStatus.FAILED, failureCause)) {
+							postRunCleanup();
+							break;
 						}
 					}
-					// done transitioning the state
-
-					// also, notify waiters
-					progressLock.notifyAll();
+					else {
+						fail(new Exception("ExecutionGraph went into final state from state " + current));
+					}
 				}
+				// done transitioning the state
+
+				// also, notify waiters
+				progressLock.notifyAll();
 			}
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/2298cfed/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index 59b3bb6..e53bc10 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -474,7 +474,7 @@ public class ExecutionJobVertex implements Serializable {
 		for (InputSplit split : splits) {
 			// check that split has exactly one local host
 			if(!(split instanceof LocatableInputSplit)) {
-				new JobException("Invalid InputSplit type " + split.getClass().getCanonicalName() + ". " +
+				throw new JobException("Invalid InputSplit type " + split.getClass().getCanonicalName() + ". " +
 						"Strictly local assignment requires LocatableInputSplit");
 			}
 			LocatableInputSplit lis = (LocatableInputSplit) split;

http://git-wip-us.apache.org/repos/asf/flink/blob/2298cfed/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
new file mode 100644
index 0000000..8cba6ca
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/TerminalStateDeadlockTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.executiongraph;
+
+import akka.actor.ActorRef;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.instance.HardwareDescription;
+import org.apache.flink.runtime.instance.Instance;
+import org.apache.flink.runtime.instance.InstanceConnectionInfo;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.instance.SimpleSlot;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.operators.testutils.DummyInvokable;
+
+import org.junit.Test;
+
+import scala.concurrent.duration.FiniteDuration;
+
+import java.lang.reflect.Field;
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.*;
+
+public class TerminalStateDeadlockTest {
+	
+	private final Field stateField;
+	private final Field resourceField;
+	private final Field execGraphStateField;
+	private final Field execGraphSchedulerField;
+	
+	private final SimpleSlot resource;
+
+
+	public TerminalStateDeadlockTest() {
+		try {
+			// the reflection fields to access the private fields
+			this.stateField = Execution.class.getDeclaredField("state");
+			this.stateField.setAccessible(true);
+
+			this.resourceField = Execution.class.getDeclaredField("assignedResource");
+			this.resourceField.setAccessible(true);
+
+			this.execGraphStateField = ExecutionGraph.class.getDeclaredField("state");
+			this.execGraphStateField.setAccessible(true);
+
+			this.execGraphSchedulerField = ExecutionGraph.class.getDeclaredField("scheduler");
+			this.execGraphSchedulerField.setAccessible(true);
+			
+			// the dummy resource
+			InetAddress address = InetAddress.getByName("127.0.0.1");
+			InstanceConnectionInfo ci = new InstanceConnectionInfo(address, 12345);
+				
+			HardwareDescription resources = new HardwareDescription(4, 4000000, 3000000, 2000000);
+			Instance instance = new Instance(ActorRef.noSender(), ci, new InstanceID(), resources, 4);
+
+			this.resource = instance.allocateSimpleSlot(new JobID());
+		}
+		catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+			
+			// silence the compiler
+			throw new RuntimeException();
+		}
+	}
+	
+	 
+	
+	// ------------------------------------------------------------------------
+	
+	@Test
+	public void testProvokeDeadlock() {
+		try {
+			final JobID jobId = resource.getJobID();
+			final JobVertexID vid1 = new JobVertexID();
+			final JobVertexID vid2 = new JobVertexID();
+
+			
+			final Configuration jobConfig = new Configuration();
+			
+			final List<AbstractJobVertex> vertices;
+			{
+				AbstractJobVertex v1 = new AbstractJobVertex("v1", vid1);
+				AbstractJobVertex v2 = new AbstractJobVertex("v2", vid2);
+				v1.setParallelism(1);
+				v2.setParallelism(1);
+				v1.setInvokableClass(DummyInvokable.class);
+				v2.setInvokableClass(DummyInvokable.class);
+				vertices = Arrays.asList(v1, v2);
+			}
+			
+			final Scheduler scheduler = new Scheduler();
+			
+			final Executor executor = Executors.newFixedThreadPool(4);
+			
+			// try a lot!
+			for (int i = 0; i < 20000; i++) {
+				final TestExecGraph eg = new TestExecGraph(jobId);
+				eg.attachJobGraph(vertices);
+				eg.setDelayBeforeRetrying(0);
+				eg.setNumberOfRetriesLeft(1);
+				
+				final Execution e1 = eg.getJobVertex(vid1).getTaskVertices()[0].getCurrentExecutionAttempt();
+				final Execution e2 = eg.getJobVertex(vid2).getTaskVertices()[0].getCurrentExecutionAttempt();
+
+				initializeExecution(e1);
+				initializeExecution(e2);
+
+				execGraphStateField.set(eg, JobStatus.FAILING);
+				execGraphSchedulerField.set(eg, scheduler);
+				
+				Runnable r1 = new Runnable() {
+					@Override
+					public void run() {
+						e1.cancelingComplete();
+					}
+				};
+				Runnable r2 = new Runnable() {
+					@Override
+					public void run() {
+						e2.cancelingComplete();
+					}
+				};
+				
+				executor.execute(r1);
+				executor.execute(r2);
+				
+				eg.waitTillDone();
+			}
+		}
+		catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+	
+	private void initializeExecution(Execution exec) throws IllegalAccessException {
+		// set state to canceling
+		stateField.set(exec, ExecutionState.CANCELING);
+		
+		// assign a resource
+		resourceField.set(exec, resource);
+	}
+	
+	
+	static class TestExecGraph extends ExecutionGraph {
+
+		private static final long serialVersionUID = -7606144898417942044L;
+		
+		private static final Configuration EMPTY_CONFIG = new Configuration();
+
+		private static final FiniteDuration TIMEOUT = new FiniteDuration(30, TimeUnit.SECONDS);
+		
+		private volatile boolean done;
+
+		TestExecGraph(JobID jobId) {
+			super(jobId, "test graph", EMPTY_CONFIG, TIMEOUT);
+		}
+
+		@Override
+		public void scheduleForExecution(Scheduler scheduler) {
+			// notify that we are done with the "restarting"
+			synchronized (this) {
+				done = true;
+				this.notifyAll();
+			}
+		}
+		
+		public void waitTillDone() {
+			try {
+				synchronized (this) {
+					while (!done) {
+						this.wait();
+					}
+				}
+			}
+			catch (InterruptedException e) {
+				throw new RuntimeException(e);
+			}
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/2298cfed/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyInvokable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyInvokable.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyInvokable.java
index 5e99cf4..e4b9056 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyInvokable.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyInvokable.java
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.operators.testutils;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.util.InstantiationUtil;
 
 /**
  * An invokable that does nothing.
@@ -34,6 +33,7 @@ public class DummyInvokable extends AbstractInvokable {
 	@Override
 	public void invoke() {}
 
+	@Override
 	public ClassLoader getUserCodeClassLoader() {
 		return getClass().getClassLoader();
 	}