You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2016/02/10 17:52:52 UTC

[1/6] flink git commit: [FLINK-3341] Make 'auto.offset.reset' compatible with Kafka 0.8 and 0.9

Repository: flink
Updated Branches:
  refs/heads/master 8ccd7544e -> 6968a57a1


[FLINK-3341] Make 'auto.offset.reset' compatible with Kafka 0.8 and 0.9

This closes #1597


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9173825a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9173825a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9173825a

Branch: refs/heads/master
Commit: 9173825aa6a1525d72a78cda16cb4ae1e9b8a8e4
Parents: 8ccd754
Author: Robert Metzger <rm...@apache.org>
Authored: Sat Feb 6 13:27:06 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:12:34 2016 +0100

----------------------------------------------------------------------
 .../connectors/kafka/FlinkKafkaConsumer08.java  |  2 +-
 .../kafka/internals/LegacyFetcher.java          |  3 ++-
 .../connectors/kafka/Kafka08ITCase.java         | 22 ++++++++++----------
 .../kafka/KafkaTestEnvironmentImpl.java         | 20 +++---------------
 .../kafka/KafkaTestEnvironmentImpl.java         | 22 +++++---------------
 .../connectors/kafka/KafkaConsumerTestBase.java |  5 ++---
 .../connectors/kafka/KafkaTestBase.java         |  2 --
 .../connectors/kafka/KafkaTestEnvironment.java  |  3 ---
 .../flink/yarn/YARNSessionFIFOITCase.java       |  1 +
 9 files changed, 25 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
index bdea37f..1cdfffe 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
@@ -70,7 +70,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
  *             <li>socket.timeout.ms</li>
  *             <li>socket.receive.buffer.bytes</li>
  *             <li>fetch.message.max.bytes</li>
- *             <li>auto.offset.reset with the values "latest", "earliest" (unlike 0.8.2 behavior)</li>
+ *             <li>auto.offset.reset with the values "largest", "smallest"</li>
  *             <li>fetch.wait.max.ms</li>
  *         </ul>
  *     </li>

http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
index fe7f777..10f4c41 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
@@ -576,7 +576,8 @@ public class LegacyFetcher implements Fetcher {
 
 		private static long getInvalidOffsetBehavior(Properties config) {
 			long timeType;
-			if (config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest").equals("latest")) {
+			String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest");
+			if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9
 				timeType = OffsetRequest.LatestTime();
 			} else {
 				timeType = OffsetRequest.EarliestTime();

http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
index 6a2fa27..a3e815e 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
@@ -93,13 +93,13 @@ public class Kafka08ITCase extends KafkaConsumerTestBase {
 
 		// set invalid offset:
 		CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl)kafkaServer).createCuratorClient();
-		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardCC.groupId(), topic, 0, 1234);
+		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234);
 		curatorClient.close();
 
 		// read from topic
 		final int valuesCount = 20;
 		final int startFrom = 0;
-		readSequence(env, standardCC.props().props(), parallelism, topic, valuesCount, startFrom);
+		readSequence(env, standardProps, parallelism, topic, valuesCount, startFrom);
 
 		deleteTestTopic(topic);
 	}
@@ -188,9 +188,9 @@ public class Kafka08ITCase extends KafkaConsumerTestBase {
 
 		CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl)kafkaServer).createCuratorClient();
 
-		long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardCC.groupId(), topicName, 0);
-		long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardCC.groupId(), topicName, 1);
-		long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardCC.groupId(), topicName, 2);
+		long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 0);
+		long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 1);
+		long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 2);
 
 		LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
 
@@ -201,9 +201,9 @@ public class Kafka08ITCase extends KafkaConsumerTestBase {
 		LOG.info("Manipulating offsets");
 
 		// set the offset to 50 for the three partitions
-		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardCC.groupId(), topicName, 0, 49);
-		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardCC.groupId(), topicName, 1, 49);
-		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardCC.groupId(), topicName, 2, 49);
+		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 0, 49);
+		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 1, 49);
+		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 2, 49);
 
 		curatorClient.close();
 
@@ -250,9 +250,9 @@ public class Kafka08ITCase extends KafkaConsumerTestBase {
 		// get the offset
 		CuratorFramework curatorFramework = ((KafkaTestEnvironmentImpl)kafkaServer).createCuratorClient();
 
-		long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardCC.groupId(), topicName, 0);
-		long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardCC.groupId(), topicName, 1);
-		long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardCC.groupId(), topicName, 2);
+		long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 0);
+		long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 1);
+		long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 2);
 
 		LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
index 348b75d..6f56ede 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
@@ -21,7 +21,6 @@ package org.apache.flink.streaming.connectors.kafka;
 import kafka.admin.AdminUtils;
 import kafka.api.PartitionMetadata;
 import kafka.common.KafkaException;
-import kafka.consumer.ConsumerConfig;
 import kafka.network.SocketServer;
 import kafka.server.KafkaConfig;
 import kafka.server.KafkaServer;
@@ -68,19 +67,11 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 	private String zookeeperConnectionString;
 	private String brokerConnectionString = "";
 	private Properties standardProps;
-	private ConsumerConfig standardCC;
-
 
 	public String getBrokerConnectionString() {
 		return brokerConnectionString;
 	}
 
-
-	@Override
-	public ConsumerConfig getStandardConsumerConfig() {
-		return standardCC;
-	}
-
 	@Override
 	public Properties getStandardProperties() {
 		return standardProps;
@@ -187,13 +178,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 		standardProps.setProperty("auto.commit.enable", "false");
 		standardProps.setProperty("zookeeper.session.timeout.ms", "12000"); // 6 seconds is default. Seems to be too small for travis.
 		standardProps.setProperty("zookeeper.connection.timeout.ms", "20000");
-		standardProps.setProperty("auto.offset.reset", "earliest"); // read from the beginning.
+		standardProps.setProperty("auto.offset.reset", "smallest"); // read from the beginning. (smallest is kafka 0.8)
 		standardProps.setProperty("fetch.message.max.bytes", "256"); // make a lot of fetches (MESSAGES MUST BE SMALLER!)
-
-		Properties consumerConfigProps = new Properties();
-		consumerConfigProps.putAll(standardProps);
-		consumerConfigProps.setProperty("auto.offset.reset", "smallest");
-		standardCC = new ConsumerConfig(consumerConfigProps);
 	}
 
 	@Override
@@ -274,8 +260,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 	}
 
 	private ZkClient createZkClient() {
-		return new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(),
-				standardCC.zkConnectionTimeoutMs(), new ZooKeeperStringSerializer());
+		return new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
+				Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
index 0855ba6..50dcab8 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
@@ -65,19 +65,12 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 	private String zookeeperConnectionString;
 	private String brokerConnectionString = "";
 	private Properties standardProps;
-	private ConsumerConfig standardCC;
-
 
 	public String getBrokerConnectionString() {
 		return brokerConnectionString;
 	}
 
 	@Override
-	public ConsumerConfig getStandardConsumerConfig() {
-		return standardCC;
-	}
-
-	@Override
 	public Properties getStandardProperties() {
 		return standardProps;
 	}
@@ -184,13 +177,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 		standardProps.setProperty("auto.commit.enable", "false");
 		standardProps.setProperty("zookeeper.session.timeout.ms", "12000"); // 6 seconds is default. Seems to be too small for travis.
 		standardProps.setProperty("zookeeper.connection.timeout.ms", "20000");
-		standardProps.setProperty("auto.offset.reset", "earliest"); // read from the beginning.
+		standardProps.setProperty("auto.offset.reset", "earliest"); // read from the beginning. (earliest is kafka 0.9 value)
 		standardProps.setProperty("fetch.message.max.bytes", "256"); // make a lot of fetches (MESSAGES MUST BE SMALLER!)
-
-		Properties consumerConfigProps = new Properties();
-		consumerConfigProps.putAll(standardProps);
-		consumerConfigProps.setProperty("auto.offset.reset", "smallest");
-		standardCC = new ConsumerConfig(consumerConfigProps);
 	}
 
 	@Override
@@ -233,8 +221,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 	}
 
 	public ZkUtils getZkUtils() {
-		ZkClient creator = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(),
-				standardCC.zkConnectionTimeoutMs(), new ZooKeeperStringSerializer());
+		ZkClient creator = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
+				Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());
 		return ZkUtils.apply(creator, false);
 	}
 
@@ -280,8 +268,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 		try {
 			LOG.info("Deleting topic {}", topic);
 
-			ZkClient zk = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(),
-					standardCC.zkConnectionTimeoutMs(), new ZooKeeperStringSerializer());
+			ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
+				Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());
 
 			AdminUtils.deleteTopic(zkUtils, topic);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
index 2d9f2fc..680e4ec 100644
--- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
+++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
@@ -1284,11 +1284,10 @@ public abstract class KafkaConsumerTestBase extends KafkaTestBase {
 			throws IOException
 	{
 		// write the sequence to log for debugging purposes
-		Properties stdProps = standardCC.props().props();
-		Properties newProps = new Properties(stdProps);
+		Properties newProps = new Properties(standardProps);
 		newProps.setProperty("group.id", "topic-printer"+ UUID.randomUUID().toString());
 		newProps.setProperty("auto.offset.reset", "smallest");
-		newProps.setProperty("zookeeper.connect", standardCC.zkConnect());
+		newProps.setProperty("zookeeper.connect", standardProps.getProperty("zookeeper.connect"));
 
 		ConsumerConfig printerConfig = new ConsumerConfig(newProps);
 		printTopic(topicName, printerConfig, deserializer, elements);

http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
index 73cd2f9..ab1d5b6 100644
--- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
+++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
@@ -66,7 +66,6 @@ public abstract class KafkaTestBase extends TestLogger {
 
 	protected static String brokerConnectionStrings;
 
-	protected static ConsumerConfig standardCC;
 	protected static Properties standardProps;
 	
 	protected static ForkableFlinkMiniCluster flink;
@@ -98,7 +97,6 @@ public abstract class KafkaTestBase extends TestLogger {
 		kafkaServer.prepare(NUMBER_OF_KAFKA_SERVERS);
 
 		standardProps = kafkaServer.getStandardProperties();
-		standardCC = kafkaServer.getStandardConsumerConfig();
 		brokerConnectionStrings = kafkaServer.getBrokerConnectionString();
 
 		// start also a re-usable Flink mini cluster

http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java
index 40be8a1..76a284b 100644
--- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java
+++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java
@@ -17,7 +17,6 @@
 
 package org.apache.flink.streaming.connectors.kafka;
 
-import kafka.consumer.ConsumerConfig;
 import kafka.server.KafkaServer;
 import org.apache.flink.streaming.connectors.kafka.partitioner.KafkaPartitioner;
 import org.apache.flink.streaming.util.serialization.DeserializationSchema;
@@ -25,7 +24,6 @@ import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema;
 import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchemaWrapper;
 import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema;
 
-import java.net.UnknownHostException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
@@ -45,7 +43,6 @@ public abstract class KafkaTestEnvironment {
 
 	public abstract void createTestTopic(String topic, int numberOfPartitions, int replicationFactor);
 
-	public abstract ConsumerConfig getStandardConsumerConfig();
 
 	public abstract Properties getStandardProperties();
 

http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index 8c9a9c7..98dc85f 100644
--- a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -155,6 +155,7 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
 			List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
 			Assert.assertEquals(1, apps.size()); // Only one running
 			ApplicationReport app = apps.get(0);
+
 			Assert.assertEquals("MyCustomName", app.getName());
 			ApplicationId id = app.getApplicationId();
 			yc.killApplication(id);


[2/6] flink git commit: [FLINK-3350] [tests] Increase default test Akka ask and ZooKeeper timeouts

Posted by se...@apache.org.
[FLINK-3350] [tests] Increase default test Akka ask and ZooKeeper timeouts


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b8f40251
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b8f40251
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b8f40251

Branch: refs/heads/master
Commit: b8f40251c6c45379118254c21b0d553c2d3b8937
Parents: 9173825
Author: Ufuk Celebi <uc...@apache.org>
Authored: Mon Feb 8 14:24:43 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:26:43 2016 +0100

----------------------------------------------------------------------
 .../runtime/minicluster/FlinkMiniCluster.scala  | 20 ++++++++++++++++++--
 .../minicluster/LocalFlinkMiniCluster.scala     |  2 ++
 .../runtime/testutils/ZooKeeperTestUtils.java   |  5 +++--
 .../runtime/testingUtils/TestingCluster.scala   |  2 ++
 .../kafka/KafkaTestEnvironmentImpl.java         |  7 ++++---
 .../kafka/KafkaTestEnvironmentImpl.java         |  7 ++++---
 ...ctTaskManagerProcessFailureRecoveryTest.java |  3 +++
 .../JobManagerCheckpointRecoveryITCase.java     |  8 ++++++--
 .../recovery/ProcessFailureCancelingITCase.java |  2 +-
 9 files changed, 43 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
index 4cdda3f..0346d6d 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
@@ -42,7 +42,7 @@ import org.apache.flink.runtime.webmonitor.{WebMonitorUtils, WebMonitor}
 
 import org.slf4j.LoggerFactory
 
-import scala.concurrent.duration.FiniteDuration
+import scala.concurrent.duration.{Duration, FiniteDuration}
 import scala.concurrent._
 import scala.concurrent.forkjoin.ForkJoinPool
 
@@ -86,7 +86,7 @@ abstract class FlinkMiniCluster(
   
   implicit val executionContext = ExecutionContext.global
 
-  implicit val timeout = AkkaUtils.getTimeout(userConfiguration)
+  implicit val timeout = AkkaUtils.getTimeout(configuration)
 
   val recoveryMode = RecoveryMode.fromConfig(configuration)
 
@@ -188,6 +188,22 @@ abstract class FlinkMiniCluster(
     AkkaUtils.getAkkaConfig(configuration, Some((hostname, resolvedPort)))
   }
 
+  /**
+    * Sets CI environment (Travis) specific config defaults.
+    */
+  def setDefaultCiConfig(config: Configuration) : Unit = {
+    // https://docs.travis-ci.com/user/environment-variables#Default-Environment-Variables
+    if (sys.env.contains("CI")) {
+      // Only set if nothing specified in config
+      if (config.getString(ConfigConstants.AKKA_ASK_TIMEOUT, null) == null) {
+        val duration = Duration(ConfigConstants.DEFAULT_AKKA_ASK_TIMEOUT) * 10
+        config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, s"${duration.toSeconds}s")
+
+        LOG.info(s"Akka ask timeout set to ${duration.toSeconds}s")
+      }
+    }
+  }
+
   // --------------------------------------------------------------------------
   //                          Start/Stop Methods
   // --------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index 913aec0..c803429 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -48,6 +48,8 @@ class LocalFlinkMiniCluster(
   override def generateConfiguration(userConfiguration: Configuration): Configuration = {
     val config = getDefaultConfig
 
+    setDefaultCiConfig(config)
+
     config.addAll(userConfiguration)
     setMemory(config)
     initializeIOFormatClasses(config)

http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java
index 6c33835..75569ec 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java
@@ -70,9 +70,9 @@ public class ZooKeeperTestUtils {
 		config.setString(ConfigConstants.ZOOKEEPER_QUORUM_KEY, zooKeeperQuorum);
 
 		int connTimeout = 5000;
-		if (System.getenv().get("CI") != null) {
+		if (System.getenv().containsKey("CI")) {
 			// The regular timeout is to aggressive for Travis and connections are often lost.
-			connTimeout = 20000;
+			connTimeout = 30000;
 		}
 
 		config.setInteger(ConfigConstants.ZOOKEEPER_CONNECTION_TIMEOUT, connTimeout);
@@ -87,6 +87,7 @@ public class ZooKeeperTestUtils {
 		config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "1000 ms");
 		config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "6 s");
 		config.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 9);
+		config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
 		config.setString(ConfigConstants.EXECUTION_RETRY_DELAY_KEY, "10 s");
 
 		return config;

http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
index 5eee4e5..bd56040 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
@@ -63,6 +63,8 @@ class TestingCluster(
     cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 10)
     cfg.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, -1)
 
+    setDefaultCiConfig(cfg)
+
     cfg.addAll(userConfig)
     cfg
   }

http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
index 6f56ede..3d9e4f5 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
@@ -176,8 +176,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 		standardProps.setProperty("bootstrap.servers", brokerConnectionString);
 		standardProps.setProperty("group.id", "flink-tests");
 		standardProps.setProperty("auto.commit.enable", "false");
-		standardProps.setProperty("zookeeper.session.timeout.ms", "12000"); // 6 seconds is default. Seems to be too small for travis.
-		standardProps.setProperty("zookeeper.connection.timeout.ms", "20000");
+		standardProps.setProperty("zookeeper.session.timeout.ms", "30000"); // 6 seconds is default. Seems to be too small for travis.
+		standardProps.setProperty("zookeeper.connection.timeout.ms", "30000");
 		standardProps.setProperty("auto.offset.reset", "smallest"); // read from the beginning. (smallest is kafka 0.8)
 		standardProps.setProperty("fetch.message.max.bytes", "256"); // make a lot of fetches (MESSAGES MUST BE SMALLER!)
 	}
@@ -292,7 +292,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 		kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
 
 		// for CI stability, increase zookeeper session timeout
-		kafkaProperties.put("zookeeper.session.timeout.ms", "20000");
+		kafkaProperties.put("zookeeper.session.timeout.ms", "30000");
+		kafkaProperties.put("zookeeper.connection.timeout.ms", "30000");
 
 		final int numTries = 5;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
index 50dcab8..9dfd021 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
@@ -175,8 +175,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 		standardProps.setProperty("bootstrap.servers", brokerConnectionString);
 		standardProps.setProperty("group.id", "flink-tests");
 		standardProps.setProperty("auto.commit.enable", "false");
-		standardProps.setProperty("zookeeper.session.timeout.ms", "12000"); // 6 seconds is default. Seems to be too small for travis.
-		standardProps.setProperty("zookeeper.connection.timeout.ms", "20000");
+		standardProps.setProperty("zookeeper.session.timeout.ms", "30000"); // 6 seconds is default. Seems to be too small for travis.
+		standardProps.setProperty("zookeeper.connection.timeout.ms", "30000");
 		standardProps.setProperty("auto.offset.reset", "earliest"); // read from the beginning. (earliest is kafka 0.9 value)
 		standardProps.setProperty("fetch.message.max.bytes", "256"); // make a lot of fetches (MESSAGES MUST BE SMALLER!)
 	}
@@ -296,7 +296,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
 		kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
 
 		// for CI stability, increase zookeeper session timeout
-		kafkaProperties.put("zookeeper.session.timeout.ms", "20000");
+		kafkaProperties.put("zookeeper.session.timeout.ms", "30000");
+		kafkaProperties.put("zookeeper.connection.timeout.ms", "30000");
 
 		final int numTries = 5;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
index 5dd870f..7b4c9b2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.jobmanager.JobManager;
 import org.apache.flink.runtime.jobmanager.MemoryArchivist;
 import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.minicluster.FlinkMiniCluster;
 import org.apache.flink.runtime.taskmanager.TaskManager;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.util.NetUtils;
@@ -122,6 +123,7 @@ public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends Test
 			jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "6 s");
 			jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 9);
 			jmConfig.setString(ConfigConstants.EXECUTION_RETRY_DELAY_KEY, "10 s");
+			jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
 
 			jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<Tuple2<String, Object>>(localAddress));
 			ActorRef jmActor = JobManager.startJobManagerActors(
@@ -376,6 +378,7 @@ public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends Test
 				cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 4);
 				cfg.setInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, 100);
 				cfg.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
+				cfg.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
 
 				TaskManager.selectNetworkInterfaceAndRunTaskManager(cfg, TaskManager.class);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
index 737d39a..5d1b2c5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
@@ -156,6 +156,7 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 		Configuration config = ZooKeeperTestUtils.createZooKeeperRecoveryModeConfig(ZooKeeper
 				.getConnectString(), FileStateBackendBasePath.getAbsoluteFile().toURI().toString());
 		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, Parallelism);
+		config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
 
 		ActorSystem testSystem = null;
 		JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
@@ -182,7 +183,8 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 			leaderRetrievalService.start(leaderListener);
 
 			// The task manager
-			taskManagerSystem = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
+			taskManagerSystem = AkkaUtils.createActorSystem(
+					config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
 			TaskManager.startTaskManagerComponentsAndActor(
 					config, taskManagerSystem, "localhost",
 					Option.<String>empty(), Option.<LeaderRetrievalService>empty(),
@@ -297,6 +299,7 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 				fileStateBackendPath);
 
 		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 2);
+		config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
 
 		JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
 		LeaderRetrievalService leaderRetrievalService = null;
@@ -321,7 +324,8 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 			leaderRetrievalService.start(leaderListener);
 
 			// The task manager
-			taskManagerSystem = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
+			taskManagerSystem = AkkaUtils.createActorSystem(
+					config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
 			TaskManager.startTaskManagerComponentsAndActor(
 					config, taskManagerSystem, "localhost",
 					Option.<String>empty(), Option.<LeaderRetrievalService>empty(),

http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
index 37e4e38..2b49c08 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
@@ -97,7 +97,7 @@ public class ProcessFailureCancelingITCase {
 			jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "5 s");
 			jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "2000 s");
 			jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 10);
-			jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "10 s");
+			jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
 
 			jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<Tuple2<String, Object>>(localAddress));
 			ActorRef jmActor = JobManager.startJobManagerActors(


[3/6] flink git commit: [hotfix] [tests] Ignore ZooKeeper logs in process tests

Posted by se...@apache.org.
[hotfix] [tests] Ignore ZooKeeper logs in process tests


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3a643c07
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3a643c07
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3a643c07

Branch: refs/heads/master
Commit: 3a643c07792c62142c1f8cda172d4f4c3442c9b3
Parents: b8f4025
Author: Ufuk Celebi <uc...@apache.org>
Authored: Tue Feb 9 11:01:39 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:27:16 2016 +0100

----------------------------------------------------------------------
 .../JobManagerSubmittedJobGraphsRecoveryITCase.java          | 6 +++++-
 .../org/apache/flink/runtime/testutils/CommonTestUtils.java  | 1 +
 .../AbstractJobManagerProcessFailureRecoveryITCase.java      | 8 +++++---
 .../org/apache/flink/test/recovery/ChaosMonkeyITCase.java    | 4 ++++
 .../test/recovery/JobManagerCheckpointRecoveryITCase.java    | 8 ++++++++
 5 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java
index 99f7bd7..59c7c39 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java
@@ -343,6 +343,10 @@ public class JobManagerSubmittedJobGraphsRecoveryITCase extends TestLogger {
 			assertEquals(2, jobSubmitSuccessMessages);
 		}
 		catch (Throwable t) {
+			// Print early (in some situations the process logs get too big
+			// for Travis and the root problem is not shown)
+			t.printStackTrace();
+
 			// In case of an error, print the job manager process logs.
 			if (jobManagerProcess[0] != null) {
 				jobManagerProcess[0].printProcessLog();
@@ -352,7 +356,7 @@ public class JobManagerSubmittedJobGraphsRecoveryITCase extends TestLogger {
 				jobManagerProcess[1].printProcessLog();
 			}
 
-			t.printStackTrace();
+			throw t;
 		}
 		finally {
 			if (jobManagerProcess[0] != null) {

http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
index 069b6af..bbb6a89 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
@@ -147,6 +147,7 @@ public class CommonTestUtils {
 			writer.println("log4j.appender.console.layout=org.apache.log4j.PatternLayout");
 			writer.println("log4j.appender.console.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n");
 			writer.println("log4j.logger.org.eclipse.jetty.util.log=OFF");
+			writer.println("log4j.logger.org.apache.zookeeper=OFF");
 
 			writer.flush();
 			writer.close();

http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java
index 2f6b762..6122352 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java
@@ -246,8 +246,10 @@ public abstract class AbstractJobManagerProcessFailureRecoveryITCase extends Tes
 				fail("The program encountered a " + error.getClass().getSimpleName() + " : " + error.getMessage());
 			}
 		}
-		catch (Exception e) {
-			e.printStackTrace();
+		catch (Throwable t) {
+			// Print early (in some situations the process logs get too big
+			// for Travis and the root problem is not shown)
+			t.printStackTrace();
 
 			for (JobManagerProcess p : jmProcess) {
 				if (p != null) {
@@ -255,7 +257,7 @@ public abstract class AbstractJobManagerProcessFailureRecoveryITCase extends Tes
 				}
 			}
 
-			fail(e.getMessage());
+			throw t;
 		}
 		finally {
 			for (int i = 0; i < numberOfTaskManagers; i++) {

http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
index 6ae0d46..76f96ef 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
@@ -320,6 +320,10 @@ public class ChaosMonkeyITCase extends TestLogger {
 			LOG.info("Recovery state clean");
 		}
 		catch (Throwable t) {
+			// Print early (in some situations the process logs get too big
+			// for Travis and the root problem is not shown)
+			t.printStackTrace();
+
 			System.out.println("#################################################");
 			System.out.println(" TASK MANAGERS");
 			System.out.println("#################################################");

http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
index 5d1b2c5..59a05ff 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
@@ -248,6 +248,10 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 			}
 		}
 		catch (Throwable t) {
+			// Print early (in some situations the process logs get too big
+			// for Travis and the root problem is not shown)
+			t.printStackTrace();
+
 			// In case of an error, print the job manager process logs.
 			if (jobManagerProcess[0] != null) {
 				jobManagerProcess[0].printProcessLog();
@@ -400,6 +404,10 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 			assertTrue("Did not find expected output in logs.", success);
 		}
 		catch (Throwable t) {
+			// Print early (in some situtations the process logs get too big
+			// for Travis and the root problem is not shown)
+			t.printStackTrace();
+
 			// In case of an error, print the job manager process logs.
 			if (jobManagerProcess[0] != null) {
 				jobManagerProcess[0].printProcessLog();


[5/6] flink git commit: [hotfix] [tests] Reset state to allow retry on failure

Posted by se...@apache.org.
[hotfix] [tests] Reset state to allow retry on failure

This closes #1611


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/48b74546
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/48b74546
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/48b74546

Branch: refs/heads/master
Commit: 48b745466202ebbb68608930e13cb6ed4a35e6e7
Parents: 756cbaf
Author: Ufuk Celebi <uc...@apache.org>
Authored: Tue Feb 9 12:45:41 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:27:41 2016 +0100

----------------------------------------------------------------------
 .../JobManagerCheckpointRecoveryITCase.java      | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/48b74546/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
index 59a05ff..ea30c58 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
@@ -116,15 +116,15 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 
 	private static final int Parallelism = 8;
 
-	private static final CountDownLatch CompletedCheckpointsLatch = new CountDownLatch(2);
+	private static CountDownLatch CompletedCheckpointsLatch = new CountDownLatch(2);
 
-	private static final AtomicLongArray RecoveredStates = new AtomicLongArray(Parallelism);
+	private static AtomicLongArray RecoveredStates = new AtomicLongArray(Parallelism);
 
-	private static final CountDownLatch FinalCountLatch = new CountDownLatch(1);
+	private static CountDownLatch FinalCountLatch = new CountDownLatch(1);
 
-	private static final AtomicReference<Long> FinalCount = new AtomicReference<>();
+	private static AtomicReference<Long> FinalCount = new AtomicReference<>();
 
-	private static final long LastElement = -1;
+	private static long LastElement = -1;
 
 	/**
 	 * Simple checkpointed streaming sum.
@@ -156,7 +156,6 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 		Configuration config = ZooKeeperTestUtils.createZooKeeperRecoveryModeConfig(ZooKeeper
 				.getConnectString(), FileStateBackendBasePath.getAbsoluteFile().toURI().toString());
 		config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, Parallelism);
-		config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
 
 		ActorSystem testSystem = null;
 		JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
@@ -248,6 +247,13 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 			}
 		}
 		catch (Throwable t) {
+			// Reset all static state for test retries
+			CompletedCheckpointsLatch = new CountDownLatch(2);
+			RecoveredStates = new AtomicLongArray(Parallelism);
+			FinalCountLatch = new CountDownLatch(1);
+			FinalCount = new AtomicReference<>();
+			LastElement = -1;
+
 			// Print early (in some situations the process logs get too big
 			// for Travis and the root problem is not shown)
 			t.printStackTrace();
@@ -303,7 +309,6 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
 				fileStateBackendPath);
 
 		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 2);
-		config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
 
 		JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
 		LeaderRetrievalService leaderRetrievalService = null;


[4/6] flink git commit: [hotfix] [tests] Log retry rule failures on warn level

Posted by se...@apache.org.
[hotfix] [tests] Log retry rule failures on warn level


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/756cbaff
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/756cbaff
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/756cbaff

Branch: refs/heads/master
Commit: 756cbafff1fd25f67268ca84b62c8a479156bf88
Parents: 3a643c0
Author: Ufuk Celebi <uc...@apache.org>
Authored: Tue Feb 9 11:25:37 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:27:28 2016 +0100

----------------------------------------------------------------------
 .../test/java/org/apache/flink/testutils/junit/RetryRule.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/756cbaff/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java b/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
index a4aff86..2b3a37a 100644
--- a/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
+++ b/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
@@ -113,7 +113,7 @@ public class RetryRule implements TestRule {
 					break; // success
 				}
 				catch (Throwable t) {
-					LOG.debug(String.format("Test run failed (%d/%d).",
+					LOG.warn(String.format("Test run failed (%d/%d).",
 							currentRun, timesOnFailure + 1), t);
 
 					// Throw the failure if retried too often
@@ -156,7 +156,7 @@ public class RetryRule implements TestRule {
 					break; // success
 				}
 				catch (Throwable t) {
-					LOG.debug(String.format("Test run failed (%d/%d).", currentRun, timesOnFailure + 1), t);
+					LOG.warn(String.format("Test run failed (%d/%d).", currentRun, timesOnFailure + 1), t);
 
 					if (!exceptionClass.isAssignableFrom(t.getClass()) || currentRun >= timesOnFailure) {
 						// Throw the failure if retried too often, or if it is the wrong exception


[6/6] flink git commit: [FLINK-3260] [runtime] Enforce terminal state of Executions

Posted by se...@apache.org.
[FLINK-3260] [runtime] Enforce terminal state of Executions

This commit fixes the problem that Executions could leave their terminal state
FINISHED to transition to FAILED. Such a transition will be propagated to the
ExecutionGraph where it entails JobStatus changes. Since the Execution already
reached a terminal state, it should not again affect the ExecutionGraph. This
can lead to an inconsistent state in case of a restart where the old Executions
get disassociated from the ExecutionGraph.

This closes #1613


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6968a57a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6968a57a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6968a57a

Branch: refs/heads/master
Commit: 6968a57a1a31a11b33bacd2c94d6559bcabd6eb9
Parents: 48b7454
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Feb 9 10:30:12 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:34:37 2016 +0100

----------------------------------------------------------------------
 .../flink/runtime/executiongraph/Execution.java |  14 +-
 .../ExecutionGraphRestartTest.java              |  90 +++++++++++++
 .../runtime/testingUtils/TestingCluster.scala   |   6 +-
 .../testingUtils/TestingTaskManagerLike.scala   |   4 +-
 .../runtime/testingUtils/TestingUtils.scala     | 133 ++++++++++++++++++-
 5 files changed, 233 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index eb2e68c..db037bb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -435,7 +435,7 @@ public class Execution implements Serializable {
 				return;
 			}
 			else if (current == CREATED || current == SCHEDULED) {
-				// from here, we can directly switch to cancelled, because the no task has been deployed
+				// from here, we can directly switch to cancelled, because no task has been deployed
 				if (transitionState(current, CANCELED)) {
 					
 					// we skip the canceling state. set the timestamp, for a consistent appearance
@@ -754,11 +754,10 @@ public class Execution implements Serializable {
 				return false;
 			}
 
-			if (current == CANCELED) {
-				// we are already aborting or are already aborted
+			if (current == CANCELED || current == FINISHED) {
+				// we are already aborting or are already aborted or we are already finished
 				if (LOG.isDebugEnabled()) {
-					LOG.debug(String.format("Ignoring transition of vertex %s to %s while being %s", 
-							getVertexWithAttempt(), FAILED, CANCELED));
+					LOG.debug("Ignoring transition of vertex {} to {} while being {}.", getVertexWithAttempt(), FAILED, current);
 				}
 				return false;
 			}
@@ -928,6 +927,11 @@ public class Execution implements Serializable {
 	}
 
 	private boolean transitionState(ExecutionState currentState, ExecutionState targetState, Throwable error) {
+		// sanity check
+		if (currentState.isTerminal()) {
+			throw new IllegalStateException("Cannot leave terminal state " + currentState + " to transition to " + targetState + ".");
+		}
+
 		if (STATE_UPDATER.compareAndSet(this, currentState, targetState)) {
 			markTimestamp(targetState);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
index 0c3af8f..47a48a2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
@@ -36,6 +36,7 @@ import org.junit.Test;
 import scala.concurrent.duration.Deadline;
 import scala.concurrent.duration.FiniteDuration;
 
+import java.util.Iterator;
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.SimpleActorGateway;
@@ -387,6 +388,95 @@ public class ExecutionGraphRestartTest {
 		assertEquals(1, eg.getNumberOfRetriesLeft());
 	}
 
+	/**
+	 * Tests that a failing execution does not affect a restarted job. This is important if a
+	 * callback handler fails an execution after it has already reached a final state and the job
+	 * has been restarted.
+	 */
+	@Test
+	public void testFailingExecutionAfterRestart() throws Exception {
+		Instance instance = ExecutionGraphTestUtils.getInstance(
+			new SimpleActorGateway(TestingUtils.directExecutionContext()),
+			2);
+
+		Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
+		scheduler.newInstanceAvailable(instance);
+
+		JobVertex sender = new JobVertex("Task1");
+		sender.setInvokableClass(Tasks.NoOpInvokable.class);
+		sender.setParallelism(1);
+
+		JobVertex receiver = new JobVertex("Task2");
+		receiver.setInvokableClass(Tasks.NoOpInvokable.class);
+		receiver.setParallelism(1);
+
+		JobGraph jobGraph = new JobGraph("Pointwise job", sender, receiver);
+
+		ExecutionGraph eg = new ExecutionGraph(
+			TestingUtils.defaultExecutionContext(),
+			new JobID(),
+			"test job",
+			new Configuration(),
+			AkkaUtils.getDefaultTimeout());
+		eg.setNumberOfRetriesLeft(1);
+		eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());
+
+		assertEquals(JobStatus.CREATED, eg.getState());
+
+		eg.scheduleForExecution(scheduler);
+		assertEquals(JobStatus.RUNNING, eg.getState());
+
+		Iterator<ExecutionVertex> executionVertices = eg.getAllExecutionVertices().iterator();
+
+		Execution finishedExecution = executionVertices.next().getCurrentExecutionAttempt();
+		Execution failedExecution = executionVertices.next().getCurrentExecutionAttempt();
+
+		finishedExecution.markFinished();
+		failedExecution.fail(new Exception("Test Exception"));
+
+		failedExecution.cancelingComplete();
+
+		FiniteDuration timeout = new FiniteDuration(2, TimeUnit.MINUTES);
+
+		Deadline deadline = timeout.fromNow();
+
+		while (deadline.hasTimeLeft() && eg.getState() != JobStatus.RUNNING) {
+			Thread.sleep(100);
+		}
+
+		assertEquals(JobStatus.RUNNING, eg.getState());
+
+		// Wait for deploying after async restart
+		deadline = timeout.fromNow();
+		boolean success = false;
+
+		while (deadline.hasTimeLeft() && !success) {
+			success = true;
+
+			for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
+				if (vertex.getCurrentExecutionAttempt().getAssignedResource() == null) {
+					success = false;
+					Thread.sleep(100);
+					break;
+				} else {
+					vertex.getCurrentExecutionAttempt().switchToRunning();
+				}
+			}
+		}
+
+		// fail old finished execution, this should not affect the execution
+		finishedExecution.fail(new Exception("This should have no effect"));
+
+		for (ExecutionVertex vertex: eg.getAllExecutionVertices()) {
+			vertex.getCurrentExecutionAttempt().markFinished();
+		}
+
+		// the state of the finished execution should have not changed since it is terminal
+		assertEquals(ExecutionState.FINISHED, finishedExecution.getState());
+
+		assertEquals(JobStatus.FINISHED, eg.getState());
+	}
+
 	private static void restartAfterFailure(ExecutionGraph eg, FiniteDuration timeout, boolean haltAfterRestart) throws InterruptedException {
 
 		eg.getAllExecutionVertices().iterator().next().fail(new Exception("Test Exception"));

http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
index bd56040..22b0d29 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
@@ -42,9 +42,9 @@ import scala.concurrent.{Await, Future}
  *                          otherwise false
  */
 class TestingCluster(
-                      userConfiguration: Configuration,
-                      singleActorSystem: Boolean,
-                      synchronousDispatcher: Boolean)
+    userConfiguration: Configuration,
+    singleActorSystem: Boolean,
+    synchronousDispatcher: Boolean)
   extends FlinkMiniCluster(
     userConfiguration,
     singleActorSystem) {

http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
index c10e83e..e9dbdde 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.testingUtils
 import akka.actor.{Terminated, ActorRef}
 import org.apache.flink.api.common.JobID
 import org.apache.flink.runtime.FlinkActor
-import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor
 import org.apache.flink.runtime.execution.ExecutionState
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
 import org.apache.flink.runtime.messages.JobManagerMessages.{ResponseLeaderSessionID,
@@ -32,8 +31,7 @@ AcknowledgeRegistration}
 import org.apache.flink.runtime.messages.TaskMessages.{SubmitTask, UpdateTaskExecutionState, TaskInFinalState}
 import org.apache.flink.runtime.taskmanager.TaskManager
 import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobRemoved
-import org.apache.flink.runtime.testingUtils.TestingMessages.{DisableDisconnect,
-CheckIfJobRemoved, Alive}
+import org.apache.flink.runtime.testingUtils.TestingMessages.{DisableDisconnect, CheckIfJobRemoved, Alive}
 import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages._
 
 import scala.concurrent.duration._

http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index 98faa34..679dc71 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -26,8 +26,11 @@ import com.google.common.util.concurrent.MoreExecutors
 
 import com.typesafe.config.ConfigFactory
 import grizzled.slf4j.Logger
+import org.apache.flink.api.common.JobExecutionResult
 
 import org.apache.flink.configuration.{ConfigConstants, Configuration}
+import org.apache.flink.runtime.client.JobClient
+import org.apache.flink.runtime.jobgraph.JobGraph
 import org.apache.flink.runtime.jobmanager.{MemoryArchivist, JobManager}
 import org.apache.flink.runtime.{LogMessages, LeaderSessionMessageFilter, FlinkActor}
 import org.apache.flink.runtime.akka.AkkaUtils
@@ -168,6 +171,26 @@ object TestingUtils {
   }
 
   def createTaskManager(
+    actorSystem: ActorSystem,
+    jobManager: ActorGateway,
+    configuration: Configuration,
+    useLocalCommunication: Boolean,
+    waitForRegistration: Boolean,
+    taskManagerClass: Class[_ <: TaskManager])
+  : ActorGateway = {
+    val jobManagerURL = AkkaUtils.getAkkaURL(actorSystem, jobManager.actor)
+
+    createTaskManager(
+      actorSystem,
+      jobManagerURL,
+      configuration,
+      useLocalCommunication,
+      waitForRegistration,
+      taskManagerClass
+    )
+  }
+
+  def createTaskManager(
       actorSystem: ActorSystem,
       jobManager: ActorGateway,
       configuration: Configuration,
@@ -200,11 +223,30 @@ object TestingUtils {
     * @return ActorGateway of the created TaskManager
     */
   def createTaskManager(
+    actorSystem: ActorSystem,
+    jobManagerURL: String,
+    configuration: Configuration,
+    useLocalCommunication: Boolean,
+    waitForRegistration: Boolean)
+  : ActorGateway = {
+    createTaskManager(
+      actorSystem,
+      jobManagerURL,
+      configuration,
+      useLocalCommunication,
+      waitForRegistration,
+      classOf[TestingTaskManager]
+    )
+  }
+
+
+  def createTaskManager(
       actorSystem: ActorSystem,
       jobManagerURL: String,
       configuration: Configuration,
       useLocalCommunication: Boolean,
-      waitForRegistration: Boolean)
+      waitForRegistration: Boolean,
+      taskManagerClass: Class[_ <: TaskManager])
     : ActorGateway = {
 
     val resultingConfiguration = new Configuration()
@@ -222,7 +264,7 @@ object TestingUtils {
       None,
       leaderRetrievalService,
       useLocalCommunication,
-      classOf[TestingTaskManager]
+      taskManagerClass
     )
 
     if (waitForRegistration) {
@@ -264,6 +306,72 @@ object TestingUtils {
       actorSystem: ActorSystem,
       configuration: Configuration)
     : ActorGateway = {
+    createJobManager(
+      actorSystem,
+      configuration,
+      classOf[TestingJobManager]
+    )
+  }
+
+  def createJobManager(
+      actorSystem: ActorSystem,
+      configuration: Configuration,
+      executionContext: ExecutionContext)
+    : ActorGateway = {
+
+    val (_,
+    instanceManager,
+    scheduler,
+    libraryCacheManager,
+    executionRetries,
+    delayBetweenRetries,
+    timeout,
+    archiveCount,
+    leaderElectionService,
+    submittedJobGraphs,
+    checkpointRecoveryFactory) = JobManager.createJobManagerComponents(
+      configuration,
+      None
+    )
+
+    val archiveProps = Props(classOf[TestingMemoryArchivist], archiveCount)
+
+    val archive: ActorRef = actorSystem.actorOf(archiveProps, JobManager.ARCHIVE_NAME)
+
+    val jobManagerProps = Props(
+      classOf[TestingJobManager],
+      configuration,
+      executionContext,
+      instanceManager,
+      scheduler,
+      libraryCacheManager,
+      archive,
+      executionRetries,
+      delayBetweenRetries,
+      timeout,
+      leaderElectionService,
+      submittedJobGraphs,
+      checkpointRecoveryFactory)
+
+    val jobManager: ActorRef = actorSystem.actorOf(jobManagerProps, JobManager.JOB_MANAGER_NAME)
+
+    new AkkaActorGateway(jobManager, null)
+  }
+
+
+  /**
+    * Creates a JobManager of the given class using the default recovery mode (standalone)
+    *
+    * @param actorSystem ActorSystem to use
+    * @param configuration Configuration to use
+    * @param jobManagerClass JobManager class to instantiate
+    * @return
+    */
+  def createJobManager(
+      actorSystem: ActorSystem,
+      configuration: Configuration,
+      jobManagerClass: Class[_ <: JobManager])
+    : ActorGateway = {
 
     configuration.setString(ConfigConstants.RECOVERY_MODE, ConfigConstants.DEFAULT_RECOVERY_MODE)
 
@@ -272,7 +380,7 @@ object TestingUtils {
         actorSystem,
         Some(JobManager.JOB_MANAGER_NAME),
         Some(JobManager.ARCHIVE_NAME),
-        classOf[TestingJobManager],
+        jobManagerClass,
         classOf[MemoryArchivist])
 
     new AkkaActorGateway(actor, null)
@@ -312,6 +420,25 @@ object TestingUtils {
     new AkkaActorGateway(actor, null)
   }
 
+  def submitJobAndWait(
+      actorSystem: ActorSystem,
+      jobManager: ActorGateway,
+      jobGraph: JobGraph)
+    : JobExecutionResult = {
+
+    val jobManagerURL = AkkaUtils.getAkkaURL(actorSystem, jobManager.actor)
+    val leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManagerURL)
+
+    JobClient.submitJobAndWait(
+      actorSystem,
+      leaderRetrievalService,
+      jobGraph,
+      TESTING_DURATION,
+      false,
+      Thread.currentThread().getContextClassLoader
+    )
+  }
+
   class ForwardingActor(val target: ActorRef, val leaderSessionID: Option[UUID])
     extends FlinkActor with LeaderSessionMessageFilter with LogMessages {