You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2016/02/10 17:52:52 UTC
[1/6] flink git commit: [FLINK-3341] Make 'auto.offset.reset'
compatible with Kafka 0.8 and 0.9
Repository: flink
Updated Branches:
refs/heads/master 8ccd7544e -> 6968a57a1
[FLINK-3341] Make 'auto.offset.reset' compatible with Kafka 0.8 and 0.9
This closes #1597
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9173825a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9173825a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9173825a
Branch: refs/heads/master
Commit: 9173825aa6a1525d72a78cda16cb4ae1e9b8a8e4
Parents: 8ccd754
Author: Robert Metzger <rm...@apache.org>
Authored: Sat Feb 6 13:27:06 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:12:34 2016 +0100
----------------------------------------------------------------------
.../connectors/kafka/FlinkKafkaConsumer08.java | 2 +-
.../kafka/internals/LegacyFetcher.java | 3 ++-
.../connectors/kafka/Kafka08ITCase.java | 22 ++++++++++----------
.../kafka/KafkaTestEnvironmentImpl.java | 20 +++---------------
.../kafka/KafkaTestEnvironmentImpl.java | 22 +++++---------------
.../connectors/kafka/KafkaConsumerTestBase.java | 5 ++---
.../connectors/kafka/KafkaTestBase.java | 2 --
.../connectors/kafka/KafkaTestEnvironment.java | 3 ---
.../flink/yarn/YARNSessionFIFOITCase.java | 1 +
9 files changed, 25 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
index bdea37f..1cdfffe 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
@@ -70,7 +70,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
* <li>socket.timeout.ms</li>
* <li>socket.receive.buffer.bytes</li>
* <li>fetch.message.max.bytes</li>
- * <li>auto.offset.reset with the values "latest", "earliest" (unlike 0.8.2 behavior)</li>
+ * <li>auto.offset.reset with the values "largest", "smallest"</li>
* <li>fetch.wait.max.ms</li>
* </ul>
* </li>
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
index fe7f777..10f4c41 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
@@ -576,7 +576,8 @@ public class LegacyFetcher implements Fetcher {
private static long getInvalidOffsetBehavior(Properties config) {
long timeType;
- if (config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest").equals("latest")) {
+ String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest");
+ if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9
timeType = OffsetRequest.LatestTime();
} else {
timeType = OffsetRequest.EarliestTime();
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
index 6a2fa27..a3e815e 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java
@@ -93,13 +93,13 @@ public class Kafka08ITCase extends KafkaConsumerTestBase {
// set invalid offset:
CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl)kafkaServer).createCuratorClient();
- ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardCC.groupId(), topic, 0, 1234);
+ ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234);
curatorClient.close();
// read from topic
final int valuesCount = 20;
final int startFrom = 0;
- readSequence(env, standardCC.props().props(), parallelism, topic, valuesCount, startFrom);
+ readSequence(env, standardProps, parallelism, topic, valuesCount, startFrom);
deleteTestTopic(topic);
}
@@ -188,9 +188,9 @@ public class Kafka08ITCase extends KafkaConsumerTestBase {
CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl)kafkaServer).createCuratorClient();
- long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardCC.groupId(), topicName, 0);
- long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardCC.groupId(), topicName, 1);
- long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardCC.groupId(), topicName, 2);
+ long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 0);
+ long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 1);
+ long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 2);
LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
@@ -201,9 +201,9 @@ public class Kafka08ITCase extends KafkaConsumerTestBase {
LOG.info("Manipulating offsets");
// set the offset to 50 for the three partitions
- ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardCC.groupId(), topicName, 0, 49);
- ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardCC.groupId(), topicName, 1, 49);
- ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardCC.groupId(), topicName, 2, 49);
+ ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 0, 49);
+ ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 1, 49);
+ ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topicName, 2, 49);
curatorClient.close();
@@ -250,9 +250,9 @@ public class Kafka08ITCase extends KafkaConsumerTestBase {
// get the offset
CuratorFramework curatorFramework = ((KafkaTestEnvironmentImpl)kafkaServer).createCuratorClient();
- long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardCC.groupId(), topicName, 0);
- long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardCC.groupId(), topicName, 1);
- long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardCC.groupId(), topicName, 2);
+ long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 0);
+ long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 1);
+ long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 2);
LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
index 348b75d..6f56ede 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
@@ -21,7 +21,6 @@ package org.apache.flink.streaming.connectors.kafka;
import kafka.admin.AdminUtils;
import kafka.api.PartitionMetadata;
import kafka.common.KafkaException;
-import kafka.consumer.ConsumerConfig;
import kafka.network.SocketServer;
import kafka.server.KafkaConfig;
import kafka.server.KafkaServer;
@@ -68,19 +67,11 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
private String zookeeperConnectionString;
private String brokerConnectionString = "";
private Properties standardProps;
- private ConsumerConfig standardCC;
-
public String getBrokerConnectionString() {
return brokerConnectionString;
}
-
- @Override
- public ConsumerConfig getStandardConsumerConfig() {
- return standardCC;
- }
-
@Override
public Properties getStandardProperties() {
return standardProps;
@@ -187,13 +178,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
standardProps.setProperty("auto.commit.enable", "false");
standardProps.setProperty("zookeeper.session.timeout.ms", "12000"); // 6 seconds is default. Seems to be too small for travis.
standardProps.setProperty("zookeeper.connection.timeout.ms", "20000");
- standardProps.setProperty("auto.offset.reset", "earliest"); // read from the beginning.
+ standardProps.setProperty("auto.offset.reset", "smallest"); // read from the beginning. (smallest is kafka 0.8)
standardProps.setProperty("fetch.message.max.bytes", "256"); // make a lot of fetches (MESSAGES MUST BE SMALLER!)
-
- Properties consumerConfigProps = new Properties();
- consumerConfigProps.putAll(standardProps);
- consumerConfigProps.setProperty("auto.offset.reset", "smallest");
- standardCC = new ConsumerConfig(consumerConfigProps);
}
@Override
@@ -274,8 +260,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
}
private ZkClient createZkClient() {
- return new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(),
- standardCC.zkConnectionTimeoutMs(), new ZooKeeperStringSerializer());
+ return new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
+ Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
index 0855ba6..50dcab8 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
@@ -65,19 +65,12 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
private String zookeeperConnectionString;
private String brokerConnectionString = "";
private Properties standardProps;
- private ConsumerConfig standardCC;
-
public String getBrokerConnectionString() {
return brokerConnectionString;
}
@Override
- public ConsumerConfig getStandardConsumerConfig() {
- return standardCC;
- }
-
- @Override
public Properties getStandardProperties() {
return standardProps;
}
@@ -184,13 +177,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
standardProps.setProperty("auto.commit.enable", "false");
standardProps.setProperty("zookeeper.session.timeout.ms", "12000"); // 6 seconds is default. Seems to be too small for travis.
standardProps.setProperty("zookeeper.connection.timeout.ms", "20000");
- standardProps.setProperty("auto.offset.reset", "earliest"); // read from the beginning.
+ standardProps.setProperty("auto.offset.reset", "earliest"); // read from the beginning. (earliest is kafka 0.9 value)
standardProps.setProperty("fetch.message.max.bytes", "256"); // make a lot of fetches (MESSAGES MUST BE SMALLER!)
-
- Properties consumerConfigProps = new Properties();
- consumerConfigProps.putAll(standardProps);
- consumerConfigProps.setProperty("auto.offset.reset", "smallest");
- standardCC = new ConsumerConfig(consumerConfigProps);
}
@Override
@@ -233,8 +221,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
}
public ZkUtils getZkUtils() {
- ZkClient creator = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(),
- standardCC.zkConnectionTimeoutMs(), new ZooKeeperStringSerializer());
+ ZkClient creator = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
+ Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());
return ZkUtils.apply(creator, false);
}
@@ -280,8 +268,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
try {
LOG.info("Deleting topic {}", topic);
- ZkClient zk = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(),
- standardCC.zkConnectionTimeoutMs(), new ZooKeeperStringSerializer());
+ ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
+ Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());
AdminUtils.deleteTopic(zkUtils, topic);
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
index 2d9f2fc..680e4ec 100644
--- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
+++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java
@@ -1284,11 +1284,10 @@ public abstract class KafkaConsumerTestBase extends KafkaTestBase {
throws IOException
{
// write the sequence to log for debugging purposes
- Properties stdProps = standardCC.props().props();
- Properties newProps = new Properties(stdProps);
+ Properties newProps = new Properties(standardProps);
newProps.setProperty("group.id", "topic-printer"+ UUID.randomUUID().toString());
newProps.setProperty("auto.offset.reset", "smallest");
- newProps.setProperty("zookeeper.connect", standardCC.zkConnect());
+ newProps.setProperty("zookeeper.connect", standardProps.getProperty("zookeeper.connect"));
ConsumerConfig printerConfig = new ConsumerConfig(newProps);
printTopic(topicName, printerConfig, deserializer, elements);
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
index 73cd2f9..ab1d5b6 100644
--- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
+++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java
@@ -66,7 +66,6 @@ public abstract class KafkaTestBase extends TestLogger {
protected static String brokerConnectionStrings;
- protected static ConsumerConfig standardCC;
protected static Properties standardProps;
protected static ForkableFlinkMiniCluster flink;
@@ -98,7 +97,6 @@ public abstract class KafkaTestBase extends TestLogger {
kafkaServer.prepare(NUMBER_OF_KAFKA_SERVERS);
standardProps = kafkaServer.getStandardProperties();
- standardCC = kafkaServer.getStandardConsumerConfig();
brokerConnectionStrings = kafkaServer.getBrokerConnectionString();
// start also a re-usable Flink mini cluster
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java
index 40be8a1..76a284b 100644
--- a/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java
+++ b/flink-streaming-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironment.java
@@ -17,7 +17,6 @@
package org.apache.flink.streaming.connectors.kafka;
-import kafka.consumer.ConsumerConfig;
import kafka.server.KafkaServer;
import org.apache.flink.streaming.connectors.kafka.partitioner.KafkaPartitioner;
import org.apache.flink.streaming.util.serialization.DeserializationSchema;
@@ -25,7 +24,6 @@ import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema;
import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchemaWrapper;
import org.apache.flink.streaming.util.serialization.KeyedSerializationSchema;
-import java.net.UnknownHostException;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
@@ -45,7 +43,6 @@ public abstract class KafkaTestEnvironment {
public abstract void createTestTopic(String topic, int numberOfPartitions, int replicationFactor);
- public abstract ConsumerConfig getStandardConsumerConfig();
public abstract Properties getStandardProperties();
http://git-wip-us.apache.org/repos/asf/flink/blob/9173825a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index 8c9a9c7..98dc85f 100644
--- a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -155,6 +155,7 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
Assert.assertEquals(1, apps.size()); // Only one running
ApplicationReport app = apps.get(0);
+
Assert.assertEquals("MyCustomName", app.getName());
ApplicationId id = app.getApplicationId();
yc.killApplication(id);
[2/6] flink git commit: [FLINK-3350] [tests] Increase default test
Akka ask and ZooKeeper timeouts
Posted by se...@apache.org.
[FLINK-3350] [tests] Increase default test Akka ask and ZooKeeper timeouts
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b8f40251
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b8f40251
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b8f40251
Branch: refs/heads/master
Commit: b8f40251c6c45379118254c21b0d553c2d3b8937
Parents: 9173825
Author: Ufuk Celebi <uc...@apache.org>
Authored: Mon Feb 8 14:24:43 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:26:43 2016 +0100
----------------------------------------------------------------------
.../runtime/minicluster/FlinkMiniCluster.scala | 20 ++++++++++++++++++--
.../minicluster/LocalFlinkMiniCluster.scala | 2 ++
.../runtime/testutils/ZooKeeperTestUtils.java | 5 +++--
.../runtime/testingUtils/TestingCluster.scala | 2 ++
.../kafka/KafkaTestEnvironmentImpl.java | 7 ++++---
.../kafka/KafkaTestEnvironmentImpl.java | 7 ++++---
...ctTaskManagerProcessFailureRecoveryTest.java | 3 +++
.../JobManagerCheckpointRecoveryITCase.java | 8 ++++++--
.../recovery/ProcessFailureCancelingITCase.java | 2 +-
9 files changed, 43 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
index 4cdda3f..0346d6d 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
@@ -42,7 +42,7 @@ import org.apache.flink.runtime.webmonitor.{WebMonitorUtils, WebMonitor}
import org.slf4j.LoggerFactory
-import scala.concurrent.duration.FiniteDuration
+import scala.concurrent.duration.{Duration, FiniteDuration}
import scala.concurrent._
import scala.concurrent.forkjoin.ForkJoinPool
@@ -86,7 +86,7 @@ abstract class FlinkMiniCluster(
implicit val executionContext = ExecutionContext.global
- implicit val timeout = AkkaUtils.getTimeout(userConfiguration)
+ implicit val timeout = AkkaUtils.getTimeout(configuration)
val recoveryMode = RecoveryMode.fromConfig(configuration)
@@ -188,6 +188,22 @@ abstract class FlinkMiniCluster(
AkkaUtils.getAkkaConfig(configuration, Some((hostname, resolvedPort)))
}
+ /**
+ * Sets CI environment (Travis) specific config defaults.
+ */
+ def setDefaultCiConfig(config: Configuration) : Unit = {
+ // https://docs.travis-ci.com/user/environment-variables#Default-Environment-Variables
+ if (sys.env.contains("CI")) {
+ // Only set if nothing specified in config
+ if (config.getString(ConfigConstants.AKKA_ASK_TIMEOUT, null) == null) {
+ val duration = Duration(ConfigConstants.DEFAULT_AKKA_ASK_TIMEOUT) * 10
+ config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, s"${duration.toSeconds}s")
+
+ LOG.info(s"Akka ask timeout set to ${duration.toSeconds}s")
+ }
+ }
+ }
+
// --------------------------------------------------------------------------
// Start/Stop Methods
// --------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index 913aec0..c803429 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -48,6 +48,8 @@ class LocalFlinkMiniCluster(
override def generateConfiguration(userConfiguration: Configuration): Configuration = {
val config = getDefaultConfig
+ setDefaultCiConfig(config)
+
config.addAll(userConfiguration)
setMemory(config)
initializeIOFormatClasses(config)
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java
index 6c33835..75569ec 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/ZooKeeperTestUtils.java
@@ -70,9 +70,9 @@ public class ZooKeeperTestUtils {
config.setString(ConfigConstants.ZOOKEEPER_QUORUM_KEY, zooKeeperQuorum);
int connTimeout = 5000;
- if (System.getenv().get("CI") != null) {
+ if (System.getenv().containsKey("CI")) {
// The regular timeout is to aggressive for Travis and connections are often lost.
- connTimeout = 20000;
+ connTimeout = 30000;
}
config.setInteger(ConfigConstants.ZOOKEEPER_CONNECTION_TIMEOUT, connTimeout);
@@ -87,6 +87,7 @@ public class ZooKeeperTestUtils {
config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "1000 ms");
config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "6 s");
config.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 9);
+ config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
config.setString(ConfigConstants.EXECUTION_RETRY_DELAY_KEY, "10 s");
return config;
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
index 5eee4e5..bd56040 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
@@ -63,6 +63,8 @@ class TestingCluster(
cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 10)
cfg.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, -1)
+ setDefaultCiConfig(cfg)
+
cfg.addAll(userConfig)
cfg
}
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
index 6f56ede..3d9e4f5 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
@@ -176,8 +176,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
standardProps.setProperty("bootstrap.servers", brokerConnectionString);
standardProps.setProperty("group.id", "flink-tests");
standardProps.setProperty("auto.commit.enable", "false");
- standardProps.setProperty("zookeeper.session.timeout.ms", "12000"); // 6 seconds is default. Seems to be too small for travis.
- standardProps.setProperty("zookeeper.connection.timeout.ms", "20000");
+ standardProps.setProperty("zookeeper.session.timeout.ms", "30000"); // 6 seconds is default. Seems to be too small for travis.
+ standardProps.setProperty("zookeeper.connection.timeout.ms", "30000");
standardProps.setProperty("auto.offset.reset", "smallest"); // read from the beginning. (smallest is kafka 0.8)
standardProps.setProperty("fetch.message.max.bytes", "256"); // make a lot of fetches (MESSAGES MUST BE SMALLER!)
}
@@ -292,7 +292,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
// for CI stability, increase zookeeper session timeout
- kafkaProperties.put("zookeeper.session.timeout.ms", "20000");
+ kafkaProperties.put("zookeeper.session.timeout.ms", "30000");
+ kafkaProperties.put("zookeeper.connection.timeout.ms", "30000");
final int numTries = 5;
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
index 50dcab8..9dfd021 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java
@@ -175,8 +175,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
standardProps.setProperty("bootstrap.servers", brokerConnectionString);
standardProps.setProperty("group.id", "flink-tests");
standardProps.setProperty("auto.commit.enable", "false");
- standardProps.setProperty("zookeeper.session.timeout.ms", "12000"); // 6 seconds is default. Seems to be too small for travis.
- standardProps.setProperty("zookeeper.connection.timeout.ms", "20000");
+ standardProps.setProperty("zookeeper.session.timeout.ms", "30000"); // 6 seconds is default. Seems to be too small for travis.
+ standardProps.setProperty("zookeeper.connection.timeout.ms", "30000");
standardProps.setProperty("auto.offset.reset", "earliest"); // read from the beginning. (earliest is kafka 0.9 value)
standardProps.setProperty("fetch.message.max.bytes", "256"); // make a lot of fetches (MESSAGES MUST BE SMALLER!)
}
@@ -296,7 +296,8 @@ public class KafkaTestEnvironmentImpl extends KafkaTestEnvironment {
kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
// for CI stability, increase zookeeper session timeout
- kafkaProperties.put("zookeeper.session.timeout.ms", "20000");
+ kafkaProperties.put("zookeeper.session.timeout.ms", "30000");
+ kafkaProperties.put("zookeeper.connection.timeout.ms", "30000");
final int numTries = 5;
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
index 5dd870f..7b4c9b2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.jobmanager.JobManager;
import org.apache.flink.runtime.jobmanager.MemoryArchivist;
import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.minicluster.FlinkMiniCluster;
import org.apache.flink.runtime.taskmanager.TaskManager;
import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.apache.flink.util.NetUtils;
@@ -122,6 +123,7 @@ public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends Test
jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "6 s");
jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 9);
jmConfig.setString(ConfigConstants.EXECUTION_RETRY_DELAY_KEY, "10 s");
+ jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<Tuple2<String, Object>>(localAddress));
ActorRef jmActor = JobManager.startJobManagerActors(
@@ -376,6 +378,7 @@ public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends Test
cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 4);
cfg.setInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, 100);
cfg.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
+ cfg.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
TaskManager.selectNetworkInterfaceAndRunTaskManager(cfg, TaskManager.class);
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
index 737d39a..5d1b2c5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
@@ -156,6 +156,7 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
Configuration config = ZooKeeperTestUtils.createZooKeeperRecoveryModeConfig(ZooKeeper
.getConnectString(), FileStateBackendBasePath.getAbsoluteFile().toURI().toString());
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, Parallelism);
+ config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
ActorSystem testSystem = null;
JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
@@ -182,7 +183,8 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
leaderRetrievalService.start(leaderListener);
// The task manager
- taskManagerSystem = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
+ taskManagerSystem = AkkaUtils.createActorSystem(
+ config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
TaskManager.startTaskManagerComponentsAndActor(
config, taskManagerSystem, "localhost",
Option.<String>empty(), Option.<LeaderRetrievalService>empty(),
@@ -297,6 +299,7 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
fileStateBackendPath);
config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 2);
+ config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
LeaderRetrievalService leaderRetrievalService = null;
@@ -321,7 +324,8 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
leaderRetrievalService.start(leaderListener);
// The task manager
- taskManagerSystem = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
+ taskManagerSystem = AkkaUtils.createActorSystem(
+ config, Option.apply(new Tuple2<String, Object>("localhost", 0)));
TaskManager.startTaskManagerComponentsAndActor(
config, taskManagerSystem, "localhost",
Option.<String>empty(), Option.<LeaderRetrievalService>empty(),
http://git-wip-us.apache.org/repos/asf/flink/blob/b8f40251/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
index 37e4e38..2b49c08 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
@@ -97,7 +97,7 @@ public class ProcessFailureCancelingITCase {
jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "5 s");
jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "2000 s");
jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 10);
- jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "10 s");
+ jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<Tuple2<String, Object>>(localAddress));
ActorRef jmActor = JobManager.startJobManagerActors(
[3/6] flink git commit: [hotfix] [tests] Ignore ZooKeeper logs in
process tests
Posted by se...@apache.org.
[hotfix] [tests] Ignore ZooKeeper logs in process tests
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3a643c07
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3a643c07
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3a643c07
Branch: refs/heads/master
Commit: 3a643c07792c62142c1f8cda172d4f4c3442c9b3
Parents: b8f4025
Author: Ufuk Celebi <uc...@apache.org>
Authored: Tue Feb 9 11:01:39 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:27:16 2016 +0100
----------------------------------------------------------------------
.../JobManagerSubmittedJobGraphsRecoveryITCase.java | 6 +++++-
.../org/apache/flink/runtime/testutils/CommonTestUtils.java | 1 +
.../AbstractJobManagerProcessFailureRecoveryITCase.java | 8 +++++---
.../org/apache/flink/test/recovery/ChaosMonkeyITCase.java | 4 ++++
.../test/recovery/JobManagerCheckpointRecoveryITCase.java | 8 ++++++++
5 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java
index 99f7bd7..59c7c39 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerSubmittedJobGraphsRecoveryITCase.java
@@ -343,6 +343,10 @@ public class JobManagerSubmittedJobGraphsRecoveryITCase extends TestLogger {
assertEquals(2, jobSubmitSuccessMessages);
}
catch (Throwable t) {
+ // Print early (in some situations the process logs get too big
+ // for Travis and the root problem is not shown)
+ t.printStackTrace();
+
// In case of an error, print the job manager process logs.
if (jobManagerProcess[0] != null) {
jobManagerProcess[0].printProcessLog();
@@ -352,7 +356,7 @@ public class JobManagerSubmittedJobGraphsRecoveryITCase extends TestLogger {
jobManagerProcess[1].printProcessLog();
}
- t.printStackTrace();
+ throw t;
}
finally {
if (jobManagerProcess[0] != null) {
http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
index 069b6af..bbb6a89 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
@@ -147,6 +147,7 @@ public class CommonTestUtils {
writer.println("log4j.appender.console.layout=org.apache.log4j.PatternLayout");
writer.println("log4j.appender.console.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n");
writer.println("log4j.logger.org.eclipse.jetty.util.log=OFF");
+ writer.println("log4j.logger.org.apache.zookeeper=OFF");
writer.flush();
writer.close();
http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java
index 2f6b762..6122352 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractJobManagerProcessFailureRecoveryITCase.java
@@ -246,8 +246,10 @@ public abstract class AbstractJobManagerProcessFailureRecoveryITCase extends Tes
fail("The program encountered a " + error.getClass().getSimpleName() + " : " + error.getMessage());
}
}
- catch (Exception e) {
- e.printStackTrace();
+ catch (Throwable t) {
+ // Print early (in some situations the process logs get too big
+ // for Travis and the root problem is not shown)
+ t.printStackTrace();
for (JobManagerProcess p : jmProcess) {
if (p != null) {
@@ -255,7 +257,7 @@ public abstract class AbstractJobManagerProcessFailureRecoveryITCase extends Tes
}
}
- fail(e.getMessage());
+ throw t;
}
finally {
for (int i = 0; i < numberOfTaskManagers; i++) {
http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
index 6ae0d46..76f96ef 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ChaosMonkeyITCase.java
@@ -320,6 +320,10 @@ public class ChaosMonkeyITCase extends TestLogger {
LOG.info("Recovery state clean");
}
catch (Throwable t) {
+ // Print early (in some situations the process logs get too big
+ // for Travis and the root problem is not shown)
+ t.printStackTrace();
+
System.out.println("#################################################");
System.out.println(" TASK MANAGERS");
System.out.println("#################################################");
http://git-wip-us.apache.org/repos/asf/flink/blob/3a643c07/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
index 5d1b2c5..59a05ff 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
@@ -248,6 +248,10 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
}
}
catch (Throwable t) {
+ // Print early (in some situations the process logs get too big
+ // for Travis and the root problem is not shown)
+ t.printStackTrace();
+
// In case of an error, print the job manager process logs.
if (jobManagerProcess[0] != null) {
jobManagerProcess[0].printProcessLog();
@@ -400,6 +404,10 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
assertTrue("Did not find expected output in logs.", success);
}
catch (Throwable t) {
+ // Print early (in some situtations the process logs get too big
+ // for Travis and the root problem is not shown)
+ t.printStackTrace();
+
// In case of an error, print the job manager process logs.
if (jobManagerProcess[0] != null) {
jobManagerProcess[0].printProcessLog();
[5/6] flink git commit: [hotfix] [tests] Reset state to allow retry
on failure
Posted by se...@apache.org.
[hotfix] [tests] Reset state to allow retry on failure
This closes #1611
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/48b74546
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/48b74546
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/48b74546
Branch: refs/heads/master
Commit: 48b745466202ebbb68608930e13cb6ed4a35e6e7
Parents: 756cbaf
Author: Ufuk Celebi <uc...@apache.org>
Authored: Tue Feb 9 12:45:41 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:27:41 2016 +0100
----------------------------------------------------------------------
.../JobManagerCheckpointRecoveryITCase.java | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/48b74546/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
index 59a05ff..ea30c58 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerCheckpointRecoveryITCase.java
@@ -116,15 +116,15 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
private static final int Parallelism = 8;
- private static final CountDownLatch CompletedCheckpointsLatch = new CountDownLatch(2);
+ private static CountDownLatch CompletedCheckpointsLatch = new CountDownLatch(2);
- private static final AtomicLongArray RecoveredStates = new AtomicLongArray(Parallelism);
+ private static AtomicLongArray RecoveredStates = new AtomicLongArray(Parallelism);
- private static final CountDownLatch FinalCountLatch = new CountDownLatch(1);
+ private static CountDownLatch FinalCountLatch = new CountDownLatch(1);
- private static final AtomicReference<Long> FinalCount = new AtomicReference<>();
+ private static AtomicReference<Long> FinalCount = new AtomicReference<>();
- private static final long LastElement = -1;
+ private static long LastElement = -1;
/**
* Simple checkpointed streaming sum.
@@ -156,7 +156,6 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
Configuration config = ZooKeeperTestUtils.createZooKeeperRecoveryModeConfig(ZooKeeper
.getConnectString(), FileStateBackendBasePath.getAbsoluteFile().toURI().toString());
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, Parallelism);
- config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
ActorSystem testSystem = null;
JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
@@ -248,6 +247,13 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
}
}
catch (Throwable t) {
+ // Reset all static state for test retries
+ CompletedCheckpointsLatch = new CountDownLatch(2);
+ RecoveredStates = new AtomicLongArray(Parallelism);
+ FinalCountLatch = new CountDownLatch(1);
+ FinalCount = new AtomicReference<>();
+ LastElement = -1;
+
// Print early (in some situations the process logs get too big
// for Travis and the root problem is not shown)
t.printStackTrace();
@@ -303,7 +309,6 @@ public class JobManagerCheckpointRecoveryITCase extends TestLogger {
fileStateBackendPath);
config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 2);
- config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100 s");
JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
LeaderRetrievalService leaderRetrievalService = null;
[4/6] flink git commit: [hotfix] [tests] Log retry rule failures on
warn level
Posted by se...@apache.org.
[hotfix] [tests] Log retry rule failures on warn level
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/756cbaff
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/756cbaff
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/756cbaff
Branch: refs/heads/master
Commit: 756cbafff1fd25f67268ca84b62c8a479156bf88
Parents: 3a643c0
Author: Ufuk Celebi <uc...@apache.org>
Authored: Tue Feb 9 11:25:37 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:27:28 2016 +0100
----------------------------------------------------------------------
.../test/java/org/apache/flink/testutils/junit/RetryRule.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/756cbaff/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java b/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
index a4aff86..2b3a37a 100644
--- a/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
+++ b/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
@@ -113,7 +113,7 @@ public class RetryRule implements TestRule {
break; // success
}
catch (Throwable t) {
- LOG.debug(String.format("Test run failed (%d/%d).",
+ LOG.warn(String.format("Test run failed (%d/%d).",
currentRun, timesOnFailure + 1), t);
// Throw the failure if retried too often
@@ -156,7 +156,7 @@ public class RetryRule implements TestRule {
break; // success
}
catch (Throwable t) {
- LOG.debug(String.format("Test run failed (%d/%d).", currentRun, timesOnFailure + 1), t);
+ LOG.warn(String.format("Test run failed (%d/%d).", currentRun, timesOnFailure + 1), t);
if (!exceptionClass.isAssignableFrom(t.getClass()) || currentRun >= timesOnFailure) {
// Throw the failure if retried too often, or if it is the wrong exception
[6/6] flink git commit: [FLINK-3260] [runtime] Enforce terminal state
of Executions
Posted by se...@apache.org.
[FLINK-3260] [runtime] Enforce terminal state of Executions
This commit fixes the problem that Executions could leave their terminal state
FINISHED to transition to FAILED. Such a transition will be propagated to the
ExecutionGraph where it entails JobStatus changes. Since the Execution already
reached a terminal state, it should not again affect the ExecutionGraph. This
can lead to an inconsistent state in case of a restart where the old Executions
get disassociated from the ExecutionGraph.
This closes #1613
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6968a57a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6968a57a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6968a57a
Branch: refs/heads/master
Commit: 6968a57a1a31a11b33bacd2c94d6559bcabd6eb9
Parents: 48b7454
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Feb 9 10:30:12 2016 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 10 15:34:37 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/executiongraph/Execution.java | 14 +-
.../ExecutionGraphRestartTest.java | 90 +++++++++++++
.../runtime/testingUtils/TestingCluster.scala | 6 +-
.../testingUtils/TestingTaskManagerLike.scala | 4 +-
.../runtime/testingUtils/TestingUtils.scala | 133 ++++++++++++++++++-
5 files changed, 233 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index eb2e68c..db037bb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -435,7 +435,7 @@ public class Execution implements Serializable {
return;
}
else if (current == CREATED || current == SCHEDULED) {
- // from here, we can directly switch to cancelled, because the no task has been deployed
+ // from here, we can directly switch to cancelled, because no task has been deployed
if (transitionState(current, CANCELED)) {
// we skip the canceling state. set the timestamp, for a consistent appearance
@@ -754,11 +754,10 @@ public class Execution implements Serializable {
return false;
}
- if (current == CANCELED) {
- // we are already aborting or are already aborted
+ if (current == CANCELED || current == FINISHED) {
+ // we are already aborting or are already aborted or we are already finished
if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("Ignoring transition of vertex %s to %s while being %s",
- getVertexWithAttempt(), FAILED, CANCELED));
+ LOG.debug("Ignoring transition of vertex {} to {} while being {}.", getVertexWithAttempt(), FAILED, current);
}
return false;
}
@@ -928,6 +927,11 @@ public class Execution implements Serializable {
}
private boolean transitionState(ExecutionState currentState, ExecutionState targetState, Throwable error) {
+ // sanity check
+ if (currentState.isTerminal()) {
+ throw new IllegalStateException("Cannot leave terminal state " + currentState + " to transition to " + targetState + ".");
+ }
+
if (STATE_UPDATER.compareAndSet(this, currentState, targetState)) {
markTimestamp(targetState);
http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
index 0c3af8f..47a48a2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
@@ -36,6 +36,7 @@ import org.junit.Test;
import scala.concurrent.duration.Deadline;
import scala.concurrent.duration.FiniteDuration;
+import java.util.Iterator;
import java.util.concurrent.TimeUnit;
import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.SimpleActorGateway;
@@ -387,6 +388,95 @@ public class ExecutionGraphRestartTest {
assertEquals(1, eg.getNumberOfRetriesLeft());
}
+ /**
+ * Tests that a failing execution does not affect a restarted job. This is important if a
+ * callback handler fails an execution after it has already reached a final state and the job
+ * has been restarted.
+ */
+ @Test
+ public void testFailingExecutionAfterRestart() throws Exception {
+ Instance instance = ExecutionGraphTestUtils.getInstance(
+ new SimpleActorGateway(TestingUtils.directExecutionContext()),
+ 2);
+
+ Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
+ scheduler.newInstanceAvailable(instance);
+
+ JobVertex sender = new JobVertex("Task1");
+ sender.setInvokableClass(Tasks.NoOpInvokable.class);
+ sender.setParallelism(1);
+
+ JobVertex receiver = new JobVertex("Task2");
+ receiver.setInvokableClass(Tasks.NoOpInvokable.class);
+ receiver.setParallelism(1);
+
+ JobGraph jobGraph = new JobGraph("Pointwise job", sender, receiver);
+
+ ExecutionGraph eg = new ExecutionGraph(
+ TestingUtils.defaultExecutionContext(),
+ new JobID(),
+ "test job",
+ new Configuration(),
+ AkkaUtils.getDefaultTimeout());
+ eg.setNumberOfRetriesLeft(1);
+ eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());
+
+ assertEquals(JobStatus.CREATED, eg.getState());
+
+ eg.scheduleForExecution(scheduler);
+ assertEquals(JobStatus.RUNNING, eg.getState());
+
+ Iterator<ExecutionVertex> executionVertices = eg.getAllExecutionVertices().iterator();
+
+ Execution finishedExecution = executionVertices.next().getCurrentExecutionAttempt();
+ Execution failedExecution = executionVertices.next().getCurrentExecutionAttempt();
+
+ finishedExecution.markFinished();
+ failedExecution.fail(new Exception("Test Exception"));
+
+ failedExecution.cancelingComplete();
+
+ FiniteDuration timeout = new FiniteDuration(2, TimeUnit.MINUTES);
+
+ Deadline deadline = timeout.fromNow();
+
+ while (deadline.hasTimeLeft() && eg.getState() != JobStatus.RUNNING) {
+ Thread.sleep(100);
+ }
+
+ assertEquals(JobStatus.RUNNING, eg.getState());
+
+ // Wait for deploying after async restart
+ deadline = timeout.fromNow();
+ boolean success = false;
+
+ while (deadline.hasTimeLeft() && !success) {
+ success = true;
+
+ for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
+ if (vertex.getCurrentExecutionAttempt().getAssignedResource() == null) {
+ success = false;
+ Thread.sleep(100);
+ break;
+ } else {
+ vertex.getCurrentExecutionAttempt().switchToRunning();
+ }
+ }
+ }
+
+ // fail old finished execution, this should not affect the execution
+ finishedExecution.fail(new Exception("This should have no effect"));
+
+ for (ExecutionVertex vertex: eg.getAllExecutionVertices()) {
+ vertex.getCurrentExecutionAttempt().markFinished();
+ }
+
+ // the state of the finished execution should have not changed since it is terminal
+ assertEquals(ExecutionState.FINISHED, finishedExecution.getState());
+
+ assertEquals(JobStatus.FINISHED, eg.getState());
+ }
+
private static void restartAfterFailure(ExecutionGraph eg, FiniteDuration timeout, boolean haltAfterRestart) throws InterruptedException {
eg.getAllExecutionVertices().iterator().next().fail(new Exception("Test Exception"));
http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
index bd56040..22b0d29 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
@@ -42,9 +42,9 @@ import scala.concurrent.{Await, Future}
* otherwise false
*/
class TestingCluster(
- userConfiguration: Configuration,
- singleActorSystem: Boolean,
- synchronousDispatcher: Boolean)
+ userConfiguration: Configuration,
+ singleActorSystem: Boolean,
+ synchronousDispatcher: Boolean)
extends FlinkMiniCluster(
userConfiguration,
singleActorSystem) {
http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
index c10e83e..e9dbdde 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManagerLike.scala
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.testingUtils
import akka.actor.{Terminated, ActorRef}
import org.apache.flink.api.common.JobID
import org.apache.flink.runtime.FlinkActor
-import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor
import org.apache.flink.runtime.execution.ExecutionState
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
import org.apache.flink.runtime.messages.JobManagerMessages.{ResponseLeaderSessionID,
@@ -32,8 +31,7 @@ AcknowledgeRegistration}
import org.apache.flink.runtime.messages.TaskMessages.{SubmitTask, UpdateTaskExecutionState, TaskInFinalState}
import org.apache.flink.runtime.taskmanager.TaskManager
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobRemoved
-import org.apache.flink.runtime.testingUtils.TestingMessages.{DisableDisconnect,
-CheckIfJobRemoved, Alive}
+import org.apache.flink.runtime.testingUtils.TestingMessages.{DisableDisconnect, CheckIfJobRemoved, Alive}
import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages._
import scala.concurrent.duration._
http://git-wip-us.apache.org/repos/asf/flink/blob/6968a57a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index 98faa34..679dc71 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -26,8 +26,11 @@ import com.google.common.util.concurrent.MoreExecutors
import com.typesafe.config.ConfigFactory
import grizzled.slf4j.Logger
+import org.apache.flink.api.common.JobExecutionResult
import org.apache.flink.configuration.{ConfigConstants, Configuration}
+import org.apache.flink.runtime.client.JobClient
+import org.apache.flink.runtime.jobgraph.JobGraph
import org.apache.flink.runtime.jobmanager.{MemoryArchivist, JobManager}
import org.apache.flink.runtime.{LogMessages, LeaderSessionMessageFilter, FlinkActor}
import org.apache.flink.runtime.akka.AkkaUtils
@@ -168,6 +171,26 @@ object TestingUtils {
}
def createTaskManager(
+ actorSystem: ActorSystem,
+ jobManager: ActorGateway,
+ configuration: Configuration,
+ useLocalCommunication: Boolean,
+ waitForRegistration: Boolean,
+ taskManagerClass: Class[_ <: TaskManager])
+ : ActorGateway = {
+ val jobManagerURL = AkkaUtils.getAkkaURL(actorSystem, jobManager.actor)
+
+ createTaskManager(
+ actorSystem,
+ jobManagerURL,
+ configuration,
+ useLocalCommunication,
+ waitForRegistration,
+ taskManagerClass
+ )
+ }
+
+ def createTaskManager(
actorSystem: ActorSystem,
jobManager: ActorGateway,
configuration: Configuration,
@@ -200,11 +223,30 @@ object TestingUtils {
* @return ActorGateway of the created TaskManager
*/
def createTaskManager(
+ actorSystem: ActorSystem,
+ jobManagerURL: String,
+ configuration: Configuration,
+ useLocalCommunication: Boolean,
+ waitForRegistration: Boolean)
+ : ActorGateway = {
+ createTaskManager(
+ actorSystem,
+ jobManagerURL,
+ configuration,
+ useLocalCommunication,
+ waitForRegistration,
+ classOf[TestingTaskManager]
+ )
+ }
+
+
+ def createTaskManager(
actorSystem: ActorSystem,
jobManagerURL: String,
configuration: Configuration,
useLocalCommunication: Boolean,
- waitForRegistration: Boolean)
+ waitForRegistration: Boolean,
+ taskManagerClass: Class[_ <: TaskManager])
: ActorGateway = {
val resultingConfiguration = new Configuration()
@@ -222,7 +264,7 @@ object TestingUtils {
None,
leaderRetrievalService,
useLocalCommunication,
- classOf[TestingTaskManager]
+ taskManagerClass
)
if (waitForRegistration) {
@@ -264,6 +306,72 @@ object TestingUtils {
actorSystem: ActorSystem,
configuration: Configuration)
: ActorGateway = {
+ createJobManager(
+ actorSystem,
+ configuration,
+ classOf[TestingJobManager]
+ )
+ }
+
+ def createJobManager(
+ actorSystem: ActorSystem,
+ configuration: Configuration,
+ executionContext: ExecutionContext)
+ : ActorGateway = {
+
+ val (_,
+ instanceManager,
+ scheduler,
+ libraryCacheManager,
+ executionRetries,
+ delayBetweenRetries,
+ timeout,
+ archiveCount,
+ leaderElectionService,
+ submittedJobGraphs,
+ checkpointRecoveryFactory) = JobManager.createJobManagerComponents(
+ configuration,
+ None
+ )
+
+ val archiveProps = Props(classOf[TestingMemoryArchivist], archiveCount)
+
+ val archive: ActorRef = actorSystem.actorOf(archiveProps, JobManager.ARCHIVE_NAME)
+
+ val jobManagerProps = Props(
+ classOf[TestingJobManager],
+ configuration,
+ executionContext,
+ instanceManager,
+ scheduler,
+ libraryCacheManager,
+ archive,
+ executionRetries,
+ delayBetweenRetries,
+ timeout,
+ leaderElectionService,
+ submittedJobGraphs,
+ checkpointRecoveryFactory)
+
+ val jobManager: ActorRef = actorSystem.actorOf(jobManagerProps, JobManager.JOB_MANAGER_NAME)
+
+ new AkkaActorGateway(jobManager, null)
+ }
+
+
+ /**
+ * Creates a JobManager of the given class using the default recovery mode (standalone)
+ *
+ * @param actorSystem ActorSystem to use
+ * @param configuration Configuration to use
+ * @param jobManagerClass JobManager class to instantiate
+ * @return
+ */
+ def createJobManager(
+ actorSystem: ActorSystem,
+ configuration: Configuration,
+ jobManagerClass: Class[_ <: JobManager])
+ : ActorGateway = {
configuration.setString(ConfigConstants.RECOVERY_MODE, ConfigConstants.DEFAULT_RECOVERY_MODE)
@@ -272,7 +380,7 @@ object TestingUtils {
actorSystem,
Some(JobManager.JOB_MANAGER_NAME),
Some(JobManager.ARCHIVE_NAME),
- classOf[TestingJobManager],
+ jobManagerClass,
classOf[MemoryArchivist])
new AkkaActorGateway(actor, null)
@@ -312,6 +420,25 @@ object TestingUtils {
new AkkaActorGateway(actor, null)
}
+ def submitJobAndWait(
+ actorSystem: ActorSystem,
+ jobManager: ActorGateway,
+ jobGraph: JobGraph)
+ : JobExecutionResult = {
+
+ val jobManagerURL = AkkaUtils.getAkkaURL(actorSystem, jobManager.actor)
+ val leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManagerURL)
+
+ JobClient.submitJobAndWait(
+ actorSystem,
+ leaderRetrievalService,
+ jobGraph,
+ TESTING_DURATION,
+ false,
+ Thread.currentThread().getContextClassLoader
+ )
+ }
+
class ForwardingActor(val target: ActorRef, val leaderSessionID: Option[UUID])
extends FlinkActor with LeaderSessionMessageFilter with LogMessages {