You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2017/01/07 04:58:37 UTC
twill git commit: (TWILL-187) Added max start time
Repository: twill
Updated Branches:
refs/heads/master 7d92f0170 -> 5edc8ddbe
(TWILL-187) Added max start time
This closes #22 on Github
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/5edc8ddb
Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/5edc8ddb
Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/5edc8ddb
Branch: refs/heads/master
Commit: 5edc8ddbe0e980b243248704a73ed01ab2c30dc3
Parents: 7d92f01
Author: Terence Yim <ch...@apache.org>
Authored: Fri Jan 6 17:56:53 2017 -0800
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Jan 6 20:58:30 2017 -0800
----------------------------------------------------------------------
.../org/apache/twill/api/TwillPreparer.java | 14 ++++++++++-
.../apache/twill/yarn/YarnTwillController.java | 26 +++++++++-----------
.../twill/yarn/YarnTwillControllerFactory.java | 4 ++-
.../apache/twill/yarn/YarnTwillPreparer.java | 8 +++++-
.../twill/yarn/YarnTwillRunnerService.java | 6 +++--
5 files changed, 39 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/twill/blob/5edc8ddb/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
index 575ba8f..7004d89 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
@@ -22,6 +22,7 @@ import org.apache.twill.api.logging.LogHandler;
import java.net.URI;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
/**
* This interface exposes methods to set up the Twill runtime environment and start a Twill application.
@@ -257,8 +258,19 @@ public interface TwillPreparer {
TwillPreparer setLogLevels(String runnableName, Map<String, LogEntry.Level> logLevelsForRunnable);
/**
- * Starts the application.
+ * Starts the application. It's the same as calling {@link #start(long, TimeUnit)} with timeout of 60 seconds.
+ *
* @return A {@link TwillController} for controlling the running application.
*/
TwillController start();
+
+ /**
+ * Starts the application. The application must be started within the given timeout. An application may fail to start
+ * within a given time if there is insufficient resources.
+ *
+ * @param timeout maximum time to wait for the application to start
+ * @param timeoutUnit unit for the timeout
+ * @return A {@link TwillController} for controlling the running application.
+ */
+ TwillController start(long timeout, TimeUnit timeoutUnit);
}
http://git-wip-us.apache.org/repos/asf/twill/blob/5edc8ddb/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
index c3d2fdc..265c453 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
@@ -58,6 +58,8 @@ final class YarnTwillController extends AbstractTwillController implements Twill
private final String appName;
private final Callable<ProcessController<YarnApplicationReport>> startUp;
+ private final long startTimeout;
+ private final TimeUnit startTimeoutUnit;
private ProcessController<YarnApplicationReport> processController;
private ResourceReportClient resourcesClient;
@@ -70,14 +72,18 @@ final class YarnTwillController extends AbstractTwillController implements Twill
*/
YarnTwillController(String appName, RunId runId, ZKClient zkClient,
Callable<ProcessController<YarnApplicationReport>> startUp) {
- this(appName, runId, zkClient, ImmutableList.<LogHandler>of(), startUp);
+ this(appName, runId, zkClient, ImmutableList.<LogHandler>of(), startUp,
+ Constants.APPLICATION_MAX_START_SECONDS, TimeUnit.SECONDS);
}
YarnTwillController(String appName, RunId runId, ZKClient zkClient, Iterable<LogHandler> logHandlers,
- Callable<ProcessController<YarnApplicationReport>> startUp) {
+ Callable<ProcessController<YarnApplicationReport>> startUp,
+ long startTimeout, TimeUnit startTimeoutUnit) {
super(runId, zkClient, logHandlers);
this.appName = appName;
this.startUp = startUp;
+ this.startTimeout = startTimeout;
+ this.startTimeoutUnit = startTimeoutUnit;
}
@@ -101,23 +107,18 @@ final class YarnTwillController extends AbstractTwillController implements Twill
LOG.info("Application {} with id {} submitted", appName, appId);
YarnApplicationState state = report.getYarnApplicationState();
- Stopwatch stopWatch = new Stopwatch();
- stopWatch.start();
- long maxTime = TimeUnit.MILLISECONDS.convert(Constants.APPLICATION_MAX_START_SECONDS, TimeUnit.SECONDS);
+ Stopwatch stopWatch = new Stopwatch().start();
LOG.debug("Checking yarn application status for {} {}", appName, appId);
- while (!hasRun(state) && stopWatch.elapsedTime(TimeUnit.MILLISECONDS) < maxTime) {
+ while (!hasRun(state) && stopWatch.elapsedTime(startTimeoutUnit) < startTimeout) {
report = processController.getReport();
state = report.getYarnApplicationState();
LOG.debug("Yarn application status for {} {}: {}", appName, appId, state);
TimeUnit.SECONDS.sleep(1);
- stopWatch.reset();
- stopWatch.start();
}
LOG.info("Yarn application {} {} is in state {}", appName, appId, state);
if (state != YarnApplicationState.RUNNING) {
- LOG.info("Yarn application {} {} is not in running state. Shutting down controller.",
- appName, appId, Constants.APPLICATION_MAX_START_SECONDS);
+ LOG.info("Yarn application {} {} is not in running state. Shutting down controller.", appName, appId);
forceShutDown();
} else {
try {
@@ -155,8 +156,7 @@ final class YarnTwillController extends AbstractTwillController implements Twill
// Poll application status from yarn
try (ProcessController<YarnApplicationReport> processController = this.processController) {
- Stopwatch stopWatch = new Stopwatch();
- stopWatch.start();
+ Stopwatch stopWatch = new Stopwatch().start();
long maxTime = TimeUnit.MILLISECONDS.convert(Constants.APPLICATION_MAX_STOP_SECONDS, TimeUnit.SECONDS);
YarnApplicationReport report = processController.getReport();
@@ -166,8 +166,6 @@ final class YarnTwillController extends AbstractTwillController implements Twill
stopWatch.elapsedTime(TimeUnit.MILLISECONDS) < maxTime) {
LOG.debug("Yarn application final status for {} {}: {}", appName, appId, finalStatus);
TimeUnit.SECONDS.sleep(1);
- stopWatch.reset();
- stopWatch.start();
finalStatus = processController.getReport().getFinalApplicationStatus();
}
LOG.debug("Yarn application {} {} completed with status {}", appName, appId, finalStatus);
http://git-wip-us.apache.org/repos/asf/twill/blob/5edc8ddb/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java
index 11c2ae6..61fb7cc 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java
@@ -23,6 +23,7 @@ import org.apache.twill.internal.ProcessController;
import org.apache.twill.internal.yarn.YarnApplicationReport;
import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
/**
* Factory for creating {@link YarnTwillController}.
@@ -30,5 +31,6 @@ import java.util.concurrent.Callable;
interface YarnTwillControllerFactory {
YarnTwillController create(RunId runId, Iterable<LogHandler> logHandlers,
- Callable<ProcessController<YarnApplicationReport>> startUp);
+ Callable<ProcessController<YarnApplicationReport>> startUp,
+ long startTimeout, TimeUnit startTimeoutUnit);
}
http://git-wip-us.apache.org/repos/asf/twill/blob/5edc8ddb/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index 2aa24ab..562d1da 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -102,6 +102,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
@@ -322,6 +323,11 @@ final class YarnTwillPreparer implements TwillPreparer {
@Override
public TwillController start() {
+ return start(Constants.APPLICATION_MAX_START_SECONDS, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public TwillController start(long timeout, TimeUnit timeoutUnit) {
try {
final ProcessLauncher<ApplicationMasterInfo> launcher = yarnAppClient.createLauncher(twillSpec, schedulerQueue);
final ApplicationMasterInfo appMasterInfo = launcher.getContainerInfo();
@@ -376,7 +382,7 @@ final class YarnTwillPreparer implements TwillPreparer {
}
};
- YarnTwillController controller = controllerFactory.create(runId, logHandlers, submitTask);
+ YarnTwillController controller = controllerFactory.create(runId, logHandlers, submitTask, timeout, timeoutUnit);
controller.start();
return controller;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/twill/blob/5edc8ddb/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
index 79edc65..1b7c4b9 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -282,10 +282,12 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
locationFactory, jvmOptions, LogEntry.Level.INFO, new YarnTwillControllerFactory() {
@Override
public YarnTwillController create(RunId runId, Iterable<LogHandler> logHandlers,
- Callable<ProcessController<YarnApplicationReport>> startUp) {
+ Callable<ProcessController<YarnApplicationReport>> startUp,
+ long startTimeout, TimeUnit startTimeoutUnit) {
ZKClient zkClient = ZKClients.namespace(zkClientService, "/" + appName);
YarnTwillController controller = listenController(new YarnTwillController(appName, runId, zkClient,
- logHandlers, startUp));
+ logHandlers, startUp,
+ startTimeout, startTimeoutUnit));
synchronized (YarnTwillRunnerService.this) {
Preconditions.checkArgument(!controllers.contains(appName, runId),
"Application %s with runId %s is already running.", appName, runId);