You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2016/08/29 23:00:07 UTC
twill git commit: TWILL-175 Avoid caching a YarnClient,
since each YarnClient is fixed to a particular UserGroupInformation.
Repository: twill
Updated Branches:
refs/heads/master 44e6dfd93 -> 4dfd44968
TWILL-175 Avoid caching a YarnClient, since each YarnClient is fixed to a particular UserGroupInformation.
This closes #6 on Github
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/4dfd4496
Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/4dfd4496
Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/4dfd4496
Branch: refs/heads/master
Commit: 4dfd44968452518af73380497d5330d15ed28df6
Parents: 44e6dfd
Author: Ali Anwar <an...@berkeley.edu>
Authored: Thu Aug 25 23:45:39 2016 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Aug 29 15:59:55 2016 -0700
----------------------------------------------------------------------
.../twill/internal/ProcessController.java | 2 +-
.../internal/yarn/Hadoop20YarnAppClient.java | 44 ++++++++++++--------
.../internal/yarn/Hadoop21YarnAppClient.java | 42 +++++++++++--------
.../appmaster/RunnableProcessLauncher.java | 5 +++
.../twill/internal/yarn/YarnAppClient.java | 3 +-
.../apache/twill/yarn/YarnTwillController.java | 2 +-
.../twill/yarn/YarnTwillRunnerService.java | 2 -
.../java/org/apache/twill/yarn/TwillTester.java | 2 -
8 files changed, 59 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java b/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java
index 4453838..9a4cbe7 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java
@@ -24,7 +24,7 @@ import org.apache.twill.common.Cancellable;
*
* @param <R> Report type.
*/
-public interface ProcessController<R> extends Cancellable {
+public interface ProcessController<R> extends AutoCloseable, Cancellable {
R getReport();
http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
index bfa494c..bb494d4 100644
--- a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
+++ b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
@@ -59,21 +59,29 @@ import javax.annotation.Nullable;
* Apache Hadoop 2.0.
* </p>
*/
-public final class Hadoop20YarnAppClient extends AbstractIdleService implements YarnAppClient {
+public final class Hadoop20YarnAppClient implements YarnAppClient {
private static final Logger LOG = LoggerFactory.getLogger(Hadoop20YarnAppClient.class);
- private final YarnClient yarnClient;
+ private final Configuration configuration;
private String user;
public Hadoop20YarnAppClient(Configuration configuration) {
- this.yarnClient = new YarnClientImpl();
- yarnClient.init(configuration);
+ this.configuration = configuration;
this.user = System.getProperty("user.name");
}
+ // Creates and starts a yarn client
+ private YarnClient createYarnClient() {
+ YarnClient yarnClient = new YarnClientImpl();
+ yarnClient.init(configuration);
+ yarnClient.start();
+ return yarnClient;
+ }
+
@Override
public ProcessLauncher<ApplicationMasterInfo> createLauncher(TwillSpecification twillSpec,
@Nullable String schedulerQueue) throws Exception {
+ final YarnClient yarnClient = createYarnClient();
// Request for new application
final GetNewApplicationResponse response = yarnClient.getNewApplication();
final ApplicationId appId = response.getApplicationId();
@@ -100,7 +108,7 @@ public final class Hadoop20YarnAppClient extends AbstractIdleService implements
@Override
public ProcessController<YarnApplicationReport> submit(YarnLaunchContext launchContext) {
ContainerLaunchContext context = launchContext.getLaunchContext();
- addRMToken(context);
+ addRMToken(context, yarnClient);
context.setUser(appSubmissionContext.getUser());
context.setResource(adjustMemory(response, capability));
appSubmissionContext.setAMContainerSpec(context);
@@ -131,7 +139,7 @@ public final class Hadoop20YarnAppClient extends AbstractIdleService implements
return capability;
}
- private void addRMToken(ContainerLaunchContext context) {
+ private void addRMToken(ContainerLaunchContext context, YarnClient yarnClient) {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
}
@@ -176,22 +184,17 @@ public final class Hadoop20YarnAppClient extends AbstractIdleService implements
@Override
public ProcessController<YarnApplicationReport> createProcessController(ApplicationId appId) {
- return new ProcessControllerImpl(yarnClient, appId);
+ return new ProcessControllerImpl(createYarnClient(), appId);
}
@Override
public List<NodeReport> getNodeReports() throws Exception {
- return this.yarnClient.getNodeReports();
- }
-
- @Override
- protected void startUp() throws Exception {
- yarnClient.start();
- }
-
- @Override
- protected void shutDown() throws Exception {
- yarnClient.stop();
+ YarnClient yarnClient = createYarnClient();
+ try {
+ return yarnClient.getNodeReports();
+ } finally {
+ yarnClient.stop();
+ }
}
private static final class ProcessControllerImpl implements ProcessController<YarnApplicationReport> {
@@ -222,5 +225,10 @@ public final class Hadoop20YarnAppClient extends AbstractIdleService implements
throw Throwables.propagate(e);
}
}
+
+ @Override
+ public void close() throws Exception {
+ yarnClient.stop();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
index 6fd11e5..f24630f 100644
--- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
+++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
@@ -55,19 +55,27 @@ import javax.annotation.Nullable;
* Apache Hadoop 2.1 and beyond.
* </p>
*/
-public final class Hadoop21YarnAppClient extends AbstractIdleService implements YarnAppClient {
+public final class Hadoop21YarnAppClient implements YarnAppClient {
private static final Logger LOG = LoggerFactory.getLogger(Hadoop21YarnAppClient.class);
- private final YarnClient yarnClient;
+ private final Configuration configuration;
public Hadoop21YarnAppClient(Configuration configuration) {
- this.yarnClient = YarnClient.createYarnClient();
+ this.configuration = configuration;
+ }
+
+ // Creates and starts a yarn client
+ private YarnClient createYarnClient() {
+ YarnClient yarnClient = YarnClient.createYarnClient();
yarnClient.init(configuration);
+ yarnClient.start();
+ return yarnClient;
}
@Override
public ProcessLauncher<ApplicationMasterInfo> createLauncher(TwillSpecification twillSpec,
@Nullable String schedulerQueue) throws Exception {
+ final YarnClient yarnClient = createYarnClient();
// Request for new application
YarnClientApplication application = yarnClient.createApplication();
final GetNewApplicationResponse response = application.getNewApplicationResponse();
@@ -93,7 +101,7 @@ public final class Hadoop21YarnAppClient extends AbstractIdleService implements
public ProcessController<YarnApplicationReport> submit(YarnLaunchContext context) {
ContainerLaunchContext launchContext = context.getLaunchContext();
- addRMToken(launchContext);
+ addRMToken(launchContext, yarnClient);
appSubmissionContext.setAMContainerSpec(launchContext);
appSubmissionContext.setResource(capability);
appSubmissionContext.setMaxAppAttempts(2);
@@ -122,7 +130,7 @@ public final class Hadoop21YarnAppClient extends AbstractIdleService implements
return capability;
}
- private void addRMToken(ContainerLaunchContext context) {
+ private void addRMToken(ContainerLaunchContext context, YarnClient yarnClient) {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
}
@@ -156,22 +164,17 @@ public final class Hadoop21YarnAppClient extends AbstractIdleService implements
@Override
public ProcessController<YarnApplicationReport> createProcessController(ApplicationId appId) {
- return new ProcessControllerImpl(yarnClient, appId);
+ return new ProcessControllerImpl(createYarnClient(), appId);
}
@Override
public List<NodeReport> getNodeReports() throws Exception {
- return this.yarnClient.getNodeReports();
- }
-
- @Override
- protected void startUp() throws Exception {
- yarnClient.start();
- }
-
- @Override
- protected void shutDown() throws Exception {
- yarnClient.stop();
+ YarnClient yarnClient = createYarnClient();
+ try {
+ return yarnClient.getNodeReports();
+ } finally {
+ yarnClient.stop();
+ }
}
private static final class ProcessControllerImpl implements ProcessController<YarnApplicationReport> {
@@ -202,5 +205,10 @@ public final class Hadoop21YarnAppClient extends AbstractIdleService implements
throw Throwables.propagate(e);
}
}
+
+ @Override
+ public void close() throws Exception {
+ yarnClient.stop();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
index 0e3aea5..e48dcbb 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
@@ -76,6 +76,11 @@ public final class RunnableProcessLauncher extends AbstractYarnProcessLauncher<Y
return new ProcessController<R>() {
@Override
+ public void close() throws Exception {
+ // no-op
+ }
+
+ @Override
public R getReport() {
// No reporting support for runnable launch yet.
return null;
http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
index df956bf..fbbe350 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
@@ -17,7 +17,6 @@
*/
package org.apache.twill.internal.yarn;
-import com.google.common.util.concurrent.Service;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.twill.api.TwillSpecification;
@@ -31,7 +30,7 @@ import javax.annotation.Nullable;
/**
* Interface for launching Yarn application from client.
*/
-public interface YarnAppClient extends Service {
+public interface YarnAppClient {
/**
* Creates a {@link ProcessLauncher} for launching the application represented by the given spec. If scheduler queue
http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
index 3efcda4..1dd6c25 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
@@ -154,7 +154,7 @@ final class YarnTwillController extends AbstractTwillController implements Twill
}
// Poll application status from yarn
- try {
+ try (ProcessController<YarnApplicationReport> processController = this.processController) {
Stopwatch stopWatch = new Stopwatch();
stopWatch.start();
long maxTime = TimeUnit.MILLISECONDS.convert(Constants.APPLICATION_MAX_STOP_SECONDS, TimeUnit.SECONDS);
http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
index 67ee2ac..8c15005 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -319,7 +319,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
}
private void startUp() throws Exception {
- yarnAppClient.startAndWait();
zkClientService.startAndWait();
// Create the root node, so that the namespace root would get created if it is missing
@@ -357,7 +356,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
}
watchCancellable.cancel();
zkClientService.stopAndWait();
- yarnAppClient.stopAndWait();
}
private Cancellable watchLiveApps() {
http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java b/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java
index a896913..4a2146e 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java
@@ -118,12 +118,10 @@ public class TwillTester extends ExternalResource {
twillRunner.start();
yarnAppClient = new VersionDetectYarnAppClientFactory().create(conf);
- yarnAppClient.startAndWait();
}
@Override
protected void after() {
- stopQuietly(yarnAppClient);
try {
twillRunner.stop();
} catch (Exception e) {