You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2016/08/29 23:00:07 UTC

twill git commit: TWILL-175 Avoid caching a YarnClient, since each YarnClient is fixed to a particular UserGroupInformation.

Repository: twill
Updated Branches:
  refs/heads/master 44e6dfd93 -> 4dfd44968


TWILL-175 Avoid caching a YarnClient, since each YarnClient is fixed to a particular UserGroupInformation.

This closes #6 on Github

Signed-off-by: Terence Yim <ch...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/4dfd4496
Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/4dfd4496
Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/4dfd4496

Branch: refs/heads/master
Commit: 4dfd44968452518af73380497d5330d15ed28df6
Parents: 44e6dfd
Author: Ali Anwar <an...@berkeley.edu>
Authored: Thu Aug 25 23:45:39 2016 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Aug 29 15:59:55 2016 -0700

----------------------------------------------------------------------
 .../twill/internal/ProcessController.java       |  2 +-
 .../internal/yarn/Hadoop20YarnAppClient.java    | 44 ++++++++++++--------
 .../internal/yarn/Hadoop21YarnAppClient.java    | 42 +++++++++++--------
 .../appmaster/RunnableProcessLauncher.java      |  5 +++
 .../twill/internal/yarn/YarnAppClient.java      |  3 +-
 .../apache/twill/yarn/YarnTwillController.java  |  2 +-
 .../twill/yarn/YarnTwillRunnerService.java      |  2 -
 .../java/org/apache/twill/yarn/TwillTester.java |  2 -
 8 files changed, 59 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java b/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java
index 4453838..9a4cbe7 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/ProcessController.java
@@ -24,7 +24,7 @@ import org.apache.twill.common.Cancellable;
  *
  * @param <R> Report type.
  */
-public interface ProcessController<R> extends Cancellable {
+public interface ProcessController<R> extends AutoCloseable, Cancellable {
 
   R getReport();
 

http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
index bfa494c..bb494d4 100644
--- a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
+++ b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
@@ -59,21 +59,29 @@ import javax.annotation.Nullable;
  * Apache Hadoop 2.0.
  * </p>
  */
-public final class Hadoop20YarnAppClient extends AbstractIdleService implements YarnAppClient {
+public final class Hadoop20YarnAppClient implements YarnAppClient {
 
   private static final Logger LOG = LoggerFactory.getLogger(Hadoop20YarnAppClient.class);
-  private final YarnClient yarnClient;
+  private final Configuration configuration;
   private String user;
 
   public Hadoop20YarnAppClient(Configuration configuration) {
-    this.yarnClient = new YarnClientImpl();
-    yarnClient.init(configuration);
+    this.configuration = configuration;
     this.user = System.getProperty("user.name");
   }
 
+  // Creates and starts a yarn client
+  private YarnClient createYarnClient() {
+    YarnClient yarnClient = new YarnClientImpl();
+    yarnClient.init(configuration);
+    yarnClient.start();
+    return yarnClient;
+  }
+
   @Override
   public ProcessLauncher<ApplicationMasterInfo> createLauncher(TwillSpecification twillSpec,
                                                                @Nullable String schedulerQueue) throws Exception {
+    final YarnClient yarnClient = createYarnClient();
     // Request for new application
     final GetNewApplicationResponse response = yarnClient.getNewApplication();
     final ApplicationId appId = response.getApplicationId();
@@ -100,7 +108,7 @@ public final class Hadoop20YarnAppClient extends AbstractIdleService implements
       @Override
       public ProcessController<YarnApplicationReport> submit(YarnLaunchContext launchContext) {
         ContainerLaunchContext context = launchContext.getLaunchContext();
-        addRMToken(context);
+        addRMToken(context, yarnClient);
         context.setUser(appSubmissionContext.getUser());
         context.setResource(adjustMemory(response, capability));
         appSubmissionContext.setAMContainerSpec(context);
@@ -131,7 +139,7 @@ public final class Hadoop20YarnAppClient extends AbstractIdleService implements
     return capability;
   }
 
-  private void addRMToken(ContainerLaunchContext context) {
+  private void addRMToken(ContainerLaunchContext context, YarnClient yarnClient) {
     if (!UserGroupInformation.isSecurityEnabled()) {
       return;
     }
@@ -176,22 +184,17 @@ public final class Hadoop20YarnAppClient extends AbstractIdleService implements
 
   @Override
   public ProcessController<YarnApplicationReport> createProcessController(ApplicationId appId) {
-    return new ProcessControllerImpl(yarnClient, appId);
+    return new ProcessControllerImpl(createYarnClient(), appId);
   }
 
   @Override
   public List<NodeReport> getNodeReports() throws Exception {
-    return this.yarnClient.getNodeReports();
-  }
-
-  @Override
-  protected void startUp() throws Exception {
-    yarnClient.start();
-  }
-
-  @Override
-  protected void shutDown() throws Exception {
-    yarnClient.stop();
+    YarnClient yarnClient = createYarnClient();
+    try {
+      return yarnClient.getNodeReports();
+    } finally {
+      yarnClient.stop();
+    }
   }
 
   private static final class ProcessControllerImpl implements ProcessController<YarnApplicationReport> {
@@ -222,5 +225,10 @@ public final class Hadoop20YarnAppClient extends AbstractIdleService implements
         throw Throwables.propagate(e);
       }
     }
+
+    @Override
+    public void close() throws Exception {
+      yarnClient.stop();
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
index 6fd11e5..f24630f 100644
--- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
+++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
@@ -55,19 +55,27 @@ import javax.annotation.Nullable;
  * Apache Hadoop 2.1 and beyond.
  * </p>
  */
-public final class Hadoop21YarnAppClient extends AbstractIdleService implements YarnAppClient {
+public final class Hadoop21YarnAppClient implements YarnAppClient {
 
   private static final Logger LOG = LoggerFactory.getLogger(Hadoop21YarnAppClient.class);
-  private final YarnClient yarnClient;
+  private final Configuration configuration;
 
   public Hadoop21YarnAppClient(Configuration configuration) {
-    this.yarnClient = YarnClient.createYarnClient();
+    this.configuration = configuration;
+  }
+
+  // Creates and starts a yarn client
+  private YarnClient createYarnClient() {
+    YarnClient yarnClient = YarnClient.createYarnClient();
     yarnClient.init(configuration);
+    yarnClient.start();
+    return yarnClient;
   }
 
   @Override
   public ProcessLauncher<ApplicationMasterInfo> createLauncher(TwillSpecification twillSpec,
                                                                @Nullable String schedulerQueue) throws Exception {
+    final YarnClient yarnClient = createYarnClient();
     // Request for new application
     YarnClientApplication application = yarnClient.createApplication();
     final GetNewApplicationResponse response = application.getNewApplicationResponse();
@@ -93,7 +101,7 @@ public final class Hadoop21YarnAppClient extends AbstractIdleService implements
       public ProcessController<YarnApplicationReport> submit(YarnLaunchContext context) {
         ContainerLaunchContext launchContext = context.getLaunchContext();
 
-        addRMToken(launchContext);
+        addRMToken(launchContext, yarnClient);
         appSubmissionContext.setAMContainerSpec(launchContext);
         appSubmissionContext.setResource(capability);
         appSubmissionContext.setMaxAppAttempts(2);
@@ -122,7 +130,7 @@ public final class Hadoop21YarnAppClient extends AbstractIdleService implements
     return capability;
   }
 
-  private void addRMToken(ContainerLaunchContext context) {
+  private void addRMToken(ContainerLaunchContext context, YarnClient yarnClient) {
     if (!UserGroupInformation.isSecurityEnabled()) {
       return;
     }
@@ -156,22 +164,17 @@ public final class Hadoop21YarnAppClient extends AbstractIdleService implements
 
   @Override
   public ProcessController<YarnApplicationReport> createProcessController(ApplicationId appId) {
-    return new ProcessControllerImpl(yarnClient, appId);
+    return new ProcessControllerImpl(createYarnClient(), appId);
   }
 
   @Override
   public List<NodeReport> getNodeReports() throws Exception {
-    return this.yarnClient.getNodeReports();
-  }
-
-  @Override
-  protected void startUp() throws Exception {
-    yarnClient.start();
-  }
-
-  @Override
-  protected void shutDown() throws Exception {
-    yarnClient.stop();
+    YarnClient yarnClient = createYarnClient();
+    try {
+      return yarnClient.getNodeReports();
+    } finally {
+      yarnClient.stop();
+    }
   }
 
   private static final class ProcessControllerImpl implements ProcessController<YarnApplicationReport> {
@@ -202,5 +205,10 @@ public final class Hadoop21YarnAppClient extends AbstractIdleService implements
         throw Throwables.propagate(e);
       }
     }
+
+    @Override
+    public void close() throws Exception {
+      yarnClient.stop();
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
index 0e3aea5..e48dcbb 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
@@ -76,6 +76,11 @@ public final class RunnableProcessLauncher extends AbstractYarnProcessLauncher<Y
 
     return new ProcessController<R>() {
       @Override
+      public void close() throws Exception {
+        // no-op
+      }
+
+      @Override
       public R getReport() {
         // No reporting support for runnable launch yet.
         return null;

http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
index df956bf..fbbe350 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
@@ -17,7 +17,6 @@
  */
 package org.apache.twill.internal.yarn;
 
-import com.google.common.util.concurrent.Service;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.twill.api.TwillSpecification;
@@ -31,7 +30,7 @@ import javax.annotation.Nullable;
 /**
  * Interface for launching Yarn application from client.
  */
-public interface YarnAppClient extends Service {
+public interface YarnAppClient {
 
   /**
    * Creates a {@link ProcessLauncher} for launching the application represented by the given spec. If scheduler queue

http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
index 3efcda4..1dd6c25 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
@@ -154,7 +154,7 @@ final class YarnTwillController extends AbstractTwillController implements Twill
     }
 
     // Poll application status from yarn
-    try {
+    try (ProcessController<YarnApplicationReport> processController = this.processController) {
       Stopwatch stopWatch = new Stopwatch();
       stopWatch.start();
       long maxTime = TimeUnit.MILLISECONDS.convert(Constants.APPLICATION_MAX_STOP_SECONDS, TimeUnit.SECONDS);

http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
index 67ee2ac..8c15005 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -319,7 +319,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
   }
 
   private void startUp() throws Exception {
-    yarnAppClient.startAndWait();
     zkClientService.startAndWait();
 
     // Create the root node, so that the namespace root would get created if it is missing
@@ -357,7 +356,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
     }
     watchCancellable.cancel();
     zkClientService.stopAndWait();
-    yarnAppClient.stopAndWait();
   }
 
   private Cancellable watchLiveApps() {

http://git-wip-us.apache.org/repos/asf/twill/blob/4dfd4496/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java b/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java
index a896913..4a2146e 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/TwillTester.java
@@ -118,12 +118,10 @@ public class TwillTester extends ExternalResource {
     twillRunner.start();
 
     yarnAppClient = new VersionDetectYarnAppClientFactory().create(conf);
-    yarnAppClient.startAndWait();
   }
 
   @Override
   protected void after() {
-    stopQuietly(yarnAppClient);
     try {
       twillRunner.stop();
     } catch (Exception e) {