[gobblin] branch master updated: GOBBLIN-1524: Limit the number of container launch threads used by YarnService (#3375)

[su...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new a010c7b  GOBBLIN-1524: Limit the number of container launch threads used by YarnService (#3375)
a010c7b is described below

commit a010c7bb45086db64cf6e3c37f232b5a9bceb2b6
Author: Sudarshan Vasudevan <su...@gmail.com>
AuthorDate: Tue Aug 24 09:08:43 2021 -0700

    GOBBLIN-1524: Limit the number of container launch threads used by YarnService (#3375)
---
 .../java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java   | 2 ++
 gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java  | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index fcc30d3..512d24f 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -78,6 +78,8 @@ public class GobblinYarnConfigurationKeys {
   // The ratio of the amount of Xmx to carve out of the container memory before adjusting for jvm memory overhead
   public static final String CONTAINER_JVM_MEMORY_XMX_RATIO_KEY = GOBBLIN_YARN_PREFIX + "container.jvmMemoryXmxRatio";
   public static final double DEFAULT_CONTAINER_JVM_MEMORY_XMX_RATIO = 1.0;
+  public static final String MAX_CONTAINER_LAUNCH_THREADS_KEY = GOBBLIN_YARN_PREFIX + "maxContainerLaunchThreads";
+  public static final Integer DEFAULT_MAX_CONTAINER_LAUNCH_THREADS = 1024;
 
   // Helix configuration properties.
   public static final String HELIX_INSTANCE_MAX_RETRIES = GOBBLIN_YARN_PREFIX + "helix.instance.max.retries";
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
index 810e5ea..d1bce22 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
@@ -239,7 +239,10 @@ public class YarnService extends AbstractIdleService {
         Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)) :
         Optional.<String>absent();
 
-    this.containerLaunchExecutor = ScalingThreadPoolExecutor.newScalingThreadPool(5, Integer.MAX_VALUE, 0L,
+    int numContainerLaunchThreads =
+        ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.MAX_CONTAINER_LAUNCH_THREADS_KEY,
+            GobblinYarnConfigurationKeys.DEFAULT_MAX_CONTAINER_LAUNCH_THREADS);
+    this.containerLaunchExecutor = ScalingThreadPoolExecutor.newScalingThreadPool(5, numContainerLaunchThreads, 0L,
         ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));
 
     this.tokens = getSecurityTokens();