You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by su...@apache.org on 2021/08/30 18:08:53 UTC
[gobblin] branch master updated: GOBBLIN-1528: Reduce overshoot of
Yarn containers in Gobblin-on-Yarn mode (#3379)
This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new a18d4e0 GOBBLIN-1528: Reduce overshoot of Yarn containers in Gobblin-on-Yarn mode (#3379)
a18d4e0 is described below
commit a18d4e01f9999426c541441ea4d7f88f82c4b2b0
Author: Sudarshan Vasudevan <su...@gmail.com>
AuthorDate: Mon Aug 30 11:08:46 2021 -0700
GOBBLIN-1528: Reduce overshoot of Yarn containers in Gobblin-on-Yarn mode (#3379)
* GOBBLIN-1528: Reduce overshoot of Yarn containers in Gobblin-on-Yarn mode
* Clarify comment
---
.../java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java | 4 ++++
gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java | 5 ++++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index 512d24f..6e374d3 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -131,4 +131,8 @@ public class GobblinYarnConfigurationKeys {
//Container classpaths properties
public static final String GOBBLIN_YARN_ADDITIONAL_CLASSPATHS = GOBBLIN_YARN_PREFIX + "additional.classpaths";
public static final String GOBBLIN_YARN_CLASSPATHS = GOBBLIN_YARN_PREFIX + "classpaths";
+
+ //Config to control Heartbeat interval for Yarn AMRM client.
+ public static final String AMRM_HEARTBEAT_INTERVAL_SECS = GOBBLIN_YARN_PREFIX + "amRmHeartbeatIntervalSecs";
+ public static final Integer DEFAULT_AMRM_HEARTBEAT_INTERVAL_SECS = 15;
}
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
index d1bce22..9fadc94 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
@@ -221,8 +221,11 @@ public class YarnService extends AbstractIdleService {
this.yarnConfiguration = yarnConfiguration;
this.fs = fs;
+ int amRmHeartbeatIntervalMillis = Long.valueOf(TimeUnit.SECONDS.toMillis(
+ ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.AMRM_HEARTBEAT_INTERVAL_SECS,
+ GobblinYarnConfigurationKeys.DEFAULT_AMRM_HEARTBEAT_INTERVAL_SECS))).intValue();
this.amrmClientAsync = closer.register(
- AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
+ AMRMClientAsync.createAMRMClientAsync(amRmHeartbeatIntervalMillis, new AMRMClientCallbackHandler()));
this.amrmClientAsync.init(this.yarnConfiguration);
this.nmClientAsync = closer.register(NMClientAsync.createNMClientAsync(getNMClientCallbackHandler()));
this.nmClientAsync.init(this.yarnConfiguration);