You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by ka...@apache.org on 2018/04/03 18:55:06 UTC
[incubator-heron] branch master updated: add config for apiserver
overrides (#2845)
This is an automated email from the ASF dual-hosted git repository.
karthikz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 860a926 add config for apiserver overrides (#2845)
860a926 is described below
commit 860a926273caa37e2fa7d07ad7a27e2cac097f08
Author: Boyang Jerry Peng <je...@gmail.com>
AuthorDate: Tue Apr 3 11:55:01 2018 -0700
add config for apiserver overrides (#2845)
---
.../java/com/twitter/heron/scheduler/nomad/NomadScheduler.java | 3 ++-
heron/spi/src/java/com/twitter/heron/spi/common/Context.java | 4 ++++
heron/spi/src/java/com/twitter/heron/spi/common/Key.java | 4 ++++
.../java/com/twitter/heron/apiserver/utils/ConfigUtils.java | 10 +++++++++-
4 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/nomad/NomadScheduler.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/nomad/NomadScheduler.java
index 482aaa9..7531f23 100644
--- a/heron/schedulers/src/java/com/twitter/heron/scheduler/nomad/NomadScheduler.java
+++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/nomad/NomadScheduler.java
@@ -70,7 +70,8 @@ public class NomadScheduler implements IScheduler {
.putAll(config)
.putAll(ConfigLoader.loadConfig(
Context.heronHome(config), Context.heronConf(config),
- null, Context.overrideFile(config)))
+ null, Context.apiserverOverrideFile(config) != null
+ ? Context.apiserverOverrideFile(config) : Context.overrideFile(config)))
.build());
this.runtimeConfig = runtime;
this.updateTopologyManager =
diff --git a/heron/spi/src/java/com/twitter/heron/spi/common/Context.java b/heron/spi/src/java/com/twitter/heron/spi/common/Context.java
index 7438bee..27a5dfb 100644
--- a/heron/spi/src/java/com/twitter/heron/spi/common/Context.java
+++ b/heron/spi/src/java/com/twitter/heron/spi/common/Context.java
@@ -297,6 +297,10 @@ public class Context {
return cfg.getStringValue(Key.OVERRIDE_YAML);
}
+ public static String apiserverOverrideFile(Config cfg) {
+ return cfg.getStringValue(Key.APISERVER_OVERRIDE_YAML);
+ }
+
public static String executorBinary(Config cfg) {
return cfg.getStringValue(Key.EXECUTOR_BINARY);
}
diff --git a/heron/spi/src/java/com/twitter/heron/spi/common/Key.java b/heron/spi/src/java/com/twitter/heron/spi/common/Key.java
index 66d3313..acd74e3 100644
--- a/heron/spi/src/java/com/twitter/heron/spi/common/Key.java
+++ b/heron/spi/src/java/com/twitter/heron/spi/common/Key.java
@@ -165,6 +165,10 @@ public enum Key {
//keys for heron configuration files on the cluster
OVERRIDE_YAML("heron.config.file.override.yaml", "${HERON_CONF}/override.yaml"),
+ // Path to the config overrides passed into the apiserver. Only applicable to submitting
+ // topologies via apiserver
+ APISERVER_OVERRIDE_YAML("heron.apiserver.file.override.yaml", Type.STRING),
+
//keys for config provided user binaries
EXECUTOR_BINARY ("heron.binaries.executor", "${HERON_BIN}/heron-executor"),
STMGR_BINARY ("heron.binaries.stmgr", "${HERON_BIN}/heron-stmgr"),
diff --git a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/ConfigUtils.java b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/ConfigUtils.java
index ddf750a..05e5673 100644
--- a/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/ConfigUtils.java
+++ b/heron/tools/apiserver/src/java/com/twitter/heron/apiserver/utils/ConfigUtils.java
@@ -32,6 +32,7 @@ import com.twitter.heron.common.basics.SysUtils;
import com.twitter.heron.scheduler.utils.SubmitterUtils;
import com.twitter.heron.spi.common.Config;
import com.twitter.heron.spi.common.ConfigLoader;
+import com.twitter.heron.spi.common.Key;
public final class ConfigUtils {
@@ -62,10 +63,17 @@ public final class ConfigUtils {
String releaseFile,
String overrideConfigurationFile) {
// TODO add release file
- return ConfigLoader.loadConfig(heronDirectory,
+ Config config = ConfigLoader.loadConfig(heronDirectory,
heronConfigDirectory,
releaseFile,
overrideConfigurationFile);
+ // Put location of the override file in the config so that schedulers invoked by
+ // the apiserver can load the override configs if needed. OVERRIDE_YAML cannot be used
+ // to set this because then the location will get passed on to the heron executors
+ return Config.newBuilder()
+ .putAll(config)
+ .put(Key.APISERVER_OVERRIDE_YAML, overrideConfigurationFile)
+ .build();
}
public static Config getTopologyConfig(String topologyPackage, String topologyBinaryFile,
--
To stop receiving notification emails like this one, please contact
karthikz@apache.org.