You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/09/04 03:17:31 UTC
git commit: SLIDER-391. appConfig.json should allow reference to
other config properties and resolve them automatically
Repository: incubator-slider
Updated Branches:
refs/heads/feature/auto_generate_storm_pkg 26b4153eb -> 02faa944d
SLIDER-391. appConfig.json should allow reference to other config properties and resolve them automatically
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/02faa944
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/02faa944
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/02faa944
Branch: refs/heads/feature/auto_generate_storm_pkg
Commit: 02faa944da56cd6252b48afbdb6ee427291f1fe0
Parents: 26b4153
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 18:17:13 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 18:17:13 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 8 +++--
app-packages/storm/appConfig.json | 7 ++--
.../providers/agent/AgentProviderService.java | 35 ++++++++++++++++----
.../agent/TestAgentProviderService.java | 29 ++++++++++++++++
4 files changed, 66 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 7df0dc3..07d1b4e 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -11,14 +11,16 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_port": "8667",
+ "site.global.ganglia_server_id": "Application1",
+ "site.global.ganglia_enabled":"true",
+
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
"site.global.monitor_protocol": "http",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index df28e8f..ac9bd49 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -11,6 +11,7 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
+
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
@@ -22,8 +23,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -31,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 61866fb..d188d49 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1196,7 +1196,8 @@ public class AgentProviderService extends AbstractProviderService implements
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- setInstallCommandConfigurations(cmd, containerId);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
+ cmd.setConfigurations(configurations);
cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
@@ -1246,12 +1247,6 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd, String containerId) throws SliderException {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
- cmd.setConfigurations(configurations);
- }
-
@VisibleForTesting
protected void addStatusCommand(String roleName,
String containerId,
@@ -1380,9 +1375,35 @@ public class AgentProviderService extends AbstractProviderService implements
configurations, tokens, containerId);
}
+ //do a final replacement of re-used configs
+ dereferenceAllConfigs(configurations);
+
return configurations;
}
+ protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
+ Map<String, String> allConfigs = new HashMap<String, String>();
+ String lookupFormat = "${site.%s.%s}";
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ allConfigs.put(String.format(lookupFormat, configType, configName), configBucket.get(configName));
+ }
+ }
+
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ String configValue = configBucket.get(configName);
+ for (String lookUpKey : allConfigs.keySet()) {
+ if (configValue != null && configValue.contains(lookUpKey)) {
+ configBucket.put(configName, configValue.replace(lookUpKey, allConfigs.get(lookUpKey)));
+ }
+ }
+ }
+ }
+ }
+
private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
Map<String, String> tokens = new HashMap<String, String>();
String nnuri = appConf.get("site.fs.defaultFS");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 16f5347..0f0e744 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -1128,4 +1128,33 @@ public class TestAgentProviderService {
Assert.assertFalse(configs.contains("bore-site"));
}
+ @Test
+ public void testDereferenceAllConfig() {
+ AgentProviderService aps = new AgentProviderService();
+ Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>();
+ Map<String, String> cfg1 = new HashMap<String, String>();
+ cfg1.put("a1", "${site.cfg-2.A1}");
+ cfg1.put("b1", "22");
+ cfg1.put("c1", "33");
+ cfg1.put("d1", "${site.cfg1.c1}AA");
+ Map<String, String> cfg2 = new HashMap<String, String>();
+ cfg2.put("A1", "11");
+ cfg2.put("B1", "${site.cfg-2.A1},${site.cfg-2.A1}");
+ cfg2.put("C1", "DD${site.cfg1.c1}");
+ cfg2.put("D1", "${14}");
+
+ allConfigs.put("cfg1", cfg1);
+ allConfigs.put("cfg-2", cfg2);
+ aps.dereferenceAllConfigs(allConfigs);
+ Assert.assertEquals("11", cfg1.get("a1"));
+ Assert.assertEquals("22", cfg1.get("b1"));
+ Assert.assertEquals("33", cfg1.get("c1"));
+ Assert.assertEquals("33AA", cfg1.get("d1"));
+
+ Assert.assertEquals("11", cfg2.get("A1"));
+ Assert.assertEquals("11,11", cfg2.get("B1"));
+ Assert.assertEquals("DD33", cfg2.get("C1"));
+ Assert.assertEquals("${14}", cfg2.get("D1"));
+ }
+
}