You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/09/08 23:25:33 UTC
[19/27] git commit: SLIDER-181. Storm package updates (changes to
accomodate storm script changing to be a shell script from python)
SLIDER-402. Add support for general purpose cross referencing of named
properties (handle multiple replacements for singl
SLIDER-181. Storm package updates (changes to accomodate storm script changing to be a shell script from python)
SLIDER-402. Add support for general purpose cross referencing of named properties (handle multiple replacements for single property)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/486ef84f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/486ef84f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/486ef84f
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 486ef84f4cb7e6e716e70a82ac08e1c43b3c497d
Parents: f14d32c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Sep 5 20:59:04 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Sep 5 20:59:24 2014 -0700
----------------------------------------------------------------------
app-packages/storm/appConfig.json | 6 +--
app-packages/storm/package/scripts/params.py | 2 +-
app-packages/storm/package/scripts/service.py | 43 ++++++++------------
.../providers/agent/AgentProviderService.java | 5 ++-
.../agent/TestAgentProviderService.java | 10 ++---
5 files changed, 28 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index ac9bd49..1eae599 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -23,8 +23,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -32,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index a70bcb9..38e25fb 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -34,7 +34,7 @@ user_group = config['configurations']['global']['user_group']
java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-storm_bin = format("{app_root}/bin/storm")
+storm_bin = format("{app_root}/bin/storm.py")
storm_env_sh_template = config['configurations']['storm-env']['content']
ganglia_installed = config['configurations']['global']['ganglia_enabled']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index dc3c733..3a5e795 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -21,6 +21,8 @@ limitations under the License.
from resource_management import *
import time
+import os
+import sys
"""
Slider package uses jps as pgrep does not list the whole process start command
@@ -31,6 +33,7 @@ def service(
import params
import status_params
+ python_binary = os.environ['PYTHON_EXE'] if 'PYTHON_EXE' in os.environ else sys.executable
pid_file = status_params.pid_files[name]
container_id = status_params.container_id
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
@@ -40,17 +43,11 @@ def service(
if name == 'ui':
crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk} > {pid_file}")
- elif name == "rest_api":
- rest_process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
- crt_pid_cmd = format("pgrep -f \"{rest_process_cmd}\" > {pid_file}")
else:
crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}")
if action == "start":
- if name == "rest_api":
- cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
- else:
- cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
+ cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
if params.security_enabled:
if name == "nimbus":
@@ -67,30 +64,22 @@ def service(
wait_for_finish=False
)
- if name == "rest_api":
+ content = None
+ for i in xrange(12):
Execute(crt_pid_cmd,
user=params.storm_user,
- logoutput=True,
- tries=6,
- try_sleep=10
+ logoutput=True
)
- else:
- content = None
- for i in xrange(12):
- Execute(crt_pid_cmd,
- user=params.storm_user,
- logoutput=True
- )
- with open(pid_file) as f:
- content = f.readline().strip()
- if content.isdigit():
- break;
- File(pid_file, action = "delete")
- time.sleep(10)
- pass
+ with open(pid_file) as f:
+ content = f.readline().strip()
+ if content.isdigit():
+ break;
+ File(pid_file, action = "delete")
+ time.sleep(10)
+ pass
- if not content.isdigit():
- raise Fail(format("Unable to start {name}"))
+ if not content.isdigit():
+ raise Fail(format("Unable to start {name}"))
elif action == "stop":
process_dont_exist = format("! ({no_op_test})")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 02ef625..063e61c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1382,7 +1382,7 @@ public class AgentProviderService extends AbstractProviderService implements
protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
Map<String, String> allConfigs = new HashMap<String, String>();
- String lookupFormat = "${site.%s.%s}";
+ String lookupFormat = "${@//site/%s/%s}";
for (String configType : configurations.keySet()) {
Map<String, String> configBucket = configurations.get(configType);
for (String configName : configBucket.keySet()) {
@@ -1396,9 +1396,10 @@ public class AgentProviderService extends AbstractProviderService implements
String configValue = configBucket.get(configName);
for (String lookUpKey : allConfigs.keySet()) {
if (configValue != null && configValue.contains(lookUpKey)) {
- configBucket.put(configName, configValue.replace(lookUpKey, allConfigs.get(lookUpKey)));
+ configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey));
}
}
+ configBucket.put(configName, configValue);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 0f0e744..3711e12 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -1133,14 +1133,14 @@ public class TestAgentProviderService {
AgentProviderService aps = new AgentProviderService();
Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>();
Map<String, String> cfg1 = new HashMap<String, String>();
- cfg1.put("a1", "${site.cfg-2.A1}");
+ cfg1.put("a1", "${@//site/cfg-2/A1}");
cfg1.put("b1", "22");
cfg1.put("c1", "33");
- cfg1.put("d1", "${site.cfg1.c1}AA");
+ cfg1.put("d1", "${@//site/cfg1/c1}AA");
Map<String, String> cfg2 = new HashMap<String, String>();
cfg2.put("A1", "11");
- cfg2.put("B1", "${site.cfg-2.A1},${site.cfg-2.A1}");
- cfg2.put("C1", "DD${site.cfg1.c1}");
+ cfg2.put("B1", "${@//site/cfg-2/A1},${@//site/cfg-2/A1},AA,${@//site/cfg1/c1}");
+ cfg2.put("C1", "DD${@//site/cfg1/c1}");
cfg2.put("D1", "${14}");
allConfigs.put("cfg1", cfg1);
@@ -1152,7 +1152,7 @@ public class TestAgentProviderService {
Assert.assertEquals("33AA", cfg1.get("d1"));
Assert.assertEquals("11", cfg2.get("A1"));
- Assert.assertEquals("11,11", cfg2.get("B1"));
+ Assert.assertEquals("11,11,AA,33", cfg2.get("B1"));
Assert.assertEquals("DD33", cfg2.get("C1"));
Assert.assertEquals("${14}", cfg2.get("D1"));
}