You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/09/08 23:25:15 UTC
[01/27] git commit: Temporary Storm package with Security
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-280_Restart_AM_fun_tests 278d5e4c7 -> f8c21c7b3
Temporary Storm package with Security
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eb6620e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eb6620e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eb6620e7
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: eb6620e77a1e8c97c5b65f8d098040d03594e478
Parents: 8bfd66e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Aug 28 18:21:18 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 28 18:21:18 2014 -0700
----------------------------------------------------------------------
app-packages/storm/README.txt | 4 +--
app-packages/storm/appConfig.json | 11 ++++----
app-packages/storm/configuration/storm-env.xml | 25 ++++++++++++++++++
app-packages/storm/metainfo.xml | 2 +-
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 ------------
...pache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE | 16 ++++++++++++
app-packages/storm/package/scripts/params.py | 11 +++++---
app-packages/storm/package/scripts/service.py | 8 ++++++
.../storm/package/templates/storm_jaas.conf.j2 | 27 +++++++++++++++++---
app-packages/storm/resources.json | 2 +-
10 files changed, 91 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index 971cf14..77474f1 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -26,10 +26,10 @@ Replace the placeholder tarball for Storm.
rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/)
- zip -r storm_v091.zip .
+ zip -r Apache_Storm_v_0_9_3.zip .
Verify the content using
- unzip -l "$@" storm_v091.zip
+ unzip -l "$@" Apache_Storm_v_0_9_3.zip
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index b8454f9..d6dee84 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,12 +3,12 @@
"metadata": {
},
"global": {
- "application.def": "storm_v091.zip",
+ "application.def": "Apache_Storm_v_0_9_3.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
"site.global.ganglia_server_host": "${NN_HOST}",
@@ -18,13 +18,14 @@
"site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
+ "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
"site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
"site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -32,7 +33,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/configuration/storm-env.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/configuration/storm-env.xml b/app-packages/storm/configuration/storm-env.xml
index f315eb3..091c08d 100644
--- a/app-packages/storm/configuration/storm-env.xml
+++ b/app-packages/storm/configuration/storm-env.xml
@@ -22,6 +22,31 @@
<configuration>
+ <property>
+ <name>kerberos_domain</name>
+ <value></value>
+ <description>The kerberos domain to be used for this Storm cluster</description>
+ </property>
+ <property>
+ <name>storm_client_principal_name</name>
+ <value></value>
+ <description>The principal name for the Storm client to be used to communicate with Nimbus and Zookeeper</description>
+ </property>
+ <property>
+ <name>storm_server_principal_name</name>
+ <value></value>
+ <description>The principal name for the Storm server to be used by Nimbus</description>
+ </property>
+ <property>
+ <name>storm_client_keytab</name>
+ <value></value>
+ <description>The keytab file path for Storm client</description>
+ </property>
+ <property>
+ <name>storm_server_keytab</name>
+ <value></value>
+ <description>The keytab file path for Storm server</description>
+ </property>
<!-- storm-env.sh -->
<property>
<name>content</name>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 70d369c..89ee01e 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -150,7 +150,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ <name>files/apache-storm-0.9.3.0.2.5.0-100.tar.gz</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
deleted file mode 100644
index dd934d5..0000000
--- a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE
new file mode 100644
index 0000000..dd934d5
--- /dev/null
+++ b/app-packages/storm/package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index 3e1620b..93c6018 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -53,6 +53,11 @@ security_enabled = ( not is_empty(_authentication) and _authentication == 'kerbe
if security_enabled:
_hostname_lowercase = config['hostname'].lower()
_kerberos_domain = config['configurations']['storm-env']['kerberos_domain']
- _storm_principal_name = config['configurations']['storm-env']['storm_principal_name']
- storm_jaas_principal = _storm_principal_name.replace('_HOST', _hostname_lowercase)
- storm_keytab_path = config['configurations']['storm-env']['storm_keytab']
+ _storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name']
+ _storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name']
+
+ storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab']
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab']
+ kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index 13fcef2..dc3c733 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -52,6 +52,14 @@ def service(
else:
cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
+ if params.security_enabled:
+ if name == "nimbus":
+ Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_user}"),
+ user=params.storm_user)
+ else:
+ Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_user}"),
+ user=params.storm_user)
+
Execute(cmd,
not_if=no_op_test,
user=params.storm_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/package/templates/storm_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2 b/app-packages/storm/package/templates/storm_jaas.conf.j2
index 4031d22..e68cd57 100644
--- a/app-packages/storm/package/templates/storm_jaas.conf.j2
+++ b/app-packages/storm/package/templates/storm_jaas.conf.j2
@@ -15,12 +15,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+StormServer {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_server_keytab_path}}"
+ storeKey=true
+ useTicketCache=true
+ renewTGT=true
+ principal="{{storm_jaas_server_principal}}";
+};
+StormClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_client_keytab_path}}"
+ storeKey=true
+ useTicketCache=true
+ renewTGT=true
+ serviceName="nimbus"
+ debug=true
+ principal="{{storm_jaas_client_principal}}";
+};
Client {
com.sun.security.auth.module.Krb5LoginModule required
useKeyTab=true
- keyTab="{{storm_keytab_path}}"
+ keyTab="{{storm_client_keytab_path}}"
storeKey=true
- useTicketCache=false
+ useTicketCache=true
+ renewTGT=true
serviceName="zookeeper"
- principal="{{storm_jaas_principal}}";
+ principal="{{storm_jaas_client_principal}}";
};
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb6620e7/app-packages/storm/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json
index b184a40..606dc39 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -13,7 +13,7 @@
},
"STORM_REST_API": {
"yarn.role.priority": "2",
- "yarn.component.instances": "1"
+ "yarn.component.instances": "0"
},
"STORM_UI_SERVER": {
"yarn.role.priority": "3",
[09/27] git commit: SLIDER-397 taskkill /f
Posted by sm...@apache.org.
SLIDER-397 taskkill /f
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ee3ad77a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ee3ad77a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ee3ad77a
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: ee3ad77a83f277e42ecef39b30fd614d66cc5530
Parents: 27f0a62
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 20:43:15 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 20:43:15 2014 +0100
----------------------------------------------------------------------
.../src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee3ad77a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 5086248..dbf5c49 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -755,7 +755,7 @@ class SliderTestUtils extends Assert {
/*
"jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
*/
- GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID "
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID\""
commandString = ["CMD", "/C", killCommand]
}
Process command = commandString.execute()
[19/27] git commit: SLIDER-181. Storm package updates (changes to
accomodate storm script changing to be a shell script from python)
SLIDER-402. Add support for general purpose cross referencing of named
properties (handle multiple replacements for singl
Posted by sm...@apache.org.
SLIDER-181. Storm package updates (changes to accomodate storm script changing to be a shell script from python)
SLIDER-402. Add support for general purpose cross referencing of named properties (handle multiple replacements for single property)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/486ef84f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/486ef84f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/486ef84f
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 486ef84f4cb7e6e716e70a82ac08e1c43b3c497d
Parents: f14d32c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Sep 5 20:59:04 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Sep 5 20:59:24 2014 -0700
----------------------------------------------------------------------
app-packages/storm/appConfig.json | 6 +--
app-packages/storm/package/scripts/params.py | 2 +-
app-packages/storm/package/scripts/service.py | 43 ++++++++------------
.../providers/agent/AgentProviderService.java | 5 ++-
.../agent/TestAgentProviderService.java | 10 ++---
5 files changed, 28 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index ac9bd49..1eae599 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -23,8 +23,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -32,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index a70bcb9..38e25fb 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -34,7 +34,7 @@ user_group = config['configurations']['global']['user_group']
java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-storm_bin = format("{app_root}/bin/storm")
+storm_bin = format("{app_root}/bin/storm.py")
storm_env_sh_template = config['configurations']['storm-env']['content']
ganglia_installed = config['configurations']['global']['ganglia_enabled']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index dc3c733..3a5e795 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -21,6 +21,8 @@ limitations under the License.
from resource_management import *
import time
+import os
+import sys
"""
Slider package uses jps as pgrep does not list the whole process start command
@@ -31,6 +33,7 @@ def service(
import params
import status_params
+ python_binary = os.environ['PYTHON_EXE'] if 'PYTHON_EXE' in os.environ else sys.executable
pid_file = status_params.pid_files[name]
container_id = status_params.container_id
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
@@ -40,17 +43,11 @@ def service(
if name == 'ui':
crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk} > {pid_file}")
- elif name == "rest_api":
- rest_process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
- crt_pid_cmd = format("pgrep -f \"{rest_process_cmd}\" > {pid_file}")
else:
crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}")
if action == "start":
- if name == "rest_api":
- cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
- else:
- cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
+ cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
if params.security_enabled:
if name == "nimbus":
@@ -67,30 +64,22 @@ def service(
wait_for_finish=False
)
- if name == "rest_api":
+ content = None
+ for i in xrange(12):
Execute(crt_pid_cmd,
user=params.storm_user,
- logoutput=True,
- tries=6,
- try_sleep=10
+ logoutput=True
)
- else:
- content = None
- for i in xrange(12):
- Execute(crt_pid_cmd,
- user=params.storm_user,
- logoutput=True
- )
- with open(pid_file) as f:
- content = f.readline().strip()
- if content.isdigit():
- break;
- File(pid_file, action = "delete")
- time.sleep(10)
- pass
+ with open(pid_file) as f:
+ content = f.readline().strip()
+ if content.isdigit():
+ break;
+ File(pid_file, action = "delete")
+ time.sleep(10)
+ pass
- if not content.isdigit():
- raise Fail(format("Unable to start {name}"))
+ if not content.isdigit():
+ raise Fail(format("Unable to start {name}"))
elif action == "stop":
process_dont_exist = format("! ({no_op_test})")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 02ef625..063e61c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1382,7 +1382,7 @@ public class AgentProviderService extends AbstractProviderService implements
protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
Map<String, String> allConfigs = new HashMap<String, String>();
- String lookupFormat = "${site.%s.%s}";
+ String lookupFormat = "${@//site/%s/%s}";
for (String configType : configurations.keySet()) {
Map<String, String> configBucket = configurations.get(configType);
for (String configName : configBucket.keySet()) {
@@ -1396,9 +1396,10 @@ public class AgentProviderService extends AbstractProviderService implements
String configValue = configBucket.get(configName);
for (String lookUpKey : allConfigs.keySet()) {
if (configValue != null && configValue.contains(lookUpKey)) {
- configBucket.put(configName, configValue.replace(lookUpKey, allConfigs.get(lookUpKey)));
+ configValue = configValue.replace(lookUpKey, allConfigs.get(lookUpKey));
}
}
+ configBucket.put(configName, configValue);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/486ef84f/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 0f0e744..3711e12 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -1133,14 +1133,14 @@ public class TestAgentProviderService {
AgentProviderService aps = new AgentProviderService();
Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>();
Map<String, String> cfg1 = new HashMap<String, String>();
- cfg1.put("a1", "${site.cfg-2.A1}");
+ cfg1.put("a1", "${@//site/cfg-2/A1}");
cfg1.put("b1", "22");
cfg1.put("c1", "33");
- cfg1.put("d1", "${site.cfg1.c1}AA");
+ cfg1.put("d1", "${@//site/cfg1/c1}AA");
Map<String, String> cfg2 = new HashMap<String, String>();
cfg2.put("A1", "11");
- cfg2.put("B1", "${site.cfg-2.A1},${site.cfg-2.A1}");
- cfg2.put("C1", "DD${site.cfg1.c1}");
+ cfg2.put("B1", "${@//site/cfg-2/A1},${@//site/cfg-2/A1},AA,${@//site/cfg1/c1}");
+ cfg2.put("C1", "DD${@//site/cfg1/c1}");
cfg2.put("D1", "${14}");
allConfigs.put("cfg1", cfg1);
@@ -1152,7 +1152,7 @@ public class TestAgentProviderService {
Assert.assertEquals("33AA", cfg1.get("d1"));
Assert.assertEquals("11", cfg2.get("A1"));
- Assert.assertEquals("11,11", cfg2.get("B1"));
+ Assert.assertEquals("11,11,AA,33", cfg2.get("B1"));
Assert.assertEquals("DD33", cfg2.get("C1"));
Assert.assertEquals("${14}", cfg2.get("D1"));
}
[23/27] git commit: Merge branch 'feature/update_storm_for_93' into
develop
Posted by sm...@apache.org.
Merge branch 'feature/update_storm_for_93' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f66c0cbc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f66c0cbc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f66c0cbc
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: f66c0cbc0f211da1ea52acf7cad8a20480a1f5ea
Parents: 8da1503 4b2f8aa
Author: Jon Maron <jm...@hortonworks.com>
Authored: Mon Sep 8 09:41:14 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Mon Sep 8 09:41:14 2014 -0400
----------------------------------------------------------------------
app-packages/storm/README.txt | 19 +++++++++++--------
app-packages/storm/appConfig.json | 4 ++--
app-packages/storm/metainfo.xml | 2 +-
app-packages/storm/package/scripts/params.py | 6 +++---
app-packages/storm/package/scripts/service.py | 6 +++---
app-packages/storm/package/scripts/storm.py | 9 ++++++---
.../storm/package/scripts/yaml_config.py | 10 +++++++++-
.../storm/package/templates/storm_jaas.conf.j2 | 11 ++++-------
app-packages/storm/resources.json | 4 ++--
9 files changed, 41 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
[18/27] git commit: Merge branch
'feature/SLIDER-401_slider-agent_tests_on_windows' into develop
Posted by sm...@apache.org.
Merge branch 'feature/SLIDER-401_slider-agent_tests_on_windows' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f14d32c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f14d32c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f14d32c6
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: f14d32c6fac2e252e587dbdc88e0e3e079f44d37
Parents: 2d6331a b323df8
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 13:59:58 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 13:59:58 2014 +0100
----------------------------------------------------------------------
pom.xml | 1 +
slider-agent/pom.xml | 4 +--
slider-agent/src/main/python/setup.py | 2 +-
slider-agent/src/test/python/python-wrap | 40 ----------------------
slider-agent/src/test/python/python-wrap.sh | 42 ++++++++++++++++++++++++
slider-agent/src/test/python/unitTests.py | 36 ++++++++++----------
slider-core/pom.xml | 4 +--
7 files changed, 65 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
[13/27] git commit: Merge branch 'feature/auto_generate_storm_pkg'
into develop
Posted by sm...@apache.org.
Merge branch 'feature/auto_generate_storm_pkg' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/56cd8667
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/56cd8667
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/56cd8667
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 56cd8667223a634eb0a806354122cc7210c5a3f8
Parents: 168a7af 4c80820
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 4 21:52:02 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 4 21:52:02 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 8 +++--
app-packages/storm/appConfig.json | 7 ++--
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 38 +++++++++++++++-----
.../agent/TestAgentProviderService.java | 29 +++++++++++++++
5 files changed, 68 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
[14/27] git commit: SLIDER-397 -fix regression surfacing on jenkins
(editing wrong "kill" command)
Posted by sm...@apache.org.
SLIDER-397 -fix regression surfacing on jenkins (editing wrong "kill" command)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2d6331a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2d6331a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2d6331a0
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 2d6331a0eca42fc19b40f741e31a1580ab8ae8b8
Parents: 56cd866
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 09:57:11 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 09:57:11 2014 +0100
----------------------------------------------------------------------
.../src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2d6331a0/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index dbf5c49..cbf4749 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -747,7 +747,7 @@ class SliderTestUtils extends Assert {
def commandString
if (!Shell.WINDOWS) {
- GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill /f $signal"
+ GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
log.info("Command command = $killCommand" )
commandString = ["bash", "-c", killCommand]
[27/27] git commit: Merge branch 'develop' into
feature/SLIDER-280_Restart_AM_fun_tests
Posted by sm...@apache.org.
Merge branch 'develop' into feature/SLIDER-280_Restart_AM_fun_tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f8c21c7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f8c21c7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f8c21c7b
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: f8c21c7b30f851bea0e422938dd432eed7b4f344
Parents: 278d5e4 bff5ed6
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Sep 8 14:24:43 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Sep 8 14:24:43 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 2 +
.../src/test/resources/appConfig_kerberos.json | 2 +
.../test/resources/appConfig_monitor_ssl.json | 2 +
.../src/test/resources/appConfig_ssl.json | 2 +
app-packages/hbase/appConfig.json | 8 ++--
app-packages/storm/README.txt | 19 ++++----
app-packages/storm/appConfig.json | 5 ++-
app-packages/storm/metainfo.xml | 2 +-
app-packages/storm/package/scripts/params.py | 8 ++--
app-packages/storm/package/scripts/service.py | 47 ++++++++------------
app-packages/storm/package/scripts/storm.py | 9 ++--
.../storm/package/scripts/yaml_config.py | 10 ++++-
.../storm/package/templates/storm_jaas.conf.j2 | 11 ++---
app-packages/storm/resources.json | 2 +-
pom.xml | 12 ++---
slider-agent/pom.xml | 4 +-
slider-agent/src/main/python/setup.py | 2 +-
slider-agent/src/test/python/python-wrap | 40 -----------------
slider-agent/src/test/python/python-wrap.sh | 42 +++++++++++++++++
slider-agent/src/test/python/unitTests.py | 36 +++++++--------
slider-assembly/pom.xml | 23 +++++++++-
slider-assembly/src/main/scripts/slider | 2 +
slider-assembly/src/main/scripts/slider.py | 10 +++--
slider-core/pom.xml | 4 +-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 39 ++++++++++++----
.../standalone/TestStandaloneAMKill.groovy | 2 +-
.../apache/slider/test/SliderTestUtils.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 4 +-
.../agent/TestAgentProviderService.java | 29 ++++++++++++
30 files changed, 231 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8c21c7b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
[15/27] git commit: SLIDER-401 slider-agent tests not running on
windows
Posted by sm...@apache.org.
SLIDER-401 slider-agent tests not running on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a9f0b985
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a9f0b985
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a9f0b985
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: a9f0b985d7bab6387a028df3019d467a69a69626
Parents: 2d6331a
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 13:15:38 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 13:15:38 2014 +0100
----------------------------------------------------------------------
pom.xml | 1 +
slider-agent/pom.xml | 4 +--
slider-agent/src/test/python/python-wrap | 40 ----------------------
slider-agent/src/test/python/python-wrap.sh | 42 ++++++++++++++++++++++++
slider-core/pom.xml | 4 +--
5 files changed, 47 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9efb5c8..d9b3813 100644
--- a/pom.xml
+++ b/pom.xml
@@ -200,6 +200,7 @@
<maven-deploy-plugin.version>2.7</maven-deploy-plugin.version>
<maven-doxia-module-markdown.version>1.4</maven-doxia-module-markdown.version>
<maven-enforcer-plugin.version>1.0</maven-enforcer-plugin.version>
+ <maven-exec-plugin.version>1.2.1</maven-exec-plugin.version>
<maven-jar-plugin.version>2.3.1</maven-jar-plugin.version>
<maven.javadoc.version>2.8</maven.javadoc.version>
<maven.project.version>2.4</maven.project.version>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/slider-agent/pom.xml
----------------------------------------------------------------------
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index f26cb11..48dd301 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -59,11 +59,11 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2</version>
+ <version>${maven-exec-plugin.version}</version>
<executions>
<execution>
<configuration>
- <executable>${project.basedir}/src/test/python/python-wrap</executable>
+ <executable>python</executable>
<workingDirectory>src/test/python</workingDirectory>
<arguments>
<argument>unitTests.py</argument>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/slider-agent/src/test/python/python-wrap
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/python-wrap b/slider-agent/src/test/python/python-wrap
deleted file mode 100755
index 40dc785..0000000
--- a/slider-agent/src/test/python/python-wrap
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2011 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export PYTHONPATH=/usr/lib/python2.6/site-packages/common_functions:$PYTHONPATH
-
-# reset settings
-unset PYTHON
-
-# checking for preferable python versions
-if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
- PYTHON=/usr/bin/python2.7
-fi
-
-if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
- PYTHON=/usr/bin/python2.6
-fi
-
-# if no preferable python versions found, try to use system one
-if [[ -z "$PYTHON" ]]; then
- PYTHON=/usr/bin/python
-fi
-
-# execute script
-$PYTHON "$@"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/slider-agent/src/test/python/python-wrap.sh
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/python-wrap.sh b/slider-agent/src/test/python/python-wrap.sh
new file mode 100755
index 0000000..8f30d67
--- /dev/null
+++ b/slider-agent/src/test/python/python-wrap.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export PYTHONPATH=/usr/lib/python2.7/site-packages/common_functions:$PYTHONPATH
+
+# reset settings
+unset PYTHON
+
+# checking for preferable python versions
+if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python2.7
+ export PYTHONPATH=/usr/lib/python2.7/site-packages/common_functions:$PYTHONPATH
+fi
+
+if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python2.6
+ export PYTHONPATH=/usr/lib/python2.6/site-packages/common_functions:$PYTHONPATH
+fi
+
+# if no preferable python versions found, try to use system one
+if [[ -z "$PYTHON" ]]; then
+ PYTHON=/usr/bin/python
+fi
+
+# execute script
+$PYTHON "$@"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index cdcea38..326af88 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -538,7 +538,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
+ <version>${maven-exec-plugin.version}</version>
<configuration>
<executable>java</executable>
<arguments>
@@ -562,7 +562,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
+ <version>${maven-exec-plugin.version}</version>
<executions>
<execution>
<goals>
[22/27] git commit: jaas fix for StormServer
Posted by sm...@apache.org.
jaas fix for StormServer
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4b2f8aad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4b2f8aad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4b2f8aad
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 4b2f8aad8d924cd1f2396b0057c885028210310b
Parents: b2f2d79 8da1503
Author: Jon Maron <jm...@hortonworks.com>
Authored: Mon Sep 8 09:27:56 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Mon Sep 8 09:27:56 2014 -0400
----------------------------------------------------------------------
README.md | 4 +-
app-packages/accumulo/appConfig.json | 7 +-
.../src/test/resources/appConfig_kerberos.json | 57 ++
.../test/resources/appConfig_monitor_ssl.json | 7 +-
.../src/test/resources/appConfig_ssl.json | 7 +-
app-packages/hbase/appConfig.json | 76 +--
app-packages/hbase/pom.xml | 2 +-
app-packages/storm/README.txt | 2 +-
app-packages/storm/appConfig.json | 13 +-
app-packages/storm/metainfo.xml | 26 +-
...pache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE | 16 -
app-packages/storm/package/scripts/params.py | 8 +-
app-packages/storm/package/scripts/service.py | 43 +-
.../storm/package/scripts/status_params.py | 4 +-
.../storm/package/templates/config.yaml.j2 | 9 -
app-packages/storm/pom.xml | 90 +++
app-packages/storm/resources.json | 12 +-
app-packages/storm/src/assembly/storm.xml | 68 +++
pom.xml | 42 +-
slider-agent/pom.xml | 4 +-
.../src/main/python/agent/Controller.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 10 +-
slider-agent/src/main/python/agent/Register.py | 5 +-
slider-agent/src/main/python/setup.py | 2 +-
.../src/test/python/agent/TestRegistration.py | 5 +-
slider-agent/src/test/python/python-wrap | 40 --
slider-agent/src/test/python/python-wrap.sh | 42 ++
slider-agent/src/test/python/unitTests.py | 36 +-
slider-assembly/pom.xml | 23 +-
slider-assembly/src/main/bash/README.md | 2 +-
slider-assembly/src/main/bash/slider_destroy | 4 +-
slider-assembly/src/main/scripts/slider | 2 +
slider-assembly/src/main/scripts/slider.py | 12 +-
slider-core/pom.xml | 4 +-
.../org/apache/slider/api/InternalKeys.java | 11 +
.../org/apache/slider/api/ResourceKeys.java | 13 -
.../org/apache/slider/client/SliderClient.java | 43 +-
.../common/params/AbstractActionArgs.java | 11 +
.../slider/common/params/ActionFreezeArgs.java | 2 +-
.../slider/common/params/ActionHelpArgs.java | 14 +-
.../slider/common/params/ActionVersionArgs.java | 12 +-
.../common/params/LaunchArgsAccessor.java | 2 +-
.../slider/common/params/SliderActions.java | 8 +-
.../apache/slider/common/tools/SliderUtils.java | 571 ++++++++++++++-----
.../slider/core/build/InstanceBuilder.java | 5 +-
.../apache/slider/core/zk/ZookeeperUtils.java | 3 +-
.../slider/providers/agent/AgentKeys.java | 2 +
.../providers/agent/AgentProviderService.java | 189 +++---
.../server/appmaster/RoleLaunchService.java | 8 +-
.../server/appmaster/SliderAppMaster.java | 71 ++-
.../server/appmaster/actions/ActionHalt.java | 6 +-
.../appmaster/actions/ActionKillContainer.java | 20 +-
.../appmaster/actions/ActionStartContainer.java | 8 +-
.../appmaster/actions/ActionStopSlider.java | 6 +-
.../server/appmaster/actions/AsyncAction.java | 47 +-
.../actions/ProviderReportedContainerLoss.java | 13 +-
.../actions/ProviderStartupCompleted.java | 4 -
.../server/appmaster/actions/QueueAccess.java | 5 +
.../server/appmaster/actions/QueueService.java | 11 +-
.../actions/RegisterComponentInstance.java | 5 +
.../actions/ReviewAndFlexApplicationSize.java | 43 ++
.../actions/UnregisterComponentInstance.java | 7 +-
.../server/appmaster/monkey/ChaosEntry.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 11 +-
.../server/appmaster/state/NodeEntry.java | 2 +-
.../server/appmaster/state/RoleHistory.java | 6 +-
.../appmaster/web/rest/agent/Register.java | 13 +
.../services/utility/EndOfServiceWaiter.java | 84 +++
.../services/workflow/ForkedProcessService.java | 21 +-
.../services/workflow/LongLivedProcess.java | 57 +-
.../agent/AgentMiniClusterTestBase.groovy | 9 +
.../agent/actions/TestActionExists.groovy | 2 +-
.../agent/actions/TestActionVersion.groovy | 5 +-
.../agent/freezethaw/TestFreezeCommands.groovy | 10 +-
.../TestFreezeThawMasterlessAM.groovy | 8 +-
.../freezethaw/TestFreezeUnknownCluster.groovy | 2 +-
.../standalone/TestBuildStandaloneAM.groovy | 2 +-
.../standalone/TestStandaloneAMDestroy.groovy | 6 +-
.../standalone/TestStandaloneAMKill.groovy | 2 +-
.../TestStandaloneAMMonkeyRestart.groovy | 8 +-
.../standalone/TestStandaloneRegistryAM.groovy | 2 +-
.../slider/client/TestCommonArgParsing.groovy | 2 +-
.../tools/TestClientResourceRegistration.groovy | 3 +
.../common/tools/TestConfigHelperHDFS.groovy | 3 +
.../tools/TestExecutionEnvironment.groovy | 42 ++
.../common/tools/TestMiscSliderUtils.groovy | 3 +
.../common/tools/TestWindowsSupport.groovy | 76 ++-
.../common/tools/TestZKIntegration.groovy | 54 +-
.../slider/providers/agent/AgentTestBase.groovy | 10 +
.../agent/TestAgentAMManagementWS.groovy | 8 +-
.../slider/providers/agent/TestAgentEcho.groovy | 6 +-
.../curator/TestRegistryRestResources.groovy | 6 +-
.../model/history/TestRoleHistoryRW.groovy | 4 +-
.../publisher/TestPublisherRestResources.groovy | 26 +-
.../apache/slider/test/SliderTestUtils.groovy | 11 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 32 +-
.../test/YarnZKMiniClusterTestBase.groovy | 2 +-
.../agent/TestAgentProviderService.java | 64 ++-
.../services/workflow/EndOfServiceWaiter.java | 56 --
.../workflow/ProcessCommandFactory.java | 21 +-
.../services/workflow/TestLongLivedProcess.java | 6 +-
.../TestWorkflowForkedProcessService.java | 4 +-
.../workflow/WorkflowServiceTestBase.java | 3 +-
.../src/test/resources/example-slider-test.xml | 4 +-
.../funtest/framework/CommandTestBase.groovy | 2 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 8 +-
.../src/test/manual/python/SliderTester.py | 4 +-
.../providers/accumulo/AccumuloTestBase.groovy | 4 +-
.../accumulo/live/TestAccFreezeThaw.groovy | 4 +-
.../funtest/HBaseClusterLifecycleIT.groovy | 8 +-
.../minicluster/HBaseMiniClusterTestBase.groovy | 1 -
.../TestFreezeThawClusterFromArchive.groovy | 2 +-
.../build/TestBuildThawClusterM1W1.groovy | 2 +-
.../flexing/TestClusterFlex1To1.groovy | 2 +-
...reezeReconfigureThawLiveRegionService.groovy | 4 +-
.../TestFreezeThawLiveRegionService.groovy | 4 +-
.../minicluster/live/TestTwoLiveClusters.groovy | 4 +-
src/test/clusters/sandbox/operations.md | 20 +-
src/test/clusters/ubuntu-secure/operations.md | 14 +-
119 files changed, 1812 insertions(+), 794 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --cc app-packages/storm/README.txt
index 2bae3d2,49adfce..288604a
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@@ -17,19 -17,16 +17,19 @@@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball and invoke mvn command
-with appropriate parameters.
+To create the app package you will need the Storm tarball copied to a specific location.
+Various configurations provided in this sample are customized for apache-storm-0.9.1.2.1.1.0-237.tar.gz.
+So if you use a different version you may need to edit a few config values.
-Command:
-mvn clean package -Pstorm-app-package -Dpkg.version=<version>
- -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
+Replace the placeholder tarball for Storm.
+ cp ~/Downloads/apache-storm-0.9.3.0.2.5.0-100.tar.gz package/files/
- rm package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE
++ rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
-Example:
-mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
- -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads
+Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/)
+ zip -r Apache_Storm_v_0_9_3.zip .
+
+Verify the content using
+ unzip -l "$@" Apache_Storm_v_0_9_3.zip
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --cc app-packages/storm/appConfig.json
index d6dee84,1eae599..33e38ac
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@@ -3,14 -3,15 +3,15 @@@
"metadata": {
},
"global": {
- "application.def": "Apache_Storm_v_0_9_3.zip",
+ "application.def": "slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
+ "system_configs": "core-site",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
-
+ "site.global.security_enabled": "false",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
@@@ -33,7 -32,7 +32,7 @@@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.0.2.5.0-100/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
++ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --cc app-packages/storm/metainfo.xml
index 89ee01e,bfd84e8..2c314b4
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@@ -150,7 -130,7 +130,7 @@@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.3.0.2.5.0-100.tar.gz</name>
- <name>files/${pkg.name}</name>
++ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/package/scripts/params.py
----------------------------------------------------------------------
diff --cc app-packages/storm/package/scripts/params.py
index e0bf9e4,38e25fb..ff85765
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@@ -57,7 -54,6 +53,7 @@@ if security_enabled
storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase)
storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab']
-- storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase)
++ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',nimbus_host.lower())
+ storm_jaas_stormclient_servicename = storm_jaas_server_principal.split("/")[0]
storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab']
kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --cc app-packages/storm/package/scripts/service.py
index 0ec6413,3a5e795..2150cd4
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@@ -47,10 -47,7 +47,7 @@@ def service
crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}")
if action == "start":
- if name == "rest_api":
- cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
- else:
- cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
- cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
++ cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
if params.security_enabled:
if name == "nimbus":
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b2f8aad/app-packages/storm/resources.json
----------------------------------------------------------------------
diff --cc app-packages/storm/resources.json
index 606dc39,017766c..f14b2c8
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@@ -11,16 -11,12 +11,12 @@@
"yarn.role.priority": "1",
"yarn.component.instances": "1"
},
- "STORM_REST_API": {
+ "STORM_UI_SERVER": {
"yarn.role.priority": "2",
- "yarn.component.instances": "1"
+ "yarn.component.instances": "0"
},
- "STORM_UI_SERVER": {
- "yarn.role.priority": "3",
- "yarn.component.instances": "1"
- },
"DRPC_SERVER": {
- "yarn.role.priority": "4",
+ "yarn.role.priority": "3",
"yarn.component.instances": "1"
},
"SUPERVISOR": {
@@@ -28,4 -24,4 +24,4 @@@
"yarn.component.instances": "1"
}
}
--}
++}
[05/27] git commit: Merge branch 'develop' into
feature/auto_generate_storm_pkg
Posted by sm...@apache.org.
Merge branch 'develop' into feature/auto_generate_storm_pkg
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26b4153e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26b4153e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26b4153e
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 26b4153ebf070521fb5030d57846f4ccd7f45cb0
Parents: e1129b0 38c816e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 13:46:44 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 13:46:44 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/api/InternalKeys.java | 11 +++++++++++
.../main/java/org/apache/slider/api/ResourceKeys.java | 13 -------------
.../apache/slider/server/appmaster/state/AppState.java | 9 +++++----
3 files changed, 16 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[24/27] git commit: SLIDER-404 Remove hadoop-2.4.1 profile where
compilation fails
Posted by sm...@apache.org.
SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3dd16210
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3dd16210
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3dd16210
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 3dd16210520ac6cb25fd3c1eb67e3d3b7640ca0b
Parents: f66c0cb
Author: tedyu <yu...@gmail.com>
Authored: Mon Sep 8 09:18:17 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Mon Sep 8 09:18:17 2014 -0700
----------------------------------------------------------------------
pom.xml | 7 -------
1 file changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dd16210/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d9b3813..c8607ff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1311,13 +1311,6 @@
<profile>
<!-- hadoop branch-2 builds -->
- <id>hadoop-2.4.1</id>
- <properties>
- <hadoop.version>2.4.1</hadoop.version>
- </properties>
- </profile>
- <profile>
- <!-- hadoop branch-2 builds -->
<id>branch-2</id>
<properties>
<hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
[04/27] git commit: SLIDER-181. Storm package updates
Posted by sm...@apache.org.
SLIDER-181. Storm package updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e1129b06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e1129b06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e1129b06
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: e1129b0609621356b47324dcbcabd612fd275be5
Parents: 201686e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 13:15:20 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 13:15:20 2014 -0700
----------------------------------------------------------------------
app-packages/storm/README.txt | 19 ++---
app-packages/storm/appConfig.json | 10 +--
app-packages/storm/metainfo.xml | 2 +-
...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16 ----
app-packages/storm/pom.xml | 90 ++++++++++++++++++++
app-packages/storm/src/assembly/storm.xml | 68 +++++++++++++++
6 files changed, 172 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index d4df85f..49adfce 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -17,19 +17,16 @@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball copied to a specific location.
-Various configurations provided in this sample are customized for apache-storm-0.9.3.2.2.0.0-578.tar.gz.
-So if you use a different version you may need to edit a few config values.
+To create the app package you will need the Storm tarball and invoke mvn command
+with appropriate parameters.
-Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/
- rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+Command:
+mvn clean package -Pstorm-app-package -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
-Create a zip package at the root of the package (<slider enlistment>/app-packages/storm/)
- zip -r apache-storm-0.9.3.zip .
-
-Verify the content using
- unzip -l "$@" apache-storm-0.9.3.zip
+Example:
+mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
+ -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index b5fe436..df28e8f 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,13 +3,13 @@
"metadata": {
},
"global": {
- "application.def": "apache-storm-0.9.3.zip",
+ "application.def": "slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"system_configs": "core-site",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
@@ -22,8 +22,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -31,7 +31,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 43535f4..bfd84e8 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -130,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.3.2.2.0.0-578.tar.gz</name>
+ <name>files/${pkg.name}</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
deleted file mode 100644
index dd934d5..0000000
--- a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
new file mode 100644
index 0000000..c08ed3d
--- /dev/null
+++ b/app-packages/storm/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.51.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-storm-app-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider Storm App Package</name>
+ <description>Slider Storm App Package</description>
+ <version>${pkg.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>storm-app-package</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/storm.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/src/assembly/storm.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
new file mode 100644
index 0000000..47d223a
--- /dev/null
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>slider-storm-v${storm.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
[21/27] git commit: more security changes
Posted by sm...@apache.org.
more security changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b2f2d796
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b2f2d796
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b2f2d796
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: b2f2d796b92455bba0dae9bcc1d8dc8fd6a6e7db
Parents: f4871c6
Author: Jon Maron <jm...@hortonworks.com>
Authored: Sun Sep 7 13:53:16 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Sun Sep 7 13:53:16 2014 -0400
----------------------------------------------------------------------
app-packages/storm/package/scripts/params.py | 7 ++++---
app-packages/storm/package/scripts/service.py | 2 +-
app-packages/storm/package/scripts/yaml_config.py | 2 +-
app-packages/storm/package/templates/storm_jaas.conf.j2 | 11 ++++-------
4 files changed, 10 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index a8e36b9..e0bf9e4 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -48,15 +48,16 @@ if ganglia_installed:
ganglia_port = config['configurations']['global']['ganglia_server_port']
security_enabled = config['configurations']['global']['security_enabled']
-hostname_lowercase = config['hostname'].lower()
if security_enabled:
+ _hostname_lowercase = config['hostname'].lower()
_kerberos_domain = config['configurations']['storm-env']['kerberos_domain']
_storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name']
_storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name']
- storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', hostname_lowercase)
+ storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase)
storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab']
- storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', hostname_lowercase)
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_jaas_stormclient_servicename = storm_jaas_server_principal.split("/")[0]
storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab']
kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index 50ac44c..0ec6413 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -50,7 +50,7 @@ def service(
if name == "rest_api":
cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
else:
- cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
+ cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
if params.security_enabled:
if name == "nimbus":
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/scripts/yaml_config.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/yaml_config.py b/app-packages/storm/package/scripts/yaml_config.py
index 0a1ffa9..5f763cc 100644
--- a/app-packages/storm/package/scripts/yaml_config.py
+++ b/app-packages/storm/package/scripts/yaml_config.py
@@ -38,7 +38,7 @@ def escape_yaml_propetry(value):
# if is map {'a':'b'}
if re.match('^\w*\{.+\}\w*$', value):
- unquouted = True
+ unquouted = True
try:
int(value)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2f2d796/app-packages/storm/package/templates/storm_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2 b/app-packages/storm/package/templates/storm_jaas.conf.j2
index e68cd57..a1ba6ea 100644
--- a/app-packages/storm/package/templates/storm_jaas.conf.j2
+++ b/app-packages/storm/package/templates/storm_jaas.conf.j2
@@ -20,8 +20,7 @@ StormServer {
useKeyTab=true
keyTab="{{storm_server_keytab_path}}"
storeKey=true
- useTicketCache=true
- renewTGT=true
+ useTicketCache=false
principal="{{storm_jaas_server_principal}}";
};
StormClient {
@@ -29,9 +28,8 @@ StormClient {
useKeyTab=true
keyTab="{{storm_client_keytab_path}}"
storeKey=true
- useTicketCache=true
- renewTGT=true
- serviceName="nimbus"
+ useTicketCache=false
+ serviceName="{{storm_jaas_stormclient_servicename}}"
debug=true
principal="{{storm_jaas_client_principal}}";
};
@@ -40,8 +38,7 @@ Client {
useKeyTab=true
keyTab="{{storm_client_keytab_path}}"
storeKey=true
- useTicketCache=true
- renewTGT=true
+ useTicketCache=false
serviceName="zookeeper"
principal="{{storm_jaas_client_principal}}";
};
[11/27] git commit: SLIDER-400 added zero port properties to sample
appConfigs
Posted by sm...@apache.org.
SLIDER-400 added zero port properties to sample appConfigs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/168a7af0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/168a7af0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/168a7af0
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 168a7af0251abb7d4b1c8017360fc21531ca83b7
Parents: e2a8032
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu Sep 4 20:22:40 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu Sep 4 20:22:40 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 2 ++
app-packages/accumulo/src/test/resources/appConfig_kerberos.json | 2 ++
.../accumulo/src/test/resources/appConfig_monitor_ssl.json | 2 ++
app-packages/accumulo/src/test/resources/appConfig_ssl.json | 2 ++
4 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/168a7af0/app-packages/accumulo/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
index 2c0c2d5..8828a64 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -40,6 +40,8 @@
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/168a7af0/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_kerberos.json b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
index c9684d4..b1bff37 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
@@ -42,6 +42,8 @@
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/168a7af0/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
index 45e6e19..61a9bb1 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
@@ -40,6 +40,8 @@
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/168a7af0/app-packages/accumulo/src/test/resources/appConfig_ssl.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_ssl.json
index 3daaaaf..e46d40e 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_ssl.json
@@ -42,6 +42,8 @@
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
[02/27] git commit: Temporary Storm package with Security
Posted by sm...@apache.org.
Temporary Storm package with Security
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0878e528
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0878e528
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0878e528
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 0878e528c78147de0aab4b251ca43afe5943e57b
Parents: eb6620e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Aug 28 18:30:42 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 28 18:30:42 2014 -0700
----------------------------------------------------------------------
app-packages/storm/README.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0878e528/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index 77474f1..2bae3d2 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -22,8 +22,8 @@ Various configurations provided in this sample are customized for apache-storm-0
So if you use a different version you may need to edit a few config values.
Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.1.2.1.1.0-237.tar.gz package/files/
- rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+ cp ~/Downloads/apache-storm-0.9.3.0.2.5.0-100.tar.gz package/files/
+ rm package/files/apache-storm-0.9.3.0.2.5.0-100.tar.gz.REPLACE
Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/)
zip -r Apache_Storm_v_0_9_3.zip .
[06/27] git commit: SLIDER-391. appConfig.json should allow reference
to other config properties and resolve them automatically
Posted by sm...@apache.org.
SLIDER-391. appConfig.json should allow reference to other config properties and resolve them automatically
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/02faa944
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/02faa944
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/02faa944
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 02faa944da56cd6252b48afbdb6ee427291f1fe0
Parents: 26b4153
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 18:17:13 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 18:17:13 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 8 +++--
app-packages/storm/appConfig.json | 7 ++--
.../providers/agent/AgentProviderService.java | 35 ++++++++++++++++----
.../agent/TestAgentProviderService.java | 29 ++++++++++++++++
4 files changed, 66 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 7df0dc3..07d1b4e 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -11,14 +11,16 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_port": "8667",
+ "site.global.ganglia_server_id": "Application1",
+ "site.global.ganglia_enabled":"true",
+
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
"site.global.monitor_protocol": "http",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index df28e8f..ac9bd49 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -11,6 +11,7 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
+
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
@@ -22,8 +23,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -31,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 61866fb..d188d49 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1196,7 +1196,8 @@ public class AgentProviderService extends AbstractProviderService implements
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- setInstallCommandConfigurations(cmd, containerId);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
+ cmd.setConfigurations(configurations);
cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
@@ -1246,12 +1247,6 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd, String containerId) throws SliderException {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
- cmd.setConfigurations(configurations);
- }
-
@VisibleForTesting
protected void addStatusCommand(String roleName,
String containerId,
@@ -1380,9 +1375,35 @@ public class AgentProviderService extends AbstractProviderService implements
configurations, tokens, containerId);
}
+ //do a final replacement of re-used configs
+ dereferenceAllConfigs(configurations);
+
return configurations;
}
+ protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
+ Map<String, String> allConfigs = new HashMap<String, String>();
+ String lookupFormat = "${site.%s.%s}";
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ allConfigs.put(String.format(lookupFormat, configType, configName), configBucket.get(configName));
+ }
+ }
+
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ String configValue = configBucket.get(configName);
+ for (String lookUpKey : allConfigs.keySet()) {
+ if (configValue != null && configValue.contains(lookUpKey)) {
+ configBucket.put(configName, configValue.replace(lookUpKey, allConfigs.get(lookUpKey)));
+ }
+ }
+ }
+ }
+ }
+
private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
Map<String, String> tokens = new HashMap<String, String>();
String nnuri = appConf.get("site.fs.defaultFS");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 16f5347..0f0e744 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -1128,4 +1128,33 @@ public class TestAgentProviderService {
Assert.assertFalse(configs.contains("bore-site"));
}
+ @Test
+ public void testDereferenceAllConfig() {
+ AgentProviderService aps = new AgentProviderService();
+ Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>();
+ Map<String, String> cfg1 = new HashMap<String, String>();
+ cfg1.put("a1", "${site.cfg-2.A1}");
+ cfg1.put("b1", "22");
+ cfg1.put("c1", "33");
+ cfg1.put("d1", "${site.cfg1.c1}AA");
+ Map<String, String> cfg2 = new HashMap<String, String>();
+ cfg2.put("A1", "11");
+ cfg2.put("B1", "${site.cfg-2.A1},${site.cfg-2.A1}");
+ cfg2.put("C1", "DD${site.cfg1.c1}");
+ cfg2.put("D1", "${14}");
+
+ allConfigs.put("cfg1", cfg1);
+ allConfigs.put("cfg-2", cfg2);
+ aps.dereferenceAllConfigs(allConfigs);
+ Assert.assertEquals("11", cfg1.get("a1"));
+ Assert.assertEquals("22", cfg1.get("b1"));
+ Assert.assertEquals("33", cfg1.get("c1"));
+ Assert.assertEquals("33AA", cfg1.get("d1"));
+
+ Assert.assertEquals("11", cfg2.get("A1"));
+ Assert.assertEquals("11,11", cfg2.get("B1"));
+ Assert.assertEquals("DD33", cfg2.get("C1"));
+ Assert.assertEquals("${14}", cfg2.get("D1"));
+ }
+
}
[08/27] git commit: SLIDER-397 taskkill /f
Posted by sm...@apache.org.
SLIDER-397 taskkill /f
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/27f0a625
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/27f0a625
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/27f0a625
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 27f0a625265de8e0d818917c94b2f0d5385b9044
Parents: 132d9f2
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 20:39:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 20:39:05 2014 +0100
----------------------------------------------------------------------
.../test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27f0a625/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 89f1a34..5086248 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -747,7 +747,7 @@ class SliderTestUtils extends Assert {
def commandString
if (!Shell.WINDOWS) {
- GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
+ GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill /f $signal"
log.info("Command command = $killCommand" )
commandString = ["bash", "-c", killCommand]
@@ -755,7 +755,7 @@ class SliderTestUtils extends Assert {
/*
"jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
*/
- GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /PID\""
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID "
commandString = ["CMD", "/C", killCommand]
}
Process command = commandString.execute()
[12/27] git commit: SLIDER-399. Agent command doesn't consider the
case where LOG_DIRS contains multiple directories
Posted by sm...@apache.org.
SLIDER-399. Agent command doesn't consider the case where LOG_DIRS contains multiple directories
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4c808204
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4c808204
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4c808204
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 4c808204d749f1504b768b12a98bf64f583e49d3
Parents: 02faa94
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 4 21:50:52 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 4 21:50:52 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/providers/agent/AgentKeys.java | 1 +
.../org/apache/slider/providers/agent/AgentProviderService.java | 3 +--
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4c808204/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 1622367..1da99b0 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -91,6 +91,7 @@ public interface AgentKeys {
String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
+ String AGENT_OUT_FILE = "agent.out";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4c808204/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index d188d49..02ef625 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -333,8 +333,7 @@ public class AgentProviderService extends AbstractProviderService implements
operation.add(debugCmd);
}
- String outfile = new File(logDir, "agent.out").toString();
- operation.add("> " + outfile + " 2>&1");
+ operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + AgentKeys.AGENT_OUT_FILE + " 2>&1");
launcher.addCommand(operation.build());
[10/27] git commit: Merge branch
'feature/SLIDER-397_windows_test_kill()_command_failing' into develop
Posted by sm...@apache.org.
Merge branch 'feature/SLIDER-397_windows_test_kill()_command_failing' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e2a80326
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e2a80326
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e2a80326
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: e2a80326a7f7ddc6f2bc8b00dbaa5265726a6bb0
Parents: 930920d ee3ad77
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 21:02:30 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 21:02:30 2014 +0100
----------------------------------------------------------------------
.../apache/slider/agent/standalone/TestStandaloneAMKill.groovy | 2 +-
.../test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 4 ++--
.../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[25/27] git commit: SLIDER-404 Remove hadoop-2.4.1 profile where
compilation fails
Posted by sm...@apache.org.
SLIDER-404 Remove hadoop-2.4.1 profile where compilation fails
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34954fd2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34954fd2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34954fd2
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 34954fd2151d6f0b42877cf9e50aacd7909136bb
Parents: 3dd1621
Author: tedyu <yu...@gmail.com>
Authored: Mon Sep 8 09:20:49 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Mon Sep 8 09:20:49 2014 -0700
----------------------------------------------------------------------
pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34954fd2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c8607ff..167b9ef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1303,8 +1303,8 @@
<!-- local builds of everything -->
<id>local</id>
<properties>
- <hadoop.version>2.4.1-SNAPSHOT</hadoop.version>
- <hbase.version>0.98.4-SNAPSHOT</hbase.version>
+ <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
+ <hbase.version>0.98.5-SNAPSHOT</hbase.version>
<accumulo.version>1.6.0-SNAPSHOT</accumulo.version>
</properties>
</profile>
[16/27] git commit: SLIDER-401 agent tests on windows
Posted by sm...@apache.org.
SLIDER-401 agent tests on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce3f7009
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce3f7009
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce3f7009
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: ce3f7009e6e9fec204b264a9577a52632b9f3487
Parents: a9f0b98
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 13:47:50 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 13:47:50 2014 +0100
----------------------------------------------------------------------
slider-agent/src/main/python/setup.py | 2 +-
slider-agent/src/test/python/unitTests.py | 31 ++++++++++++--------------
2 files changed, 15 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce3f7009/slider-agent/src/main/python/setup.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/setup.py b/slider-agent/src/main/python/setup.py
index 421b5f9..56969b6 100644
--- a/slider-agent/src/main/python/setup.py
+++ b/slider-agent/src/main/python/setup.py
@@ -17,7 +17,7 @@ from setuptools import setup
setup(
name = "slider-agent",
- version = "0.31.0-incubating-SNAPSHOT",
+ version = "0.51.0-incubating-SNAPSHOT",
packages = ['agent'],
# metadata for upload to PyPI
author = "Apache Software Foundation",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce3f7009/slider-agent/src/test/python/unitTests.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py
index b01438e..bab121f 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-'''
+"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -16,14 +16,13 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-'''
+"""
import unittest
-import doctest
-from os.path import dirname, split, isdir
-import logging.handlers
+from os.path import isdir
import logging
-from random import shuffle
+import os
+import sys
LOG_FILE_NAME='tests.log'
SELECTED_PREFIX = "_"
@@ -31,9 +30,9 @@ PY_EXT='.py'
ignoredDirs = ["mock"]
class TestAgent(unittest.TestSuite):
- def run(self, result):
+ def run(self, result, debug=False):
run = unittest.TestSuite.run
- run(self, result)
+ run(self, result, debug)
return result
@@ -41,19 +40,19 @@ def parent_dir(path):
if isdir(path):
if path.endswith(os.sep):
path = os.path.dirname(path)
- parent_dir = os.path.dirname(path)
+ parent = os.path.dirname(path)
else:
- parent_dir = os.path.dirname(os.path.dirname(path))
+ parent = os.path.dirname(os.path.dirname(path))
- return parent_dir
+ return parent
def all_tests_suite():
- src_dir = os.getcwd()
+ root_dir = os.getcwd()
files_list = []
- for directory in os.listdir(src_dir):
+ for directory in os.listdir(root_dir):
if os.path.isdir(directory) and not directory in ignoredDirs:
- files_list += os.listdir(src_dir + os.sep + directory)
+ files_list += os.listdir(root_dir + os.sep + directory)
## temporarily deleting to add more predictability
## shuffle(files_list)
files_list.sort()
@@ -99,9 +98,7 @@ def main():
logger.info('------------------------------------------------------------------------')
if __name__ == '__main__':
- import os
- import sys
- import io
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python')
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python' + os.sep + 'agent')
[20/27] git commit: SLIDER-405. Minor changes to the rpm and scripts
to facilitate easy deployment
Posted by sm...@apache.org.
SLIDER-405. Minor changes to the rpm and scripts to facilitate easy deployment
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8da15036
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8da15036
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8da15036
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 8da150363571f7ab1a0b018593bc5907d501748d
Parents: 486ef84
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Sep 6 08:32:07 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Sep 6 08:32:07 2014 -0700
----------------------------------------------------------------------
slider-assembly/pom.xml | 23 +++++++++++++++++++++--
slider-assembly/src/main/scripts/slider | 2 ++
slider-assembly/src/main/scripts/slider.py | 10 +++++++---
3 files changed, 30 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8da15036/slider-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index dbbd722..c2d0ecb 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -33,7 +33,7 @@
<rpm.bindir>${rpm.basedir}/bin</rpm.bindir>
<rpm.libdir>${rpm.basedir}/lib</rpm.libdir>
<rpm.agentdir>${rpm.basedir}/agent</rpm.agentdir>
- <rpm.username>mapred</rpm.username>
+ <rpm.username>root</rpm.username>
<rpm.groupname>hadoop</rpm.groupname>
<src.confdir>src/conf-hdp</src.confdir>
<src.libdir>${project.build.directory}/lib</src.libdir>
@@ -287,10 +287,10 @@
<!-- agent -->
<mapping>
<directory>${rpm.agentdir}</directory>
- <configuration>true</configuration>
<filemode>0755</filemode>
<username>${rpm.username}</username>
<groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>false</directoryIncluded>
<sources>
<source>
<location>${project.build.directory}/agent</location>
@@ -308,6 +308,7 @@
<filemode>0755</filemode>
<username>${rpm.username}</username>
<groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>false</directoryIncluded>
<sources>
<source>
<location>${src.agent.ini.dir}</location>
@@ -318,6 +319,24 @@
</source>
</sources>
</mapping>
+
+ <!-- needed to apply attribute to directory -->
+ <mapping>
+ <directory>${rpm.agentdir}</directory>
+ <filemode>0755</filemode>
+ <username>${rpm.username}</username>
+ <groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>true</directoryIncluded>
+ </mapping>
+
+ <mapping>
+ <directory>${rpm.agentdir}/conf</directory>
+ <filemode>0755</filemode>
+ <username>${rpm.username}</username>
+ <groupname>${rpm.groupname}</groupname>
+ <directoryIncluded>true</directoryIncluded>
+ </mapping>
+
</mappings>
<!--
Scripts. Very dangerous in RPMs unless you know exactly what you are doing.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8da15036/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
index 6cd55ad..477e237 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -22,6 +22,8 @@
# The env variable SLIDER_JVM_OPTS can be used to override
# the default JVM opts
+export JAVA_HOME=${JAVA_HOME}
+
function usage
{
echo "Usage: slider <action> <arguments>"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8da15036/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index be79973..cbc0faf 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -23,6 +23,7 @@ CONF = "conf"
LIB = "lib"
+JAVA_HOME = None
SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"
SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
@@ -148,8 +149,11 @@ def java(classname, args, classpath, jvm_opts_list):
# split the JVM opts by space
# java = "/usr/bin/java"
prg = "java"
- if which("java") is None:
- prg = os.environ["JAVA_HOME"] + "/bin/java"
+ if JAVA_HOME:
+ prg = os.path.join(JAVA_HOME, "bin", "java")
+ else:
+ if which("java") is None:
+ prg = os.path.join(os.environ["JAVA_HOME"], "bin", "java")
commandline = [prg]
commandline.extend(jvm_opts_list)
commandline.append("-classpath")
@@ -203,7 +207,7 @@ if __name__ == '__main__':
try:
returncode = main()
except Exception as e:
- print "Exception: %s " % e.message
+ print "Exception: %s " % str(e)
returncode = -1
sys.exit(returncode)
[03/27] git commit: cheanges for secure deployment
Posted by sm...@apache.org.
cheanges for secure deployment
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f4871c6f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f4871c6f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f4871c6f
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: f4871c6fcd2f093a313933de63107bd7e5c372b3
Parents: 0878e52
Author: Jon Maron <jm...@hortonworks.com>
Authored: Tue Sep 2 19:22:27 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Tue Sep 2 19:22:27 2014 -0400
----------------------------------------------------------------------
app-packages/storm/package/scripts/params.py | 9 ++++-----
app-packages/storm/package/scripts/service.py | 4 ++--
app-packages/storm/package/scripts/storm.py | 9 ++++++---
app-packages/storm/package/scripts/yaml_config.py | 10 +++++++++-
4 files changed, 21 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index 93c6018..a8e36b9 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -47,17 +47,16 @@ if ganglia_installed:
ganglia_server = config['configurations']['global']['ganglia_server_host']
ganglia_port = config['configurations']['global']['ganglia_server_port']
-_authentication = config['configurations']['core-site']['hadoop.security.authentication']
-security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+security_enabled = config['configurations']['global']['security_enabled']
+hostname_lowercase = config['hostname'].lower()
if security_enabled:
- _hostname_lowercase = config['hostname'].lower()
_kerberos_domain = config['configurations']['storm-env']['kerberos_domain']
_storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name']
_storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name']
- storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', hostname_lowercase)
storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab']
- storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', hostname_lowercase)
storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab']
kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index dc3c733..50ac44c 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -54,10 +54,10 @@ def service(
if params.security_enabled:
if name == "nimbus":
- Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_user}"),
+ Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_jaas_server_principal}"),
user=params.storm_user)
else:
- Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_user}"),
+ Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_jaas_client_principal}"),
user=params.storm_user)
Execute(cmd,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/storm.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/storm.py b/app-packages/storm/package/scripts/storm.py
index efca073..8ecb3a1 100644
--- a/app-packages/storm/package/scripts/storm.py
+++ b/app-packages/storm/package/scripts/storm.py
@@ -50,6 +50,9 @@ def storm():
)
if params.security_enabled:
- TemplateConfig( format("{conf_dir}/storm_jaas.conf"),
- owner = params.storm_user
- )
\ No newline at end of file
+ File(format("{conf_dir}/storm_jaas.conf"),
+ content=Template("storm_jaas.conf.j2"),
+ owner = params.storm_user,
+ group = params.user_group
+ )
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f4871c6f/app-packages/storm/package/scripts/yaml_config.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/yaml_config.py b/app-packages/storm/package/scripts/yaml_config.py
index d25089c..0a1ffa9 100644
--- a/app-packages/storm/package/scripts/yaml_config.py
+++ b/app-packages/storm/package/scripts/yaml_config.py
@@ -19,9 +19,13 @@ limitations under the License.
"""
import re
+import socket
from resource_management import *
def escape_yaml_propetry(value):
+ # pre-process value for any "_HOST" tokens
+ value = value.replace('_HOST', socket.getfqdn())
+
unquouted = False
unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"]
@@ -31,7 +35,11 @@ def escape_yaml_propetry(value):
# if is list [a,b,c]
if re.match('^\w*\[.+\]\w*$', value):
unquouted = True
-
+
+ # if is map {'a':'b'}
+ if re.match('^\w*\{.+\}\w*$', value):
+ unquouted = True
+
try:
int(value)
unquouted = True
[07/27] git commit: SLIDER-397 windows kill() command failing in
TestStandaloneAMKill
Posted by sm...@apache.org.
SLIDER-397 windows kill() command failing in TestStandaloneAMKill
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/132d9f28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/132d9f28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/132d9f28
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: 132d9f288a9d91790409bd4482731eb27cf58bba
Parents: 930920d
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 20:31:29 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 20:31:29 2014 +0100
----------------------------------------------------------------------
.../apache/slider/agent/standalone/TestStandaloneAMKill.groovy | 2 +-
.../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/132d9f28/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
index 75f9a2c..7552394 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
@@ -58,7 +58,7 @@ class TestStandaloneAMKill extends AgentMiniClusterTestBase {
describe("listing Java processes")
lsJavaProcesses();
describe("killing AM")
- killAM(SIGTERM);
+ assert 0 == killAM(SIGTERM);
waitWhileClusterLive(sliderClient);
//give yarn some time to notice
sleep(10000)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/132d9f28/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 7ffa64a..97cc853 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -318,10 +318,10 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
/**
- * Kill all Slider Services. That i
+ * Kill all Slider Services.
* @param signal
*/
- public void killAM(int signal) {
+ public int killAM(int signal) {
killJavaProcesses(SliderAppMaster.SERVICE_CLASSNAME_SHORT, signal)
}
[26/27] git commit: SLIDER-181. Storm package updates (fixing some
auto-build issues)
Posted by sm...@apache.org.
SLIDER-181. Storm package updates (fixing some auto-build issues)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bff5ed6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bff5ed6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bff5ed6d
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: bff5ed6d2994978a08ea709fcc83d35aa8c94566
Parents: 34954fd
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Sep 8 09:49:20 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Sep 8 09:49:20 2014 -0700
----------------------------------------------------------------------
app-packages/storm/appConfig.json | 2 +-
app-packages/storm/metainfo.xml | 2 +-
app-packages/storm/resources.json | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index 33e38ac..bb607e9 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -32,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 2c314b4..d201f29 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -130,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ <name>files/apache-storm-${pkg.version}.tar.gz</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bff5ed6d/app-packages/storm/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json
index f14b2c8..edfdc1f 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -13,7 +13,7 @@
},
"STORM_UI_SERVER": {
"yarn.role.priority": "2",
- "yarn.component.instances": "0"
+ "yarn.component.instances": "1"
},
"DRPC_SERVER": {
"yarn.role.priority": "3",
[17/27] git commit: SLIDER-401 agent tests on windows
Posted by sm...@apache.org.
SLIDER-401 agent tests on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b323df81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b323df81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b323df81
Branch: refs/heads/feature/SLIDER-280_Restart_AM_fun_tests
Commit: b323df811a138a0880b60be95d9150419d1dcb85
Parents: ce3f700
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 13:53:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 13:53:05 2014 +0100
----------------------------------------------------------------------
slider-agent/src/test/python/unitTests.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b323df81/slider-agent/src/test/python/unitTests.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py
index bab121f..6aa0167 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -70,8 +70,9 @@ def all_tests_suite():
else:
for file_name in files_list:
if file_name.endswith(PY_EXT) and not file_name == __file__:
- logger.info(file_name)
- tests_list.append(file_name.replace(PY_EXT, ''))
+ replaced = file_name.replace(PY_EXT, '')
+ logger.info(replaced)
+ tests_list.append(replaced)
logger.info('------------------------------------------------------------------------')
suite = unittest.TestLoader().loadTestsFromNames(tests_list)