You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/02/11 14:59:09 UTC
[1/2] git commit: AMBARI-4579. Add Storm REST API service to stack
definition for Managing/Configuration. (Arsen Babych via aonishuk)
Updated Branches:
refs/heads/trunk 6cbdded04 -> 557b132f1
AMBARI-4579. Add Storm REST API service to stack definition for
Managing/Configuration. (Arsen Babych via aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/557b132f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/557b132f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/557b132f
Branch: refs/heads/trunk
Commit: 557b132f1fd9d2546040dfc6e53a92a21c567368
Parents: 53ef1ce
Author: Andrew Onischuk <ao...@hortonworks.com>
Authored: Tue Feb 11 05:36:53 2014 -0800
Committer: Andrew Onischuk <ao...@hortonworks.com>
Committed: Tue Feb 11 05:58:55 2014 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.1.1/role_command_order.json | 4 +-
.../HDP/2.1.1/services/STORM/metainfo.xml | 10 ++++
.../services/STORM/package/scripts/params.py | 6 ++
.../services/STORM/package/scripts/rest_api.py | 58 ++++++++++++++++++++
.../services/STORM/package/scripts/service.py | 31 ++++++-----
.../STORM/package/scripts/status_params.py | 5 +-
.../services/STORM/package/scripts/storm.py | 6 ++
.../STORM/package/templates/config.yaml.j2 | 26 +++++++++
8 files changed, 130 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/557b132f/ambari-server/src/main/resources/stacks/HDP/2.1.1/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.1.1/role_command_order.json
index a82be1d..4a5935e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/role_command_order.json
@@ -9,6 +9,7 @@
"SUPERVISOR-START" : ["NIMBUS-START"],
"STORM_UI_SERVER-START" : ["NIMBUS-START"],
"DRPC_SERVER-START" : ["NIMBUS-START"],
+ "STORM_REST_API-START" : ["NIMBUS-START", "STORM_UI_SERVER-START", "SUPERVISOR-START", "DRPC_SERVER-START"],
"LOGVIEWER_SERVER-START" : ["NIMBUS-START"],
"HBASE_MASTER-START": ["ZOOKEEPER_SERVER-START"],
"HBASE_REGIONSERVER-START": ["HBASE_MASTER-START"],
@@ -34,7 +35,8 @@
"SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"],
"ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
"ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
- "STORM_SERVICE_CHECK-SERVICE_CHECK": ["NIMBUS-START", "SUPERVISOR-START"],
+ "STORM_SERVICE_CHECK-SERVICE_CHECK": ["NIMBUS-START", "SUPERVISOR-START", "STORM_UI_SERVER-START",
+ "DRPC_SERVER-START", "LOGVIEWER_SERVER-START"],
"ZOOKEEPER_SERVER-STOP" : ["HBASE_MASTER-STOP", "HBASE_REGIONSERVER-STOP"],
"HBASE_MASTER-STOP": ["HBASE_REGIONSERVER-STOP"],
"NIMBUS-STOP" : ["SUPERVISOR-STOP", "STORM_UI_SERVER-STOP", "DRPC_SERVER-STOP", "LOGVIEWER_SERVER-STOP"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/557b132f/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml
index a15d3a1..27b039c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml
@@ -36,6 +36,16 @@
</component>
<component>
+ <name>STORM_REST_API</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/rest_api.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
<name>SUPERVISOR</name>
<category>SLAVE</category>
<commandScript>
http://git-wip-us.apache.org/repos/asf/ambari/blob/557b132f/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/params.py
index 57ff774..ec6bcfb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/params.py
@@ -32,3 +32,9 @@ local_dir = config['configurations']['storm-site']['storm.local.dir']
user_group = config['configurations']['global']['user_group']
java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
+nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
+nimbus_host = config['configurations']['storm-site']['nimbus.host']
+rest_api_port = "8745"
+rest_api_admin_port = "8746"
+rest_api_conf_file = format("{conf_dir}/config.yaml")
+rest_lib_dir = "/usr/lib/storm/contrib/storm-rest"
http://git-wip-us.apache.org/repos/asf/ambari/blob/557b132f/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/rest_api.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/rest_api.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/rest_api.py
new file mode 100644
index 0000000..7b12597
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/rest_api.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+from service_check import ServiceCheck
+
+
+class RestApi(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ storm()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ service("rest_api", action="start")
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ service("rest_api", action="stop")
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.pid_rest_api)
+
+if __name__ == "__main__":
+ RestApi().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/557b132f/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/service.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/service.py
index 2626d1d..e12e39b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/service.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/service.py
@@ -33,24 +33,29 @@ def service(
pid_file = status_params.pid_files[name]
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
- if action == "start":
- cmd = ["/usr/bin/storm", name]
- if name == "ui":
- crt_pid_cmd = format("pgrep -f \"^java.+backtype.storm.ui.core$\" > {pid_file}")
- else :
- crt_pid_cmd = format("pgrep -f \"^java.+backtype.storm.daemon.{name}$\" > {pid_file}")
+ if name == 'ui':
+ process_cmd = "^java.+backtype.storm.ui.core$"
+ elif name == "rest_api":
+ process_cmd = format("java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
+ else:
+ process_cmd = format("^java.+backtype.storm.daemon.{name}$")
- #Execute(cmd,
- # not_if=no_op_test,
- # user=params.storm_user
- #)
+ crt_pid_cmd = format("pgrep -f \"{process_cmd}\" > {pid_file}")
- #TODO run from storm user
+ if action == "start":
+ if name == "rest_api":
+ cmd = format("env PATH=$PATH:{java64_home}/bin {process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
+ else:
+ cmd = format("env PATH=$PATH:{java64_home}/bin /usr/bin/storm {name}")
- if call(no_op_test)[0]:
- subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env={"PATH":format("{java64_home}/bin:/bin")})
+ Execute(cmd,
+ not_if=no_op_test,
+ user=params.storm_user,
+ wait_for_finish=False
+ )
Execute(crt_pid_cmd,
+ user=params.storm_user,
logoutput=True,
tries=6,
try_sleep=10
http://git-wip-us.apache.org/repos/asf/ambari/blob/557b132f/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/status_params.py
index 70b034a..66b2a57 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/status_params.py
@@ -27,9 +27,10 @@ pid_supervisor = format("{pid_dir}/supervisor.pid")
pid_drpc = format("{pid_dir}/drpc.pid")
pid_ui = format("{pid_dir}/ui.pid")
pid_logviewer = format("{pid_dir}/logviewer.pid")
-
+pid_rest_api = format("{pid_dir}/restapi.pid")
pid_files = {"logviewer":pid_logviewer,
"ui": pid_ui,
"nimbus": pid_nimbus,
"supervisor": pid_supervisor,
- "drpc": pid_drpc}
\ No newline at end of file
+ "drpc": pid_drpc,
+ "rest_api": pid_rest_api}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/557b132f/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/storm.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/storm.py
index d38909c..e790904 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/storm.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/storm.py
@@ -31,6 +31,12 @@ def storm():
recursive=True
)
+ File(format("{conf_dir}/config.yaml"),
+ content=Template("config.yaml.j2"),
+ owner = params.storm_user,
+ group = params.user_group
+ )
+
yaml_config( "storm.yaml",
conf_dir = params.conf_dir,
configurations = params.config['configurations']['storm-site'],
http://git-wip-us.apache.org/repos/asf/ambari/blob/557b132f/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/templates/config.yaml.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/templates/config.yaml.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/templates/config.yaml.j2
new file mode 100644
index 0000000..4e93fb8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/templates/config.yaml.j2
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nimbusHost: {{nimbus_host}}
+nimbusPort: {{nimbus_port}}
+
+# HTTP-specific options.
+http:
+
+ # The port on which the HTTP server listens for service requests.
+ port: {{rest_api_port}}
+
+ # The port on which the HTTP server listens for administrative requests.
+ adminPort: {{rest_api_admin_port}}
\ No newline at end of file
[2/2] git commit: AMBARI-4599. Temporary directory does not get
created during HBase client installation using Ambari (Arsen Babych via
aonishuk)
Posted by ao...@apache.org.
AMBARI-4599. Temporary directory does not get created during HBase
client installation using Ambari (Arsen Babych via aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/53ef1ce7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/53ef1ce7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/53ef1ce7
Branch: refs/heads/trunk
Commit: 53ef1ce765e3536899439995855fdd341df6036f
Parents: 6cbdded
Author: Andrew Onischuk <ao...@hortonworks.com>
Authored: Tue Feb 11 05:34:39 2014 -0800
Committer: Andrew Onischuk <ao...@hortonworks.com>
Committed: Tue Feb 11 05:58:55 2014 -0800
----------------------------------------------------------------------
.../services/HBASE/package/scripts/hbase.py | 9 ++++++--
.../services/HBASE/package/scripts/hbase.py | 11 ++++++---
.../stacks/1.3.2/HBASE/test_hbase_client.py | 10 +++++++-
.../stacks/2.0.6/HBASE/test_hbase_client.py | 24 +++++++++++++-------
4 files changed, 40 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/53ef1ce7/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py
index fddd1b7..e77a233 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py
@@ -31,7 +31,12 @@ def hbase(type=None # 'master' or 'regionserver' or 'client'
group = params.user_group,
recursive = True
)
-
+
+ Directory ( params.tmp_dir,
+ owner = params.hbase_user,
+ recursive = True
+ )
+
XmlConfig( "hbase-site.xml",
conf_dir = params.conf_dir,
configurations = params.config['configurations']['hbase-site'],
@@ -76,7 +81,7 @@ def hbase(type=None # 'master' or 'regionserver' or 'client'
recursive = True
)
- Directory ( [params.tmp_dir, params.log_dir],
+ Directory (params.log_dir,
owner = params.hbase_user,
recursive = True
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/53ef1ce7/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
index 95f3e30..c0f48dc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
@@ -31,7 +31,12 @@ def hbase(type=None # 'master' or 'regionserver' or 'client'
group = params.user_group,
recursive = True
)
-
+
+ Directory (params.tmp_dir,
+ owner = params.hbase_user,
+ recursive = True
+ )
+
XmlConfig( "hbase-site.xml",
conf_dir = params.conf_dir,
configurations = params.config['configurations']['hbase-site'],
@@ -45,7 +50,7 @@ def hbase(type=None # 'master' or 'regionserver' or 'client'
owner = params.hbase_user,
group = params.user_group
)
-
+
if 'hbase-policy' in params.config['configurations']:
XmlConfig( "hbase-policy.xml",
configurations = params.config['configurations']['hbase-policy'],
@@ -76,7 +81,7 @@ def hbase(type=None # 'master' or 'regionserver' or 'client'
recursive = True
)
- Directory ( [params.tmp_dir, params.log_dir],
+ Directory (params.log_dir,
owner = params.hbase_user,
recursive = True
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/53ef1ce7/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py
index f226616..5ef19c3 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py
@@ -35,6 +35,10 @@ class TestHBaseClient(RMFTestCase):
group = 'hadoop',
recursive = True,
)
+ self.assertResourceCalled('Directory', '/hadoop/hbase',
+ owner = 'hbase',
+ recursive = True,
+ )
self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
owner = 'hbase',
group = 'hadoop',
@@ -88,7 +92,11 @@ class TestHBaseClient(RMFTestCase):
owner = 'hbase',
group = 'hadoop',
recursive = True,
- )
+ )
+ self.assertResourceCalled('Directory', '/hadoop/hbase',
+ owner = 'hbase',
+ recursive = True,
+ )
self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
owner = 'hbase',
group = 'hadoop',
http://git-wip-us.apache.org/repos/asf/ambari/blob/53ef1ce7/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
index b0d4d31..bd3228f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
@@ -29,12 +29,16 @@ class TestHBaseClient(RMFTestCase):
command = "configure",
config_file="secured.json"
)
-
+
self.assertResourceCalled('Directory', '/etc/hbase/conf',
owner = 'hbase',
group = 'hadoop',
recursive = True,
)
+ self.assertResourceCalled('Directory', '/hadoop/hbase',
+ owner = 'hbase',
+ recursive = True,
+ )
self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
owner = 'hbase',
group = 'hadoop',
@@ -83,36 +87,40 @@ class TestHBaseClient(RMFTestCase):
command = "configure",
config_file="default.json"
)
-
+
self.assertResourceCalled('Directory', '/etc/hbase/conf',
owner = 'hbase',
group = 'hadoop',
recursive = True,
- )
+ )
+ self.assertResourceCalled('Directory', '/hadoop/hbase',
+ owner = 'hbase',
+ recursive = True,
+ )
self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
configurations = self.getConfig()['configurations']['hbase-site'], # don't hardcode all the properties
- )
+ )
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
- )
+ )
self.assertResourceCalled('File', '/etc/hbase/conf/hbase-policy.xml',
owner = 'hbase',
group = 'hadoop',
- )
+ )
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hbase-env.sh',
owner = 'hbase',
template_tag = None,
- )
+ )
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
template_tag = 'GANGLIA-RS',
- )
+ )
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/regionservers',
owner = 'hbase',
template_tag = None,