You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2017/06/28 00:24:43 UTC
[45/51] [partial] ambari git commit: AMBARI-21349. Create BigInsights
Stack Skeleton in Ambari 2.5 (alejandro)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/ams.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/ams.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/ams.py
new file mode 100755
index 0000000..e72ad82
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/ams.py
@@ -0,0 +1,388 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons.str_utils import compress_backslashes
+import glob
+import os
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def ams(name=None):
+ import params
+ if name == 'collector':
+ if not check_windows_service_exists(params.ams_collector_win_service_name):
+ Execute(format("cmd /C cd {ams_collector_home_dir} & ambari-metrics-collector.cmd setup"))
+
+ Directory(params.ams_collector_conf_dir,
+ owner=params.ams_user,
+ create_parents=True
+ )
+
+ Directory(params.ams_checkpoint_dir,
+ owner=params.ams_user,
+ create_parents=True
+ )
+
+ XmlConfig("ams-site.xml",
+ conf_dir=params.ams_collector_conf_dir,
+ configurations=params.config['configurations']['ams-site'],
+ configuration_attributes=params.config['configuration_attributes']['ams-site'],
+ owner=params.ams_user,
+ )
+
+ merged_ams_hbase_site = {}
+ merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
+ if params.security_enabled:
+ merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
+
+ XmlConfig( "hbase-site.xml",
+ conf_dir = params.ams_collector_conf_dir,
+ configurations = merged_ams_hbase_site,
+ configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
+ owner = params.ams_user,
+ )
+
+ if (params.log4j_props != None):
+ File(os.path.join(params.ams_collector_conf_dir, "log4j.properties"),
+ owner=params.ams_user,
+ content=params.log4j_props
+ )
+
+ File(os.path.join(params.ams_collector_conf_dir, "ams-env.cmd"),
+ owner=params.ams_user,
+ content=InlineTemplate(params.ams_env_sh_template)
+ )
+
+ ServiceConfig(params.ams_collector_win_service_name,
+ action="change_user",
+ username = params.ams_user,
+ password = Script.get_password(params.ams_user))
+
+ if not params.is_local_fs_rootdir:
+ # Configuration needed to support NN HA
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.ams_collector_conf_dir,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.hbase_conf_dir,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("core-site.xml",
+ conf_dir=params.ams_collector_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("core-site.xml",
+ conf_dir=params.hbase_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ else:
+ ServiceConfig(params.ams_embedded_hbase_win_service_name,
+ action="change_user",
+ username = params.ams_user,
+ password = Script.get_password(params.ams_user))
+ # creating symbolic links on ams jars to make them available to services
+ links_pairs = [
+ ("%COLLECTOR_HOME%\\hbase\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
+ "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"),
+ ]
+ for link_pair in links_pairs:
+ link, target = link_pair
+ real_link = os.path.expandvars(link)
+ target = compress_backslashes(glob.glob(os.path.expandvars(target))[0])
+ if not os.path.exists(real_link):
+ #TODO check the symlink destination too. Broken in Python 2.x on Windows.
+ Execute('cmd /c mklink "{0}" "{1}"'.format(real_link, target))
+ pass
+
+ elif name == 'monitor':
+ if not check_windows_service_exists(params.ams_monitor_win_service_name):
+ Execute(format("cmd /C cd {ams_monitor_home_dir} & ambari-metrics-monitor.cmd setup"))
+
+ # creating symbolic links on ams jars to make them available to services
+ links_pairs = [
+ ("%HADOOP_HOME%\\share\\hadoop\\common\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
+ "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"),
+ ("%HBASE_HOME%\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
+ "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"),
+ ]
+ for link_pair in links_pairs:
+ link, target = link_pair
+ real_link = os.path.expandvars(link)
+ target = compress_backslashes(glob.glob(os.path.expandvars(target))[0])
+ if not os.path.exists(real_link):
+ #TODO check the symlink destination too. Broken in Python 2.x on Windows.
+ Execute('cmd /c mklink "{0}" "{1}"'.format(real_link, target))
+
+ Directory(params.ams_monitor_conf_dir,
+ owner=params.ams_user,
+ create_parents=True
+ )
+
+ TemplateConfig(
+ os.path.join(params.ams_monitor_conf_dir, "metric_monitor.ini"),
+ owner=params.ams_user,
+ template_tag=None
+ )
+
+ TemplateConfig(
+ os.path.join(params.ams_monitor_conf_dir, "metric_groups.conf"),
+ owner=params.ams_user,
+ template_tag=None
+ )
+
+ ServiceConfig(params.ams_monitor_win_service_name,
+ action="change_user",
+ username = params.ams_user,
+ password = Script.get_password(params.ams_user))
+
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def ams(name=None):
+ import params
+
+ if name == 'collector':
+ Directory(params.ams_collector_conf_dir,
+ owner=params.ams_user,
+ group=params.user_group,
+ create_parents=True
+ )
+
+ Execute(('chown', '-R', params.ams_user, params.ams_collector_conf_dir),
+ sudo=True
+ )
+
+ Directory(params.ams_checkpoint_dir,
+ owner=params.ams_user,
+ group=params.user_group,
+ cd_access="a",
+ create_parents=True
+ )
+
+ Execute(('chown', '-R', params.ams_user, params.ams_checkpoint_dir),
+ sudo=True
+ )
+
+ XmlConfig("ams-site.xml",
+ conf_dir=params.ams_collector_conf_dir,
+ configurations=params.config['configurations']['ams-site'],
+ configuration_attributes=params.config['configuration_attributes']['ams-site'],
+ owner=params.ams_user,
+ group=params.user_group
+ )
+
+ merged_ams_hbase_site = {}
+ merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
+ if params.security_enabled:
+ merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
+
+ # Add phoenix client side overrides
+ merged_ams_hbase_site['phoenix.query.maxGlobalMemoryPercentage'] = str(params.phoenix_max_global_mem_percent)
+ merged_ams_hbase_site['phoenix.spool.directory'] = params.phoenix_client_spool_dir
+
+ XmlConfig( "hbase-site.xml",
+ conf_dir = params.ams_collector_conf_dir,
+ configurations = merged_ams_hbase_site,
+ configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
+ owner = params.ams_user,
+ group = params.user_group
+ )
+
+ if params.security_enabled:
+ TemplateConfig(os.path.join(params.hbase_conf_dir, "ams_collector_jaas.conf"),
+ owner = params.ams_user,
+ template_tag = None)
+
+ if (params.log4j_props != None):
+ File(format("{params.ams_collector_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.ams_user,
+ content=params.log4j_props
+ )
+
+ File(format("{ams_collector_conf_dir}/ams-env.sh"),
+ owner=params.ams_user,
+ content=InlineTemplate(params.ams_env_sh_template)
+ )
+
+ Directory(params.ams_collector_log_dir,
+ owner=params.ams_user,
+ group=params.user_group,
+ cd_access="a",
+ create_parents=True,
+ mode=0755,
+ )
+
+ Directory(params.ams_collector_pid_dir,
+ owner=params.ams_user,
+ group=params.user_group,
+ cd_access="a",
+ create_parents=True,
+ mode=0755,
+ )
+
+ # Hack to allow native HBase libs to be included for embedded hbase
+ File(os.path.join(params.ams_hbase_home_dir, "bin", "hadoop"),
+ owner=params.ams_user,
+ mode=0755
+ )
+
+ # On some OS this folder could be not exists, so we will create it before pushing there files
+ Directory(params.limits_conf_dir,
+ create_parents=True,
+ owner='root',
+ group='root'
+ )
+
+ # Setting up security limits
+ File(os.path.join(params.limits_conf_dir, 'ams.conf'),
+ owner='root',
+ group='root',
+ mode=0644,
+ content=Template("ams.conf.j2")
+ )
+
+ # Phoenix spool file dir if not /tmp
+ if not os.path.exists(params.phoenix_client_spool_dir):
+ Directory(params.phoenix_client_spool_dir,
+ owner=params.ams_user,
+ mode = 0755,
+ group=params.user_group,
+ cd_access="a",
+ create_parents=True
+ )
+ pass
+
+ if not params.is_local_fs_rootdir and params.is_ams_distributed:
+ # Configuration needed to support NN HA
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.ams_collector_conf_dir,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("hdfs-site.xml",
+ conf_dir=params.hbase_conf_dir,
+ configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("core-site.xml",
+ conf_dir=params.ams_collector_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ XmlConfig("core-site.xml",
+ conf_dir=params.hbase_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0644
+ )
+
+ pass
+
+ elif name == 'monitor':
+ Directory(params.ams_monitor_conf_dir,
+ owner=params.ams_user,
+ group=params.user_group,
+ create_parents=True
+ )
+
+ Directory(params.ams_monitor_log_dir,
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0755,
+ create_parents=True
+ )
+
+ Directory(params.ams_monitor_pid_dir,
+ owner=params.ams_user,
+ group=params.user_group,
+ mode=0755,
+ create_parents=True
+ )
+
+ Directory(format("{ams_monitor_dir}/psutil/build"),
+ owner=params.ams_user,
+ group=params.user_group,
+ cd_access="a",
+ create_parents=True)
+
+ Execute(format("{sudo} chown -R {ams_user}:{user_group} {ams_monitor_dir}")
+ )
+
+ TemplateConfig(
+ format("{ams_monitor_conf_dir}/metric_monitor.ini"),
+ owner=params.ams_user,
+ group=params.user_group,
+ template_tag=None
+ )
+
+ TemplateConfig(
+ format("{ams_monitor_conf_dir}/metric_groups.conf"),
+ owner=params.ams_user,
+ group=params.user_group,
+ template_tag=None
+ )
+
+ File(format("{ams_monitor_conf_dir}/ams-env.sh"),
+ owner=params.ams_user,
+ content=InlineTemplate(params.ams_env_sh_template)
+ )
+
+ # TODO
+ pass
+
+ pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/ams_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/ams_service.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/ams_service.py
new file mode 100755
index 0000000..0726802
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/ams_service.py
@@ -0,0 +1,103 @@
+# !/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from hbase_service import hbase_service
+import os
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def ams_service(name, action):
+ import params
+ if name == 'collector':
+ Service(params.ams_embedded_hbase_win_service_name, action=action)
+ Service(params.ams_collector_win_service_name, action=action)
+ elif name == 'monitor':
+ Service(params.ams_monitor_win_service_name, action=action)
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def ams_service(name, action):
+ import params
+
+ if name == 'collector':
+ cmd = format("{ams_collector_script} --config {ams_collector_conf_dir}")
+ pid_file = format("{ams_collector_pid_dir}/ambari-metrics-collector.pid")
+ #no_op_test should be much more complex to work with cumulative status of collector
+ #removing as startup script handle it also
+ #no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
+
+ if params.is_hbase_distributed:
+ hbase_service('zookeeper', action=action)
+ hbase_service('master', action=action)
+ hbase_service('regionserver', action=action)
+ cmd = format("{cmd} --distributed")
+
+ if action == 'start':
+ Execute(format("{sudo} rm -rf {hbase_tmp_dir}/*.tmp")
+ )
+
+ if not params.is_hbase_distributed and os.path.exists(format("{zookeeper_data_dir}")):
+ Directory(params.zookeeper_data_dir,
+ action='delete'
+ )
+
+
+ if params.security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {ams_collector_keytab_path} {ams_collector_jaas_princ};")
+ daemon_cmd = format("{kinit_cmd} {cmd} start")
+ else:
+ daemon_cmd = format("{cmd} start")
+
+ Execute(daemon_cmd,
+ user=params.ams_user
+ )
+
+ pass
+ elif action == 'stop':
+ daemon_cmd = format("{cmd} stop")
+ Execute(daemon_cmd,
+ user=params.ams_user
+ )
+
+ pass
+ pass
+ elif name == 'monitor':
+ cmd = format("{ams_monitor_script} --config {ams_monitor_conf_dir}")
+ pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid")
+ no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
+
+ if action == 'start':
+ daemon_cmd = format("{cmd} start")
+ Execute(daemon_cmd,
+ user=params.ams_user
+ )
+
+ pass
+ elif action == 'stop':
+
+ daemon_cmd = format("{cmd} stop")
+ Execute(daemon_cmd,
+ user=params.ams_user
+ )
+
+ pass
+ pass
+ pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/functions.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/functions.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/functions.py
new file mode 100755
index 0000000..3252dc1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/functions.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import math
+import datetime
+
+from resource_management.core.shell import checked_call
+
+def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
+ """
+ @param heapsize: str (e.g 1000m)
+ @param xmn_percent: float (e.g 0.2)
+ @param xmn_max: integer (e.g 512)
+ """
+ heapsize = int(re.search('\d+', str(heapsize_str)).group(0))
+ heapsize_unit = re.search('\D+', str(heapsize_str)).group(0)
+
+ xmn_val = int(math.floor(heapsize*xmn_percent))
+ xmn_val -= xmn_val % 8
+
+ result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
+ return str(result_xmn_val) + heapsize_unit
+
+def trim_heap_property(property, m_suffix = "m"):
+ if property and property.endswith(m_suffix):
+ property = property[:-1]
+ return property
+
+def check_append_heap_property(property, m_suffix = "m"):
+ if property and not property.endswith(m_suffix):
+ property += m_suffix
+ return property
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase.py
new file mode 100755
index 0000000..16d741f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+from ambari_commons import OSConst
+from resource_management import *
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def hbase(name=None, action = None):
+ import params
+ Directory(params.hbase_conf_dir,
+ owner = params.hadoop_user,
+ create_parents = True
+ )
+ Directory(params.hbase_tmp_dir,
+ create_parents = True,
+ owner = params.hadoop_user
+ )
+
+ Directory (os.path.join(params.local_dir, "jars"),
+ owner = params.hadoop_user,
+ create_parents = True
+ )
+
+ XmlConfig("hbase-site.xml",
+ conf_dir = params.hbase_conf_dir,
+ configurations = params.config['configurations']['ams-hbase-site'],
+ configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
+ owner = params.hadoop_user
+ )
+
+ if 'ams-hbase-policy' in params.config['configurations']:
+ XmlConfig("hbase-policy.xml",
+ conf_dir = params.hbase_conf_dir,
+ configurations = params.config['configurations']['ams-hbase-policy'],
+ configuration_attributes=params.config['configuration_attributes']['ams-hbase-policy'],
+ owner = params.hadoop_user
+ )
+ # Manually overriding ownership of file installed by hadoop package
+ else:
+ File(os.path.join(params.hbase_conf_dir, "hbase-policy.xml"),
+ owner = params.hadoop_user
+ )
+
+ # Metrics properties
+ File(os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"),
+ owner = params.hbase_user,
+ content=Template("hadoop-metrics2-hbase.properties.j2")
+ )
+
+ hbase_TemplateConfig('regionservers', user=params.hadoop_user)
+
+ if params.security_enabled:
+ hbase_TemplateConfig(format("hbase_{name}_jaas.conf"), user=params.hadoop_user)
+
+ if name != "client":
+ Directory (params.hbase_log_dir,
+ owner = params.hadoop_user,
+ create_parents = True
+ )
+
+ if (params.hbase_log4j_props != None):
+ File(os.path.join(params.hbase_conf_dir, "log4j.properties"),
+ owner=params.hadoop_user,
+ content=params.hbase_log4j_props
+ )
+ elif (os.path.exists(os.path.join(params.hbase_conf_dir,"log4j.properties"))):
+ File(os.path.join(params.hbase_conf_dir,"log4j.properties"),
+ owner=params.hadoop_user
+ )
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def hbase(name=None # 'master' or 'regionserver' or 'client'
+ , action=None):
+ import params
+
+ Directory(params.hbase_conf_dir,
+ owner = params.hbase_user,
+ group = params.user_group,
+ create_parents = True
+ )
+
+ Execute(('chown', '-R', params.hbase_user, params.hbase_conf_dir),
+ sudo=True
+ )
+
+ Directory (params.hbase_tmp_dir,
+ owner = params.hbase_user,
+ cd_access="a",
+ create_parents = True
+ )
+
+ Execute(('chown', '-R', params.hbase_user, params.hbase_tmp_dir),
+ sudo=True
+ )
+
+ Directory (os.path.join(params.local_dir, "jars"),
+ owner = params.hbase_user,
+ group = params.user_group,
+ cd_access="a",
+ mode=0775,
+ create_parents = True
+ )
+
+ merged_ams_hbase_site = {}
+ merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
+ if params.security_enabled:
+ merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
+
+ XmlConfig("hbase-site.xml",
+ conf_dir = params.hbase_conf_dir,
+ configurations = merged_ams_hbase_site,
+ configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
+ owner = params.hbase_user,
+ group = params.user_group
+ )
+
+ # Phoenix spool file dir if not /tmp
+ if not os.path.exists(params.phoenix_server_spool_dir):
+ Directory(params.phoenix_server_spool_dir,
+ owner=params.ams_user,
+ mode = 0755,
+ group=params.user_group,
+ cd_access="a",
+ create_parents=True
+ )
+ pass
+
+ if 'ams-hbase-policy' in params.config['configurations']:
+ XmlConfig("hbase-policy.xml",
+ conf_dir = params.hbase_conf_dir,
+ configurations = params.config['configurations']['ams-hbase-policy'],
+ configuration_attributes=params.config['configuration_attributes']['ams-hbase-policy'],
+ owner = params.hbase_user,
+ group = params.user_group
+ )
+ # Manually overriding ownership of file installed by hadoop package
+ else:
+ File( format("{params.hbase_conf_dir}/hbase-policy.xml"),
+ owner = params.hbase_user,
+ group = params.user_group
+ )
+
+ File(format("{hbase_conf_dir}/hbase-env.sh"),
+ owner = params.hbase_user,
+ content=InlineTemplate(params.hbase_env_sh_template)
+ )
+
+ # Metrics properties
+ File(os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"),
+ owner = params.hbase_user,
+ group = params.user_group,
+ content=Template("hadoop-metrics2-hbase.properties.j2")
+ )
+
+ # hbase_TemplateConfig( params.metric_prop_file_name,
+ # tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS'
+ # )
+
+ hbase_TemplateConfig('regionservers', user=params.hbase_user)
+
+ if params.security_enabled:
+ hbase_TemplateConfig( format("hbase_{name}_jaas.conf"), user=params.hbase_user)
+ hbase_TemplateConfig( format("hbase_client_jaas.conf"), user=params.hbase_user)
+ hbase_TemplateConfig( format("ams_zookeeper_jaas.conf"), user=params.hbase_user)
+
+ if name != "client":
+ Directory( params.hbase_pid_dir,
+ owner = params.hbase_user,
+ create_parents = True,
+ cd_access = "a",
+ mode = 0755,
+ )
+
+ Directory (params.hbase_log_dir,
+ owner = params.hbase_user,
+ create_parents = True,
+ cd_access = "a",
+ mode = 0755,
+ )
+
+ if name == "master":
+
+ if not params.is_local_fs_rootdir:
+ # If executing Stop All, HDFS is probably down
+ if action != 'stop':
+
+ params.HdfsResource(params.hbase_root_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.hbase_user,
+ mode=0775
+ )
+
+ params.HdfsResource(params.hbase_staging_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.hbase_user,
+ mode=0711
+ )
+
+ params.HdfsResource(None, action="execute")
+
+ if params.is_hbase_distributed:
+ #Workaround for status commands not aware of operating mode
+ File(format("{params.hbase_pid_dir}/distributed_mode"), action="create", mode=0644, owner=params.hbase_user)
+
+ pass
+
+ else:
+
+ local_root_dir = params.hbase_root_dir
+ #cut protocol name
+ if local_root_dir.startswith("file://"):
+ local_root_dir = local_root_dir[7:]
+ #otherwise assume dir name is provided as is
+
+ Directory(local_root_dir,
+ owner = params.hbase_user,
+ cd_access="a",
+ create_parents = True
+ )
+
+ Execute(('chown', '-R', params.hbase_user, local_root_dir),
+ sudo=True
+ )
+
+ File(format("{params.hbase_pid_dir}/distributed_mode"), action="delete", owner=params.hbase_user)
+
+ if params.hbase_log4j_props is not None:
+ File(format("{params.hbase_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hbase_user,
+ content=params.hbase_log4j_props
+ )
+ elif os.path.exists(format("{params.hbase_conf_dir}/log4j.properties")):
+ File(format("{params.hbase_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hbase_user
+ )
+
+def hbase_TemplateConfig(name, tag=None, user=None):
+ import params
+
+ TemplateConfig( os.path.join(params.hbase_conf_dir, name),
+ owner = user,
+ template_tag = tag
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_master.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_master.py
new file mode 100755
index 0000000..74f7326
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_master.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+from hbase_decommission import hbase_decommission
+
+
+class HbaseMaster(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env, action = None):
+ import params
+ env.set_params(params)
+
+ hbase('master', action)
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env, action = 'start') # for security
+
+ hbase_service( 'master',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'master',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
+ check_process_status(pid_file)
+
+ def decommission(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_decommission(env)
+
+
+if __name__ == "__main__":
+ HbaseMaster().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_regionserver.py
new file mode 100755
index 0000000..a5121b8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_regionserver.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseRegionServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env, action = None):
+ import params
+ env.set_params(params)
+
+ hbase('regionserver', action)
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env, action = 'start') # for security
+
+ hbase_service( 'regionserver',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'regionserver',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
+ check_process_status(pid_file)
+
+ def decommission(self, env):
+ print "Decommission not yet implemented!"
+
+
+if __name__ == "__main__":
+ HbaseRegionServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_service.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_service.py
new file mode 100755
index 0000000..3b542cb
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/hbase_service.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+def hbase_service(
+ name,
+ action = 'start'): # 'start' or 'stop' or 'status'
+
+ import params
+
+ role = name
+ cmd = format("{daemon_script} --config {hbase_conf_dir}")
+ pid_file = format("{hbase_pid_dir}/hbase-{hbase_user}-{role}.pid")
+ no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
+
+ if action == 'start':
+ daemon_cmd = format("{cmd} start {role}")
+
+ Execute ( daemon_cmd,
+ not_if = no_op_test,
+ user = params.hbase_user
+ )
+ elif action == 'stop':
+ daemon_cmd = format("{cmd} stop {role}")
+
+ Execute ( daemon_cmd,
+ user = params.hbase_user,
+ # BUGFIX: hbase regionserver sometimes hangs when nn is in safemode
+ timeout = 30,
+ on_timeout = format("{no_op_test} && kill -9 `cat {pid_file}`")
+ )
+
+ File(pid_file,
+ action = "delete",
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/metrics_collector.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/metrics_collector.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/metrics_collector.py
new file mode 100755
index 0000000..cf498ec
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/metrics_collector.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.libraries.functions.security_commons import build_expectations, \
+ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
+ FILE_TYPE_XML
+from ams import ams
+from ams_service import ams_service
+from hbase import hbase
+from status import check_service_status
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyImpl
+
+class AmsCollector(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env, action = None):
+ import params
+ env.set_params(params)
+ hbase('master', action)
+ hbase('regionserver', action)
+ ams(name='collector')
+
+ def start(self, env):
+ self.configure(env, action = 'start') # for security
+ # stop hanging components before start
+ ams_service('collector', action = 'stop')
+ ams_service('collector', action = 'start')
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+ # Sometimes, stop() may be called before start(), in case restart() is initiated right after installation
+ self.configure(env, action = 'stop') # for security
+ ams_service('collector', action = 'stop')
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_service_status(name='collector')
+
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class AmsCollectorDefault(AmsCollector):
+ def security_status(self, env):
+ import status_params
+
+ env.set_params(status_params)
+ props_value_check = {"hbase.security.authentication": "kerberos",
+ "hbase.security.authorization": "true"}
+
+ props_empty_check = ["hbase.zookeeper.property.authProvider.1",
+ "hbase.master.keytab.file",
+ "hbase.master.kerberos.principal",
+ "hbase.regionserver.keytab.file",
+ "hbase.regionserver.kerberos.principal"
+ ]
+ props_read_check = ['hbase.master.keytab.file', 'hbase.regionserver.keytab.file']
+ ams_hbase_site_expectations = build_expectations('hbase-site', props_value_check,
+ props_empty_check,
+ props_read_check)
+
+ expectations = {}
+ expectations.update(ams_hbase_site_expectations)
+
+ security_params = get_params_from_filesystem(status_params.ams_hbase_conf_dir,
+ {'hbase-site.xml': FILE_TYPE_XML})
+
+ is_hbase_distributed = security_params['hbase-site']['hbase.cluster.distributed']
+ # for embedded mode, when HBase is backed by file, security state is SECURED_KERBEROS by definition when cluster is secured
+ if status_params.security_enabled and not is_hbase_distributed:
+ self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+ return
+
+ result_issues = validate_security_config_properties(security_params, expectations)
+
+ if not result_issues: # If all validations passed successfully
+ try:
+ # Double check the dict before calling execute
+ if ('hbase-site' not in security_params or
+ 'hbase.master.keytab.file' not in security_params['hbase-site'] or
+ 'hbase.master.kerberos.principal' not in security_params['hbase-site']):
+ self.put_structured_out({"securityState": "UNSECURED"})
+ self.put_structured_out(
+ {"securityIssuesFound": "Keytab file or principal are not set property."})
+ return
+
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.hbase_user,
+ security_params['hbase-site']['hbase.master.keytab.file'],
+ security_params['hbase-site']['hbase.master.kerberos.principal'],
+ status_params.hostname,
+ status_params.tmp_dir)
+ self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+ except Exception as e:
+ self.put_structured_out({"securityState": "ERROR"})
+ self.put_structured_out({"securityStateErrorInfo": str(e)})
+ else:
+ issues = []
+ for cf in result_issues:
+ issues.append("Configuration file %s did not pass the validation. Reason: %s" % (
+ cf, result_issues[cf]))
+ self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+ self.put_structured_out({"securityState": "UNSECURED"})
+
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class AmsCollectorWindows(AmsCollector):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env) # for security
+
+if __name__ == "__main__":
+ AmsCollector().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/metrics_monitor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/metrics_monitor.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/metrics_monitor.py
new file mode 100755
index 0000000..38da85f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/metrics_monitor.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from ams import ams
+from ams_service import ams_service
+from status import check_service_status
+
+class AmsMonitor(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env) # for security
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ ams(name='monitor')
+
+ def start(self, env):
+ self.configure(env) # for security
+
+ ams_service( 'monitor',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ ams_service( 'monitor',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_service_status(name='monitor')
+
+
+if __name__ == "__main__":
+ AmsMonitor().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params.py
new file mode 100755
index 0000000..ba07c74
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params.py
@@ -0,0 +1,254 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from functions import calc_xmn_from_xms
+from functions import check_append_heap_property
+from functions import trim_heap_property
+
+from resource_management import *
+import status_params
+from ambari_commons import OSCheck
+
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+# server configurations
+config = Script.get_config()
+exec_tmp_dir = Script.get_tmp_dir()
+
+def get_combined_memory_mb(value1, value2):
+ try:
+ part1 = int(value1.strip()[:-1]) if value1.lower().strip()[-1:] == 'm' else int(value1)
+ part2 = int(value2.strip()[:-1]) if value2.lower().strip()[-1:] == 'm' else int(value2)
+ return str(part1 + part2) + 'm'
+ except:
+ return None
+pass
+
+#AMBARI_METRICS data
+ams_pid_dir = status_params.ams_collector_pid_dir
+
+ams_collector_script = "/usr/sbin/ambari-metrics-collector"
+ams_collector_pid_dir = status_params.ams_collector_pid_dir
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
+ metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
+else:
+ metric_collector_host = ams_collector_hosts[0]
+if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+else:
+ metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
+ if metric_collector_web_address.find(':') != -1:
+ metric_collector_port = metric_collector_web_address.split(':')[1]
+ else:
+ metric_collector_port = '6188'
+
+ams_collector_log_dir = config['configurations']['ams-env']['metrics_collector_log_dir']
+ams_monitor_log_dir = config['configurations']['ams-env']['metrics_monitor_log_dir']
+
+ams_monitor_dir = "/usr/lib/python2.6/site-packages/resource_monitoring"
+ams_monitor_pid_dir = status_params.ams_monitor_pid_dir
+ams_monitor_script = "/usr/sbin/ambari-metrics-monitor"
+
+ams_hbase_home_dir = "/usr/lib/ams-hbase/"
+
+ams_hbase_normalizer_enabled = default("/configurations/ams-hbase-site/hbase.normalizer.enabled", None)
+ams_hbase_fifo_compaction_enabled = default("/configurations/ams-site/timeline.metrics.hbase.fifo.compaction.enabled", None)
+
+#hadoop params
+
+hbase_excluded_hosts = config['commandParams']['excluded_hosts']
+hbase_drain_only = config['commandParams']['mark_draining_only']
+hbase_included_hosts = config['commandParams']['included_hosts']
+
+hbase_user = status_params.hbase_user
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+hbase_root_dir = config['configurations']['ams-hbase-site']['hbase.rootdir']
+hbase_pid_dir = status_params.hbase_pid_dir
+
+is_hbase_distributed = config['configurations']['ams-hbase-site']['hbase.cluster.distributed']
+is_local_fs_rootdir = hbase_root_dir.startswith('file://')
+is_ams_distributed = config['configurations']['ams-site']['timeline.metrics.service.operation.mode'] == 'distributed'
+
+# security is disabled for embedded mode, when HBase is backed by file
+security_enabled = False if not is_hbase_distributed else config['configurations']['cluster-env']['security_enabled']
+
+# this is "hadoop-metrics.properties" for 1.x stacks
+metric_prop_file_name = "hadoop-metrics2-hbase.properties"
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+java_version = int(config['hostLevelParams']['java_version'])
+
+metrics_collector_heapsize = default('/configurations/ams-env/metrics_collector_heapsize', "512")
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
+metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 60)
+
+hbase_log_dir = config['configurations']['ams-hbase-env']['hbase_log_dir']
+hbase_classpath_additional = default("/configurations/ams-hbase-env/hbase_classpath_additional", None)
+master_heapsize = config['configurations']['ams-hbase-env']['hbase_master_heapsize']
+regionserver_heapsize = config['configurations']['ams-hbase-env']['hbase_regionserver_heapsize']
+
+# Check if hbase java options already have appended "m". If Yes, remove the trailing m.
+metrics_collector_heapsize = check_append_heap_property(str(metrics_collector_heapsize), "m")
+master_heapsize = check_append_heap_property(str(master_heapsize), "m")
+regionserver_heapsize = check_append_heap_property(str(regionserver_heapsize), "m")
+
+regionserver_xmn_max = default('/configurations/ams-hbase-env/hbase_regionserver_xmn_max', None)
+if regionserver_xmn_max:
+ regionserver_xmn_max = int(trim_heap_property(str(regionserver_xmn_max), "m"))
+ regionserver_xmn_percent = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_ratio']
+ regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
+else:
+ regionserver_xmn_size = config['configurations']['ams-hbase-env']['regionserver_xmn_size']
+pass
+
+hbase_master_xmn_size = config['configurations']['ams-hbase-env']['hbase_master_xmn_size']
+hbase_master_maxperm_size = config['configurations']['ams-hbase-env']['hbase_master_maxperm_size']
+
+# Check if hbase java options already have appended "m". If Yes, remove the trailing m.
+hbase_master_maxperm_size = check_append_heap_property(str(hbase_master_maxperm_size), "m")
+hbase_master_xmn_size = check_append_heap_property(str(hbase_master_xmn_size), "m")
+regionserver_xmn_size = check_append_heap_property(str(regionserver_xmn_size), "m")
+
+# Choose heap size for embedded mode as sum of master + regionserver
+if not is_hbase_distributed:
+ hbase_heapsize = get_combined_memory_mb(master_heapsize, regionserver_heapsize)
+ if hbase_heapsize is None:
+ hbase_heapsize = master_heapsize
+else:
+ hbase_heapsize = master_heapsize
+
+max_open_files_limit = default("/configurations/ams-hbase-env/max_open_files_limit", "32768")
+
+if not is_hbase_distributed:
+ zookeeper_quorum_hosts = 'localhost'
+ zookeeper_clientPort = '61181'
+else:
+ zookeeper_quorum_hosts = default("/hostname", 'localhost')
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+
+ams_checkpoint_dir = config['configurations']['ams-site']['timeline.metrics.aggregator.checkpoint.dir']
+hbase_pid_dir = status_params.hbase_pid_dir
+_hbase_tmp_dir = config['configurations']['ams-hbase-site']['hbase.tmp.dir']
+hbase_tmp_dir = substitute_vars(_hbase_tmp_dir, config['configurations']['ams-hbase-site'])
+_zookeeper_data_dir = config['configurations']['ams-hbase-site']['hbase.zookeeper.property.dataDir']
+zookeeper_data_dir = substitute_vars(_zookeeper_data_dir, config['configurations']['ams-hbase-site'])
+# TODO UPGRADE default, update site during upgrade
+_local_dir_conf = default('/configurations/ams-hbase-site/hbase.local.dir', "${hbase.tmp.dir}/local")
+local_dir = substitute_vars(_local_dir_conf, config['configurations']['ams-hbase-site'])
+
+phoenix_max_global_mem_percent = default('/configurations/ams-site/phoenix.query.maxGlobalMemoryPercentage', '20')
+phoenix_client_spool_dir = default('/configurations/ams-site/phoenix.spool.directory', '/tmp')
+phoenix_server_spool_dir = default('/configurations/ams-hbase-site/phoenix.spool.directory', '/tmp')
+# Substitute vars if present
+phoenix_client_spool_dir = substitute_vars(phoenix_client_spool_dir, config['configurations']['ams-hbase-site'])
+phoenix_server_spool_dir = substitute_vars(phoenix_server_spool_dir, config['configurations']['ams-hbase-site'])
+
+client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf")
+master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf")
+regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf")
+
+rs_hosts = ["localhost"]
+
+smoke_test_user = config['configurations']['cluster-env']['smokeuser']
+smokeuser_permissions = "RWXCA"
+service_check_data = functions.get_unique_id_and_date()
+user_group = config['configurations']['cluster-env']["user_group"]
+hadoop_user = "hadoop"
+
+kinit_cmd = ""
+
+if security_enabled:
+ _hostname_lowercase = config['hostname'].lower()
+ client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf")
+ smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+ hbase_user_keytab = config['configurations']['ams-hbase-env']['hbase_user_keytab']
+
+ ams_collector_jaas_config_file = format("{hbase_conf_dir}/ams_collector_jaas.conf")
+ ams_collector_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.myclient.keytab']
+ ams_collector_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.myclient.principal'].replace('_HOST',_hostname_lowercase)
+
+ ams_zookeeper_jaas_config_file = format("{hbase_conf_dir}/ams_zookeeper_jaas.conf")
+ ams_zookeeper_keytab = config['configurations']['ams-hbase-security-site']['ams.zookeeper.keytab']
+ ams_zookeeper_principal_name = config['configurations']['ams-hbase-security-site']['ams.zookeeper.principal'].replace('_HOST',_hostname_lowercase)
+
+ master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf")
+ master_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.master.keytab.file']
+ master_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+
+ regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf")
+ regionserver_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.regionserver.keytab.file']
+ regionserver_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+
+ zk_servicename = ams_zookeeper_principal_name.rpartition('/')[0]
+
+#log4j.properties
+if (('ams-hbase-log4j' in config['configurations']) and ('content' in config['configurations']['ams-hbase-log4j'])):
+ hbase_log4j_props = config['configurations']['ams-hbase-log4j']['content']
+else:
+ hbase_log4j_props = None
+
+if (('ams-log4j' in config['configurations']) and ('content' in config['configurations']['ams-log4j'])):
+ log4j_props = config['configurations']['ams-log4j']['content']
+else:
+ log4j_props = None
+
+hbase_env_sh_template = config['configurations']['ams-hbase-env']['content']
+ams_env_sh_template = config['configurations']['ams-env']['content']
+
+hbase_staging_dir = default("/configurations/ams-hbase-site/hbase.bulkload.staging.dir", "/amshbase/staging")
+
+#for create_hdfs_directory
+hostname = config["hostname"]
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+
+
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params_linux.py
new file mode 100755
index 0000000..838e987
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params_linux.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.libraries.functions import conf_select
+from ambari_commons import OSCheck
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+
+config = Script.get_config()
+
+ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf"
+ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf/"
+ams_user = config['configurations']['ams-env']['ambari_metrics_user']
+#RPM versioning support
+rpm_version = default("/configurations/hadoop-env/rpm_version", None)
+
+#hadoop params
+if rpm_version is not None:
+ #RPM versioning support
+ rpm_version = default("/configurations/hadoop-env/rpm_version", None)
+
+hadoop_native_lib = format("/usr/lib/ams-hbase/lib/hadoop-native")
+hadoop_bin_dir = "/usr/bin"
+daemon_script = "/usr/lib/ams-hbase/bin/hbase-daemon.sh"
+region_mover = "/usr/lib/ams-hbase/bin/region_mover.rb"
+region_drainer = "/usr/lib/ams-hbase/bin/draining_servers.rb"
+hbase_cmd = "/usr/lib/ams-hbase/bin/hbase"
+
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hbase_conf_dir = "/etc/ams-hbase/conf"
+
+limits_conf_dir = "/etc/security/limits.d"
+sudo = AMBARI_SUDO_BINARY
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params_windows.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params_windows.py
new file mode 100755
index 0000000..acb5bba
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/params_windows.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management.libraries.script.script import Script
+
+
+config = Script.get_config()
+
+hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
+ams_user = hadoop_user
+
+try:
+ ams_collector_conf_dir = os.environ["COLLECTOR_CONF_DIR"]
+ ams_collector_home_dir = os.environ["COLLECTOR_HOME"]
+ hbase_cmd = os.path.join(os.environ["COLLECTOR_HOME"], "hbase", "bin", "hbase.cmd")
+ hbase_conf_dir = os.path.join(os.environ["COLLECTOR_HOME"], "hbase", "conf")
+except:
+ ams_collector_conf_dir = None
+ ams_collector_home_dir = None
+ hbase_cmd = None
+ hbase_conf_dir = None
+
+try:
+ ams_monitor_conf_dir = os.environ["MONITOR_CONF_DIR"]
+ ams_monitor_home_dir = os.environ["MONITOR_HOME"]
+except:
+ ams_monitor_conf_dir = None
+ ams_monitor_home_dir = None
+
+hadoop_native_lib = os.path.join(os.environ["HADOOP_HOME"], "bin")
+hadoop_bin_dir = os.path.join(os.environ["HADOOP_HOME"], "bin")
+hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf")
+
+from service_mapping import *
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/service_check.py
new file mode 100755
index 0000000..668db55
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/service_check.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.core.base import Fail
+from resource_management import Script
+from resource_management import Template
+
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+import httplib
+import urllib
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+import os
+import random
+import time
+import socket
+
+
+class AMSServiceCheck(Script):
+ AMS_METRICS_POST_URL = "/ws/v1/timeline/metrics/"
+ AMS_METRICS_GET_URL = "/ws/v1/timeline/metrics?%s"
+ AMS_CONNECT_TRIES = 30
+ AMS_CONNECT_TIMEOUT = 15
+
+ @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+ def service_check(self, env):
+ from resource_management.libraries.functions.windows_service_utils import check_windows_service_exists
+ import params
+
+ env.set_params(params)
+
+ #Just check that the services were correctly installed
+ #Check the monitor on all hosts
+ Logger.info("Metrics Monitor service check was started.")
+ if not check_windows_service_exists(params.ams_monitor_win_service_name):
+ raise Fail("Metrics Monitor service was not properly installed. Check the logs and retry the installation.")
+ #Check the collector only where installed
+ if params.ams_collector_home_dir and os.path.isdir(params.ams_collector_home_dir):
+ Logger.info("Metrics Collector service check was started.")
+ if not check_windows_service_exists(params.ams_collector_win_service_name):
+ raise Fail("Metrics Collector service was not properly installed. Check the logs and retry the installation.")
+
+ @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+ def service_check(self, env):
+ import params
+
+ Logger.info("Ambari Metrics service check was started.")
+ env.set_params(params)
+
+ random_value1 = random.random()
+ headers = {"Content-type": "application/json"}
+
+ for i in xrange(0, self.AMS_CONNECT_TRIES):
+ try:
+ current_time = int(time.time()) * 1000
+ metric_json = Template('smoketest_metrics.json.j2', hostname=params.hostname, random1=random_value1,
+ current_time=current_time).get_content()
+ Logger.info("Generated metrics:\n%s" % metric_json)
+
+ Logger.info("Connecting (POST) to %s:%s%s" % (params.metric_collector_host,
+ params.metric_collector_port,
+ self.AMS_METRICS_POST_URL))
+ conn = httplib.HTTPConnection(params.metric_collector_host,
+ int(params.metric_collector_port))
+ conn.request("POST", self.AMS_METRICS_POST_URL, metric_json, headers)
+
+ response = conn.getresponse()
+ Logger.info("Http response: %s %s" % (response.status, response.reason))
+ except (httplib.HTTPException, socket.error) as ex:
+ if i < self.AMS_CONNECT_TRIES - 1: #range/xrange returns items from start to end-1
+ time.sleep(self.AMS_CONNECT_TIMEOUT)
+ Logger.info("Connection failed. Next retry in %s seconds."
+ % (self.AMS_CONNECT_TIMEOUT))
+ continue
+ else:
+ raise Fail("Metrics were not saved. Service check has failed. "
+ "\nConnection failed.")
+
+ data = response.read()
+ Logger.info("Http data: %s" % data)
+ conn.close()
+
+ if response.status == 200:
+ Logger.info("Metrics were saved.")
+ break
+ else:
+ Logger.info("Metrics were not saved. Service check has failed.")
+ if i < self.AMS_CONNECT_TRIES - 1: #range/xrange returns items from start to end-1
+ time.sleep(self.AMS_CONNECT_TIMEOUT)
+ Logger.info("Next retry in %s seconds."
+ % (self.AMS_CONNECT_TIMEOUT))
+ else:
+ raise Fail("Metrics were not saved. Service check has failed. POST request status: %s %s \n%s" %
+ (response.status, response.reason, data))
+
+ get_metrics_parameters = {
+ "metricNames": "AMBARI_METRICS.SmokeTest.FakeMetric",
+ "appId": "amssmoketestfake",
+ "hostname": params.hostname,
+ "startTime": current_time - 60000,
+ "endTime": current_time + 61000,
+ "precision": "seconds",
+ "grouped": "false",
+ }
+ encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters)
+
+ Logger.info("Connecting (GET) to %s:%s%s" % (params.metric_collector_host,
+ params.metric_collector_port,
+ self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters))
+
+ conn = httplib.HTTPConnection(params.metric_collector_host,
+ int(params.metric_collector_port))
+ conn.request("GET", self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
+ response = conn.getresponse()
+ Logger.info("Http response: %s %s" % (response.status, response.reason))
+
+ data = response.read()
+ Logger.info("Http data: %s" % data)
+ conn.close()
+
+ if response.status == 200:
+ Logger.info("Metrics were retrieved.")
+ else:
+ Logger.info("Metrics were not retrieved. Service check has failed.")
+ raise Fail("Metrics were not retrieved. Service check has failed. GET request status: %s %s \n%s" %
+ (response.status, response.reason, data))
+ data_json = json.loads(data)
+
+ def floats_eq(f1, f2, delta):
+ return abs(f1-f2) < delta
+
+ for metrics_data in data_json["metrics"]:
+ if (str(current_time) in metrics_data["metrics"] and str(current_time + 1000) in metrics_data["metrics"]
+ and floats_eq(metrics_data["metrics"][str(current_time)], random_value1, 0.0000001)
+ and floats_eq(metrics_data["metrics"][str(current_time + 1000)], current_time, 1)):
+ Logger.info("Values %s and %s were found in the response." % (random_value1, current_time))
+ break
+ pass
+ else:
+ Logger.info("Values %s and %s were not found in the response." % (random_value1, current_time))
+ raise Fail("Values %s and %s were not found in the response." % (random_value1, current_time))
+
+ Logger.info("Ambari Metrics service check is finished.")
+
+if __name__ == "__main__":
+ AMSServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/service_mapping.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/service_mapping.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/service_mapping.py
new file mode 100755
index 0000000..2eeb427
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/service_mapping.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+ams_collector_win_service_name = "AmbariMetricsCollector"
+ams_monitor_win_service_name = "AmbariMetricsHostMonitoring"
+ams_embedded_hbase_win_service_name = "ams_hbase_master"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/split_points.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/split_points.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/split_points.py
new file mode 100755
index 0000000..fa4deaf
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/split_points.py
@@ -0,0 +1,236 @@
+# !/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import sys
+import re
+import math
+import collections
+import ast
+
+metric_filename_ext = '.txt'
+# 5 regions for higher order aggregate tables
+other_region_static_count = 6
+# Max equidistant points to return per service
+max_equidistant_points = 50
+
+b_bytes = 1
+k_bytes = 1 << 10 # 1024
+m_bytes = 1 << 20 # 1024^2
+g_bytes = 1 << 30 # 1024^3
+t_bytes = 1 << 40 # 1024^4
+p_bytes = 1 << 50 # 1024^5
+
+def to_number(s):
+ try:
+ return int(re.sub("\D", "", s))
+ except ValueError:
+ return None
+
+def format_Xmx_size_to_bytes(value, default='b'):
+ strvalue = str(value).lower()
+ if len(strvalue) == 0:
+ return 0
+ modifier = strvalue[-1]
+
+ if modifier == ' ' or modifier in "0123456789":
+ modifier = default
+
+ m = {
+ modifier == 'b': b_bytes,
+ modifier == 'k': k_bytes,
+ modifier == 'm': m_bytes,
+ modifier == 'g': g_bytes,
+ modifier == 't': t_bytes,
+ modifier == 'p': p_bytes
+ } [1]
+ return to_number(strvalue) * m
+
+# Class that takes AMS HBase configs as input and determines the Region
+# pre-splits based on selected services also passed as a parameter to the class.
+class FindSplitPointsForAMSRegions():
+
+ def __init__(self, ams_hbase_site, ams_hbase_env, serviceMetricsDir,
+ operation_mode = 'embedded', services = None):
+ self.ams_hbase_site = ams_hbase_site
+ self.ams_hbase_env = ams_hbase_env
+ self.serviceMetricsDir = serviceMetricsDir
+ self.services = services
+ self.mode = operation_mode
+ # Add host metrics if not present as input
+ if self.services and 'HOST' not in self.services:
+ self.services.append('HOST')
+
+ # Initialize before user
+ self.initialize()
+
+ def initialize(self):
+ # calculate regions based on available memory
+ self.initialize_region_counts()
+ self.initialize_ordered_set_of_metrics()
+
+ def initialize_region_counts(self):
+ try:
+ xmx_master_bytes = format_Xmx_size_to_bytes(self.ams_hbase_env['hbase_master_heapsize'], 'm')
+ xmx_region_bytes = 0
+ if "hbase_regionserver_heapsize" in self.ams_hbase_env:
+ xmx_region_bytes = format_Xmx_size_to_bytes(self.ams_hbase_env['hbase_regionserver_heapsize'], 'm')
+ xmx_bytes = xmx_master_bytes + xmx_region_bytes
+ if self.mode == 'distributed':
+ xmx_bytes = xmx_region_bytes
+
+ memstore_max_mem = float(self.ams_hbase_site['hbase.regionserver.global.memstore.lowerLimit']) * xmx_bytes
+ memstore_flush_size = format_Xmx_size_to_bytes(self.ams_hbase_site['hbase.hregion.memstore.flush.size'])
+
+ max_inmemory_regions = (memstore_max_mem / memstore_flush_size) - other_region_static_count
+ print 'max_inmemory_regions: %s' % max_inmemory_regions
+
+ if max_inmemory_regions > 2:
+ # Lets say total = 12, so we have 7 regions to allocate between
+ # METRIC_RECORD and METRIC_AGGREGATE tables, desired = (5, 2)
+ self.desired_precision_region_count = int(math.floor(0.8 * max_inmemory_regions))
+ self.desired_aggregate_region_count = int(max_inmemory_regions - self.desired_precision_region_count)
+ else:
+ self.desired_precision_region_count = 1
+ self.desired_aggregate_region_count = 1
+
+ except:
+ print('Bad config settings, could not calculate max regions available.')
+ pass
+
+ def initialize_ordered_set_of_metrics(self):
+ onlyServicefiles = [ f for f in os.listdir(self.serviceMetricsDir) if
+ os.path.isfile(os.path.join(self.serviceMetricsDir, f)) ]
+
+ metrics = set()
+
+ for file in onlyServicefiles:
+ # Process for services selected at deploy time or all services if
+ # services arg is not passed
+ if self.services is None or file.rstrip(metric_filename_ext) in self.services:
+ print 'Processing file: %s' % os.path.join(self.serviceMetricsDir, file)
+ service_metrics = set()
+ with open(os.path.join(self.serviceMetricsDir, file), 'r') as f:
+ for metric in f:
+ service_metrics.add(metric.strip())
+ pass
+ pass
+ metrics.update(self.find_equidistant_metrics(list(sorted(service_metrics))))
+ pass
+ pass
+
+ self.metrics = sorted(metrics)
+ print 'metrics length: %s' % len(self.metrics)
+
+ # Pick 50 metric points for each service that are equidistant from
+ # each other for a service
+ def find_equidistant_metrics(self, service_metrics):
+ equi_metrics = []
+ idx = len(service_metrics) / max_equidistant_points
+ if idx == 0:
+ return service_metrics
+ pass
+
+ index = idx
+ for i in range(0, max_equidistant_points - 1):
+ equi_metrics.append(service_metrics[index - 1])
+ index += idx
+ pass
+
+ return equi_metrics
+
+ def get_split_points(self):
+ split_points = collections.namedtuple('SplitPoints', [ 'precision', 'aggregate' ])
+ split_points.precision = []
+ split_points.aggregate = []
+
+ metric_list = list(self.metrics)
+ metrics_total = len(metric_list)
+
+ if self.desired_precision_region_count > 1:
+ idx = int(math.ceil(metrics_total / self.desired_precision_region_count))
+ index = idx
+ for i in range(0, self.desired_precision_region_count - 1):
+ if index < metrics_total - 1:
+ split_points.precision.append(metric_list[index])
+ index += idx
+
+ if self.desired_aggregate_region_count > 1:
+ idx = int(math.ceil(metrics_total / self.desired_aggregate_region_count))
+ index = idx
+ for i in range(0, self.desired_aggregate_region_count - 1):
+ if index < metrics_total - 1:
+ split_points.aggregate.append(metric_list[index])
+ index += idx
+
+ return split_points
+ pass
+
+def main(argv = None):
+ scriptDir = os.path.realpath(os.path.dirname(argv[0]))
+ serviceMetricsDir = os.path.join(scriptDir, os.pardir, 'files', 'service-metrics')
+
+ if os.path.exists(serviceMetricsDir):
+ onlyargs = argv[1:]
+ if len(onlyargs) < 3:
+ sys.stderr.write("Usage: dict(ams-hbase-site) dict(ams-hbase-env) list(services)\n")
+ sys.exit(2)
+ pass
+
+ ams_hbase_site = None
+ ams_hbase_env = None
+ services = None
+ try:
+ ams_hbase_site = ast.literal_eval(str(onlyargs[0]))
+ ams_hbase_env = ast.literal_eval(str(onlyargs[1]))
+ services = onlyargs[2]
+ if services:
+ services = str(services).split(',')
+ pass
+ except Exception, ex:
+ sys.stderr.write(str(ex))
+ sys.stderr.write("\nUsage: Expected items not found in input. Found "
+ " ams-hbase-site => {0}, ams-hbase-env => {1},"
+ " services => {2}".format(ams_hbase_site, ams_hbase_env, services))
+ sys.exit(2)
+
+ print '--------- AMS Regions Split point finder ---------'
+ print 'Services: %s' % services
+
+ mode = 'distributed' if 'hbase.rootdir' in ams_hbase_site and \
+ 'hdfs' in ams_hbase_site['hbase.rootdir'] else \
+ 'embedded'
+
+ split_point_finder = FindSplitPointsForAMSRegions(
+ ams_hbase_site, ams_hbase_env, serviceMetricsDir, mode, services)
+
+ result = split_point_finder.get_split_points()
+ print 'Split points for precision table : %s' % len(result.precision)
+ print 'precision: %s' % str(result.precision)
+ print 'Split points for aggregate table : %s' % len(result.aggregate)
+ print 'aggregate: %s' % str(result.aggregate)
+
+ return 0
+
+ else:
+ print 'Cannot find service metrics dir in %s' % scriptDir
+
+if __name__ == '__main__':
+ main(sys.argv)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status.py
new file mode 100755
index 0000000..59466ad
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status.py
@@ -0,0 +1,46 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+import os
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def check_service_status(name):
+ if name=='collector':
+ pid_file = format("{ams_collector_pid_dir}/ambari-metrics-collector.pid")
+ check_process_status(pid_file)
+ pid_file = format("{hbase_pid_dir}/hbase-{hbase_user}-master.pid")
+ check_process_status(pid_file)
+ if os.path.exists(format("{hbase_pid_dir}/distributed_mode")):
+ pid_file = format("{hbase_pid_dir}/hbase-{hbase_user}-regionserver.pid")
+ check_process_status(pid_file)
+
+ elif name == 'monitor':
+ pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid")
+ check_process_status(pid_file)
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def check_service_status(name):
+ import service_mapping
+ if name=='collector':
+ check_windows_service_status(service_mapping.ams_collector_win_service_name)
+ elif name == 'monitor':
+ check_windows_service_status(service_mapping.ams_monitor_win_service_name)