You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2017/08/07 22:49:45 UTC
[2/6] ambari git commit: AMBARI-21664. HDFS namenode rpc and
connection load metrics are not showing. (swagle)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/ams.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/ams.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/ams.py
deleted file mode 100755
index 7b1a824..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/ams.py
+++ /dev/null
@@ -1,388 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from ambari_commons import OSConst
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from ambari_commons.str_utils import compress_backslashes
-import glob
-import os
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def ams(name=None):
- import params
- if name == 'collector':
- if not check_windows_service_exists(params.ams_collector_win_service_name):
- Execute(format("cmd /C cd {ams_collector_home_dir} & ambari-metrics-collector.cmd setup"))
-
- Directory(params.ams_collector_conf_dir,
- owner=params.ams_user,
- create_parents = True
- )
-
- Directory(params.ams_checkpoint_dir,
- owner=params.ams_user,
- create_parents = True
- )
-
- XmlConfig("ams-site.xml",
- conf_dir=params.ams_collector_conf_dir,
- configurations=params.config['configurations']['ams-site'],
- configuration_attributes=params.config['configuration_attributes']['ams-site'],
- owner=params.ams_user,
- )
-
- merged_ams_hbase_site = {}
- merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
- if params.security_enabled:
- merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
-
- XmlConfig( "hbase-site.xml",
- conf_dir = params.ams_collector_conf_dir,
- configurations = merged_ams_hbase_site,
- configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
- owner = params.ams_user,
- )
-
- if (params.log4j_props != None):
- File(os.path.join(params.ams_collector_conf_dir, "log4j.properties"),
- owner=params.ams_user,
- content=params.log4j_props
- )
-
- File(os.path.join(params.ams_collector_conf_dir, "ams-env.cmd"),
- owner=params.ams_user,
- content=InlineTemplate(params.ams_env_sh_template)
- )
-
- ServiceConfig(params.ams_collector_win_service_name,
- action="change_user",
- username = params.ams_user,
- password = Script.get_password(params.ams_user))
-
- if not params.is_local_fs_rootdir:
- # Configuration needed to support NN HA
- XmlConfig("hdfs-site.xml",
- conf_dir=params.ams_collector_conf_dir,
- configurations=params.config['configurations']['hdfs-site'],
- configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
- owner=params.ams_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("hdfs-site.xml",
- conf_dir=params.hbase_conf_dir,
- configurations=params.config['configurations']['hdfs-site'],
- configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
- owner=params.ams_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("core-site.xml",
- conf_dir=params.ams_collector_conf_dir,
- configurations=params.config['configurations']['core-site'],
- configuration_attributes=params.config['configuration_attributes']['core-site'],
- owner=params.ams_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("core-site.xml",
- conf_dir=params.hbase_conf_dir,
- configurations=params.config['configurations']['core-site'],
- configuration_attributes=params.config['configuration_attributes']['core-site'],
- owner=params.ams_user,
- group=params.user_group,
- mode=0644
- )
-
- else:
- ServiceConfig(params.ams_embedded_hbase_win_service_name,
- action="change_user",
- username = params.ams_user,
- password = Script.get_password(params.ams_user))
- # creating symbolic links on ams jars to make them available to services
- links_pairs = [
- ("%COLLECTOR_HOME%\\hbase\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
- "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"),
- ]
- for link_pair in links_pairs:
- link, target = link_pair
- real_link = os.path.expandvars(link)
- target = compress_backslashes(glob.glob(os.path.expandvars(target))[0])
- if not os.path.exists(real_link):
- #TODO check the symlink destination too. Broken in Python 2.x on Windows.
- Execute('cmd /c mklink "{0}" "{1}"'.format(real_link, target))
- pass
-
- elif name == 'monitor':
- if not check_windows_service_exists(params.ams_monitor_win_service_name):
- Execute(format("cmd /C cd {ams_monitor_home_dir} & ambari-metrics-monitor.cmd setup"))
-
- # creating symbolic links on ams jars to make them available to services
- links_pairs = [
- ("%HADOOP_HOME%\\share\\hadoop\\common\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
- "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"),
- ("%HBASE_HOME%\\lib\\ambari-metrics-hadoop-sink-with-common.jar",
- "%SINK_HOME%\\hadoop-sink\\ambari-metrics-hadoop-sink-with-common-*.jar"),
- ]
- for link_pair in links_pairs:
- link, target = link_pair
- real_link = os.path.expandvars(link)
- target = compress_backslashes(glob.glob(os.path.expandvars(target))[0])
- if not os.path.exists(real_link):
- #TODO check the symlink destination too. Broken in Python 2.x on Windows.
- Execute('cmd /c mklink "{0}" "{1}"'.format(real_link, target))
-
- Directory(params.ams_monitor_conf_dir,
- owner=params.ams_user,
- create_parents = True
- )
-
- TemplateConfig(
- os.path.join(params.ams_monitor_conf_dir, "metric_monitor.ini"),
- owner=params.ams_user,
- template_tag=None
- )
-
- TemplateConfig(
- os.path.join(params.ams_monitor_conf_dir, "metric_groups.conf"),
- owner=params.ams_user,
- template_tag=None
- )
-
- ServiceConfig(params.ams_monitor_win_service_name,
- action="change_user",
- username = params.ams_user,
- password = Script.get_password(params.ams_user))
-
-
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def ams(name=None):
- import params
-
- if name == 'collector':
- Directory(params.ams_collector_conf_dir,
- owner=params.ams_user,
- group=params.user_group,
- create_parents = True
- )
-
- Execute(('chown', '-R', params.ams_user, params.ams_collector_conf_dir),
- sudo=True
- )
-
- Directory(params.ams_checkpoint_dir,
- owner=params.ams_user,
- group=params.user_group,
- cd_access="a",
- create_parents = True
- )
-
- Execute(('chown', '-R', params.ams_user, params.ams_checkpoint_dir),
- sudo=True
- )
-
- XmlConfig("ams-site.xml",
- conf_dir=params.ams_collector_conf_dir,
- configurations=params.config['configurations']['ams-site'],
- configuration_attributes=params.config['configuration_attributes']['ams-site'],
- owner=params.ams_user,
- group=params.user_group
- )
-
- merged_ams_hbase_site = {}
- merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
- if params.security_enabled:
- merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
-
- # Add phoenix client side overrides
- merged_ams_hbase_site['phoenix.query.maxGlobalMemoryPercentage'] = str(params.phoenix_max_global_mem_percent)
- merged_ams_hbase_site['phoenix.spool.directory'] = params.phoenix_client_spool_dir
-
- XmlConfig( "hbase-site.xml",
- conf_dir = params.ams_collector_conf_dir,
- configurations = merged_ams_hbase_site,
- configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
- owner = params.ams_user,
- group = params.user_group
- )
-
- if params.security_enabled:
- TemplateConfig(os.path.join(params.hbase_conf_dir, "ams_collector_jaas.conf"),
- owner = params.ams_user,
- template_tag = None)
-
- if (params.log4j_props != None):
- File(format("{params.ams_collector_conf_dir}/log4j.properties"),
- mode=0644,
- group=params.user_group,
- owner=params.ams_user,
- content=params.log4j_props
- )
-
- File(format("{ams_collector_conf_dir}/ams-env.sh"),
- owner=params.ams_user,
- content=InlineTemplate(params.ams_env_sh_template)
- )
-
- Directory(params.ams_collector_log_dir,
- owner=params.ams_user,
- group=params.user_group,
- cd_access="a",
- create_parents = True,
- mode=0755,
- )
-
- Directory(params.ams_collector_pid_dir,
- owner=params.ams_user,
- group=params.user_group,
- cd_access="a",
- create_parents = True,
- mode=0755,
- )
-
- # Hack to allow native HBase libs to be included for embedded hbase
- File(os.path.join(params.ams_hbase_home_dir, "bin", "hadoop"),
- owner=params.ams_user,
- mode=0755
- )
-
- # On some OS this folder could be not exists, so we will create it before pushing there files
- Directory(params.limits_conf_dir,
- create_parents = True,
- owner='root',
- group='root'
- )
-
- # Setting up security limits
- File(os.path.join(params.limits_conf_dir, 'ams.conf'),
- owner='root',
- group='root',
- mode=0644,
- content=Template("ams.conf.j2")
- )
-
- # Phoenix spool file dir if not /tmp
- if not os.path.exists(params.phoenix_client_spool_dir):
- Directory(params.phoenix_client_spool_dir,
- owner=params.ams_user,
- mode = 0755,
- group=params.user_group,
- cd_access="a",
- create_parents = True
- )
- pass
-
- if not params.is_local_fs_rootdir and params.is_ams_distributed:
- # Configuration needed to support NN HA
- XmlConfig("hdfs-site.xml",
- conf_dir=params.ams_collector_conf_dir,
- configurations=params.config['configurations']['hdfs-site'],
- configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
- owner=params.ams_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("hdfs-site.xml",
- conf_dir=params.hbase_conf_dir,
- configurations=params.config['configurations']['hdfs-site'],
- configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
- owner=params.ams_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("core-site.xml",
- conf_dir=params.ams_collector_conf_dir,
- configurations=params.config['configurations']['core-site'],
- configuration_attributes=params.config['configuration_attributes']['core-site'],
- owner=params.ams_user,
- group=params.user_group,
- mode=0644
- )
-
- XmlConfig("core-site.xml",
- conf_dir=params.hbase_conf_dir,
- configurations=params.config['configurations']['core-site'],
- configuration_attributes=params.config['configuration_attributes']['core-site'],
- owner=params.ams_user,
- group=params.user_group,
- mode=0644
- )
-
- pass
-
- elif name == 'monitor':
- Directory(params.ams_monitor_conf_dir,
- owner=params.ams_user,
- group=params.user_group,
- create_parents = True
- )
-
- Directory(params.ams_monitor_log_dir,
- owner=params.ams_user,
- group=params.user_group,
- mode=0755,
- create_parents = True
- )
-
- Directory(params.ams_monitor_pid_dir,
- owner=params.ams_user,
- group=params.user_group,
- mode=0755,
- create_parents = True
- )
-
- Directory(format("{ams_monitor_dir}/psutil/build"),
- owner=params.ams_user,
- group=params.user_group,
- cd_access="a",
- create_parents = True)
-
- Execute(format("{sudo} chown -R {ams_user}:{user_group} {ams_monitor_dir}")
- )
-
- TemplateConfig(
- format("{ams_monitor_conf_dir}/metric_monitor.ini"),
- owner=params.ams_user,
- group=params.user_group,
- template_tag=None
- )
-
- TemplateConfig(
- format("{ams_monitor_conf_dir}/metric_groups.conf"),
- owner=params.ams_user,
- group=params.user_group,
- template_tag=None
- )
-
- File(format("{ams_monitor_conf_dir}/ams-env.sh"),
- owner=params.ams_user,
- content=InlineTemplate(params.ams_env_sh_template)
- )
-
- # TODO
- pass
-
- pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/ams_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/ams_service.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/ams_service.py
deleted file mode 100755
index 0726802..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/ams_service.py
+++ /dev/null
@@ -1,103 +0,0 @@
-# !/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from ambari_commons import OSConst
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from hbase_service import hbase_service
-import os
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def ams_service(name, action):
- import params
- if name == 'collector':
- Service(params.ams_embedded_hbase_win_service_name, action=action)
- Service(params.ams_collector_win_service_name, action=action)
- elif name == 'monitor':
- Service(params.ams_monitor_win_service_name, action=action)
-
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def ams_service(name, action):
- import params
-
- if name == 'collector':
- cmd = format("{ams_collector_script} --config {ams_collector_conf_dir}")
- pid_file = format("{ams_collector_pid_dir}/ambari-metrics-collector.pid")
- #no_op_test should be much more complex to work with cumulative status of collector
- #removing as startup script handle it also
- #no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
-
- if params.is_hbase_distributed:
- hbase_service('zookeeper', action=action)
- hbase_service('master', action=action)
- hbase_service('regionserver', action=action)
- cmd = format("{cmd} --distributed")
-
- if action == 'start':
- Execute(format("{sudo} rm -rf {hbase_tmp_dir}/*.tmp")
- )
-
- if not params.is_hbase_distributed and os.path.exists(format("{zookeeper_data_dir}")):
- Directory(params.zookeeper_data_dir,
- action='delete'
- )
-
-
- if params.security_enabled:
- kinit_cmd = format("{kinit_path_local} -kt {ams_collector_keytab_path} {ams_collector_jaas_princ};")
- daemon_cmd = format("{kinit_cmd} {cmd} start")
- else:
- daemon_cmd = format("{cmd} start")
-
- Execute(daemon_cmd,
- user=params.ams_user
- )
-
- pass
- elif action == 'stop':
- daemon_cmd = format("{cmd} stop")
- Execute(daemon_cmd,
- user=params.ams_user
- )
-
- pass
- pass
- elif name == 'monitor':
- cmd = format("{ams_monitor_script} --config {ams_monitor_conf_dir}")
- pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid")
- no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
-
- if action == 'start':
- daemon_cmd = format("{cmd} start")
- Execute(daemon_cmd,
- user=params.ams_user
- )
-
- pass
- elif action == 'stop':
-
- daemon_cmd = format("{cmd} stop")
- Execute(daemon_cmd,
- user=params.ams_user
- )
-
- pass
- pass
- pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/functions.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/functions.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/functions.py
deleted file mode 100755
index 140c24c..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/functions.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import re
-import math
-import datetime
-
-from resource_management.core.shell import checked_call
-
-def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
- """
- @param heapsize: str (e.g 1000m)
- @param xmn_percent: float (e.g 0.2)
- @param xmn_max: integer (e.g 512)
- """
- heapsize = int(re.search('\d+', str(heapsize_str)).group(0))
- heapsize_unit = re.search('\D+', str(heapsize_str)).group(0)
-
- xmn_val = int(math.floor(heapsize*xmn_percent))
- xmn_val -= xmn_val % 8
-
- result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
- return str(result_xmn_val) + heapsize_unit
-
-def trim_heap_property(property, m_suffix = "m"):
- if property and property.endswith(m_suffix):
- property = property[:-1]
- return property
-
-def check_append_heap_property(property, m_suffix = "m"):
- if property and not property.endswith(m_suffix):
- property += m_suffix
- return property
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase.py
deleted file mode 100755
index 16d741f..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase.py
+++ /dev/null
@@ -1,267 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-import os
-from ambari_commons import OSConst
-from resource_management import *
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def hbase(name=None, action = None):
- import params
- Directory(params.hbase_conf_dir,
- owner = params.hadoop_user,
- create_parents = True
- )
- Directory(params.hbase_tmp_dir,
- create_parents = True,
- owner = params.hadoop_user
- )
-
- Directory (os.path.join(params.local_dir, "jars"),
- owner = params.hadoop_user,
- create_parents = True
- )
-
- XmlConfig("hbase-site.xml",
- conf_dir = params.hbase_conf_dir,
- configurations = params.config['configurations']['ams-hbase-site'],
- configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
- owner = params.hadoop_user
- )
-
- if 'ams-hbase-policy' in params.config['configurations']:
- XmlConfig("hbase-policy.xml",
- conf_dir = params.hbase_conf_dir,
- configurations = params.config['configurations']['ams-hbase-policy'],
- configuration_attributes=params.config['configuration_attributes']['ams-hbase-policy'],
- owner = params.hadoop_user
- )
- # Manually overriding ownership of file installed by hadoop package
- else:
- File(os.path.join(params.hbase_conf_dir, "hbase-policy.xml"),
- owner = params.hadoop_user
- )
-
- # Metrics properties
- File(os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"),
- owner = params.hbase_user,
- content=Template("hadoop-metrics2-hbase.properties.j2")
- )
-
- hbase_TemplateConfig('regionservers', user=params.hadoop_user)
-
- if params.security_enabled:
- hbase_TemplateConfig(format("hbase_{name}_jaas.conf"), user=params.hadoop_user)
-
- if name != "client":
- Directory (params.hbase_log_dir,
- owner = params.hadoop_user,
- create_parents = True
- )
-
- if (params.hbase_log4j_props != None):
- File(os.path.join(params.hbase_conf_dir, "log4j.properties"),
- owner=params.hadoop_user,
- content=params.hbase_log4j_props
- )
- elif (os.path.exists(os.path.join(params.hbase_conf_dir,"log4j.properties"))):
- File(os.path.join(params.hbase_conf_dir,"log4j.properties"),
- owner=params.hadoop_user
- )
-
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def hbase(name=None # 'master' or 'regionserver' or 'client'
- , action=None):
- import params
-
- Directory(params.hbase_conf_dir,
- owner = params.hbase_user,
- group = params.user_group,
- create_parents = True
- )
-
- Execute(('chown', '-R', params.hbase_user, params.hbase_conf_dir),
- sudo=True
- )
-
- Directory (params.hbase_tmp_dir,
- owner = params.hbase_user,
- cd_access="a",
- create_parents = True
- )
-
- Execute(('chown', '-R', params.hbase_user, params.hbase_tmp_dir),
- sudo=True
- )
-
- Directory (os.path.join(params.local_dir, "jars"),
- owner = params.hbase_user,
- group = params.user_group,
- cd_access="a",
- mode=0775,
- create_parents = True
- )
-
- merged_ams_hbase_site = {}
- merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-site'])
- if params.security_enabled:
- merged_ams_hbase_site.update(params.config['configurations']['ams-hbase-security-site'])
-
- XmlConfig("hbase-site.xml",
- conf_dir = params.hbase_conf_dir,
- configurations = merged_ams_hbase_site,
- configuration_attributes=params.config['configuration_attributes']['ams-hbase-site'],
- owner = params.hbase_user,
- group = params.user_group
- )
-
- # Phoenix spool file dir if not /tmp
- if not os.path.exists(params.phoenix_server_spool_dir):
- Directory(params.phoenix_server_spool_dir,
- owner=params.ams_user,
- mode = 0755,
- group=params.user_group,
- cd_access="a",
- create_parents=True
- )
- pass
-
- if 'ams-hbase-policy' in params.config['configurations']:
- XmlConfig("hbase-policy.xml",
- conf_dir = params.hbase_conf_dir,
- configurations = params.config['configurations']['ams-hbase-policy'],
- configuration_attributes=params.config['configuration_attributes']['ams-hbase-policy'],
- owner = params.hbase_user,
- group = params.user_group
- )
- # Manually overriding ownership of file installed by hadoop package
- else:
- File( format("{params.hbase_conf_dir}/hbase-policy.xml"),
- owner = params.hbase_user,
- group = params.user_group
- )
-
- File(format("{hbase_conf_dir}/hbase-env.sh"),
- owner = params.hbase_user,
- content=InlineTemplate(params.hbase_env_sh_template)
- )
-
- # Metrics properties
- File(os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"),
- owner = params.hbase_user,
- group = params.user_group,
- content=Template("hadoop-metrics2-hbase.properties.j2")
- )
-
- # hbase_TemplateConfig( params.metric_prop_file_name,
- # tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS'
- # )
-
- hbase_TemplateConfig('regionservers', user=params.hbase_user)
-
- if params.security_enabled:
- hbase_TemplateConfig( format("hbase_{name}_jaas.conf"), user=params.hbase_user)
- hbase_TemplateConfig( format("hbase_client_jaas.conf"), user=params.hbase_user)
- hbase_TemplateConfig( format("ams_zookeeper_jaas.conf"), user=params.hbase_user)
-
- if name != "client":
- Directory( params.hbase_pid_dir,
- owner = params.hbase_user,
- create_parents = True,
- cd_access = "a",
- mode = 0755,
- )
-
- Directory (params.hbase_log_dir,
- owner = params.hbase_user,
- create_parents = True,
- cd_access = "a",
- mode = 0755,
- )
-
- if name == "master":
-
- if not params.is_local_fs_rootdir:
- # If executing Stop All, HDFS is probably down
- if action != 'stop':
-
- params.HdfsResource(params.hbase_root_dir,
- type="directory",
- action="create_on_execute",
- owner=params.hbase_user,
- mode=0775
- )
-
- params.HdfsResource(params.hbase_staging_dir,
- type="directory",
- action="create_on_execute",
- owner=params.hbase_user,
- mode=0711
- )
-
- params.HdfsResource(None, action="execute")
-
- if params.is_hbase_distributed:
- #Workaround for status commands not aware of operating mode
- File(format("{params.hbase_pid_dir}/distributed_mode"), action="create", mode=0644, owner=params.hbase_user)
-
- pass
-
- else:
-
- local_root_dir = params.hbase_root_dir
- #cut protocol name
- if local_root_dir.startswith("file://"):
- local_root_dir = local_root_dir[7:]
- #otherwise assume dir name is provided as is
-
- Directory(local_root_dir,
- owner = params.hbase_user,
- cd_access="a",
- create_parents = True
- )
-
- Execute(('chown', '-R', params.hbase_user, local_root_dir),
- sudo=True
- )
-
- File(format("{params.hbase_pid_dir}/distributed_mode"), action="delete", owner=params.hbase_user)
-
- if params.hbase_log4j_props is not None:
- File(format("{params.hbase_conf_dir}/log4j.properties"),
- mode=0644,
- group=params.user_group,
- owner=params.hbase_user,
- content=params.hbase_log4j_props
- )
- elif os.path.exists(format("{params.hbase_conf_dir}/log4j.properties")):
- File(format("{params.hbase_conf_dir}/log4j.properties"),
- mode=0644,
- group=params.user_group,
- owner=params.hbase_user
- )
-
-def hbase_TemplateConfig(name, tag=None, user=None):
- import params
-
- TemplateConfig( os.path.join(params.hbase_conf_dir, name),
- owner = user,
- template_tag = tag
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_master.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_master.py
deleted file mode 100755
index b769a0d..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_master.py
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-from resource_management import *
-
-from hbase import hbase
-from hbase_service import hbase_service
-from hbase_decommission import hbase_decommission
-
-
-class HbaseMaster(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env, action = None):
- import params
- env.set_params(params)
-
- hbase('master', action)
-
- def start(self, env):
- import params
- env.set_params(params)
- self.configure(env, action = 'start') # for security
-
- hbase_service( 'master',
- action = 'start'
- )
-
- def stop(self, env):
- import params
- env.set_params(params)
-
- hbase_service( 'master',
- action = 'stop'
- )
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
- check_process_status(pid_file)
-
- def decommission(self, env):
- import params
- env.set_params(params)
-
- hbase_decommission(env)
-
-
-if __name__ == "__main__":
- HbaseMaster().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_regionserver.py
deleted file mode 100755
index cf0efef..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_regionserver.py
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-from resource_management import *
-
-from hbase import hbase
-from hbase_service import hbase_service
-
-
-class HbaseRegionServer(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env, action = None):
- import params
- env.set_params(params)
-
- hbase('regionserver', action)
-
- def start(self, env):
- import params
- env.set_params(params)
- self.configure(env, action = 'start') # for security
-
- hbase_service( 'regionserver',
- action = 'start'
- )
-
- def stop(self, env):
- import params
- env.set_params(params)
-
- hbase_service( 'regionserver',
- action = 'stop'
- )
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
- check_process_status(pid_file)
-
- def decommission(self, env):
- print "Decommission not yet implemented!"
-
-
-if __name__ == "__main__":
- HbaseRegionServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_service.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_service.py
deleted file mode 100755
index 5f03ca0..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/hbase_service.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-def hbase_service(
- name,
- action = 'start'): # 'start' or 'stop' or 'status'
-
- import params
-
- role = name
- cmd = format("{daemon_script} --config {hbase_conf_dir}")
- pid_file = format("{hbase_pid_dir}/hbase-{hbase_user}-{role}.pid")
- no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
-
- if action == 'start':
- daemon_cmd = format("{cmd} start {role}")
-
- Execute ( daemon_cmd,
- not_if = no_op_test,
- user = params.hbase_user
- )
- elif action == 'stop':
- daemon_cmd = format("{cmd} stop {role}")
-
- Execute ( daemon_cmd,
- user = params.hbase_user,
- # BUGFIX: hbase regionserver sometimes hangs when nn is in safemode
- timeout = 30,
- on_timeout = format("{no_op_test} && kill -9 `cat {pid_file}`")
- )
-
- File(pid_file,
- action = "delete",
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/metrics_collector.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/metrics_collector.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/metrics_collector.py
deleted file mode 100755
index cf498ec..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/metrics_collector.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from resource_management.libraries.functions.security_commons import build_expectations, \
- cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
- FILE_TYPE_XML
-from ams import ams
-from ams_service import ams_service
-from hbase import hbase
-from status import check_service_status
-from ambari_commons import OSConst
-from ambari_commons.os_family_impl import OsFamilyImpl
-
-class AmsCollector(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env, action = None):
- import params
- env.set_params(params)
- hbase('master', action)
- hbase('regionserver', action)
- ams(name='collector')
-
- def start(self, env):
- self.configure(env, action = 'start') # for security
- # stop hanging components before start
- ams_service('collector', action = 'stop')
- ams_service('collector', action = 'start')
-
- def stop(self, env):
- import params
- env.set_params(params)
- # Sometimes, stop() may be called before start(), in case restart() is initiated right after installation
- self.configure(env, action = 'stop') # for security
- ams_service('collector', action = 'stop')
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- check_service_status(name='collector')
-
-
-@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class AmsCollectorDefault(AmsCollector):
- def security_status(self, env):
- import status_params
-
- env.set_params(status_params)
- props_value_check = {"hbase.security.authentication": "kerberos",
- "hbase.security.authorization": "true"}
-
- props_empty_check = ["hbase.zookeeper.property.authProvider.1",
- "hbase.master.keytab.file",
- "hbase.master.kerberos.principal",
- "hbase.regionserver.keytab.file",
- "hbase.regionserver.kerberos.principal"
- ]
- props_read_check = ['hbase.master.keytab.file', 'hbase.regionserver.keytab.file']
- ams_hbase_site_expectations = build_expectations('hbase-site', props_value_check,
- props_empty_check,
- props_read_check)
-
- expectations = {}
- expectations.update(ams_hbase_site_expectations)
-
- security_params = get_params_from_filesystem(status_params.ams_hbase_conf_dir,
- {'hbase-site.xml': FILE_TYPE_XML})
-
- is_hbase_distributed = security_params['hbase-site']['hbase.cluster.distributed']
- # for embedded mode, when HBase is backed by file, security state is SECURED_KERBEROS by definition when cluster is secured
- if status_params.security_enabled and not is_hbase_distributed:
- self.put_structured_out({"securityState": "SECURED_KERBEROS"})
- return
-
- result_issues = validate_security_config_properties(security_params, expectations)
-
- if not result_issues: # If all validations passed successfully
- try:
- # Double check the dict before calling execute
- if ('hbase-site' not in security_params or
- 'hbase.master.keytab.file' not in security_params['hbase-site'] or
- 'hbase.master.kerberos.principal' not in security_params['hbase-site']):
- self.put_structured_out({"securityState": "UNSECURED"})
- self.put_structured_out(
- {"securityIssuesFound": "Keytab file or principal are not set property."})
- return
-
- cached_kinit_executor(status_params.kinit_path_local,
- status_params.hbase_user,
- security_params['hbase-site']['hbase.master.keytab.file'],
- security_params['hbase-site']['hbase.master.kerberos.principal'],
- status_params.hostname,
- status_params.tmp_dir)
- self.put_structured_out({"securityState": "SECURED_KERBEROS"})
- except Exception as e:
- self.put_structured_out({"securityState": "ERROR"})
- self.put_structured_out({"securityStateErrorInfo": str(e)})
- else:
- issues = []
- for cf in result_issues:
- issues.append("Configuration file %s did not pass the validation. Reason: %s" % (
- cf, result_issues[cf]))
- self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
- self.put_structured_out({"securityState": "UNSECURED"})
-
-
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class AmsCollectorWindows(AmsCollector):
- def install(self, env):
- self.install_packages(env)
- self.configure(env) # for security
-
-if __name__ == "__main__":
- AmsCollector().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/metrics_monitor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/metrics_monitor.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/metrics_monitor.py
deleted file mode 100755
index f6cce8f..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/metrics_monitor.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from ams import ams
-from ams_service import ams_service
-from status import check_service_status
-
-class AmsMonitor(Script):
- def install(self, env):
- self.install_packages(env)
- self.configure(env) # for security
-
- def configure(self, env):
- import params
- env.set_params(params)
- ams(name='monitor')
-
- def start(self, env):
- self.configure(env) # for security
-
- ams_service( 'monitor',
- action = 'start'
- )
-
- def stop(self, env):
- import params
- env.set_params(params)
-
- ams_service( 'monitor',
- action = 'stop'
- )
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- check_service_status(name='monitor')
-
-
-if __name__ == "__main__":
- AmsMonitor().execute()
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params.py
deleted file mode 100755
index b0a2b75..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params.py
+++ /dev/null
@@ -1,257 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from functions import calc_xmn_from_xms
-from functions import check_append_heap_property
-from functions import trim_heap_property
-
-from resource_management import *
-import status_params
-from ambari_commons import OSCheck
-
-
-if OSCheck.is_windows_family():
- from params_windows import *
-else:
- from params_linux import *
-# server configurations
-config = Script.get_config()
-exec_tmp_dir = Script.get_tmp_dir()
-
-def get_combined_memory_mb(value1, value2):
- try:
- part1 = int(value1.strip()[:-1]) if value1.lower().strip()[-1:] == 'm' else int(value1)
- part2 = int(value2.strip()[:-1]) if value2.lower().strip()[-1:] == 'm' else int(value2)
- return str(part1 + part2) + 'm'
- except:
- return None
-pass
-
-#AMBARI_METRICS data
-ams_pid_dir = status_params.ams_collector_pid_dir
-
-ams_collector_script = "/usr/sbin/ambari-metrics-collector"
-ams_collector_pid_dir = status_params.ams_collector_pid_dir
-ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
-if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
-else:
- metric_collector_host = ams_collector_hosts[0]
-if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
-else:
- metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
- if metric_collector_web_address.find(':') != -1:
- metric_collector_port = metric_collector_web_address.split(':')[1]
- else:
- metric_collector_port = '6188'
-
-ams_collector_log_dir = config['configurations']['ams-env']['metrics_collector_log_dir']
-ams_monitor_log_dir = config['configurations']['ams-env']['metrics_monitor_log_dir']
-
-ams_monitor_dir = "/usr/lib/python2.6/site-packages/resource_monitoring"
-ams_monitor_pid_dir = status_params.ams_monitor_pid_dir
-ams_monitor_script = "/usr/sbin/ambari-metrics-monitor"
-
-ams_hbase_home_dir = "/usr/lib/ams-hbase/"
-
-ams_hbase_normalizer_enabled = default("/configurations/ams-hbase-site/hbase.normalizer.enabled", None)
-ams_hbase_fifo_compaction_enabled = default("/configurations/ams-site/timeline.metrics.hbase.fifo.compaction.enabled", None)
-
-#hadoop params
-
-hbase_excluded_hosts = config['commandParams']['excluded_hosts']
-hbase_drain_only = config['commandParams']['mark_draining_only']
-hbase_included_hosts = config['commandParams']['included_hosts']
-
-hbase_user = status_params.hbase_user
-smokeuser = config['configurations']['cluster-env']['smokeuser']
-hbase_root_dir = config['configurations']['ams-hbase-site']['hbase.rootdir']
-hbase_pid_dir = status_params.hbase_pid_dir
-
-is_hbase_distributed = config['configurations']['ams-hbase-site']['hbase.cluster.distributed']
-is_local_fs_rootdir = hbase_root_dir.startswith('file://')
-is_ams_distributed = config['configurations']['ams-site']['timeline.metrics.service.operation.mode'] == 'distributed'
-
-# security is disabled for embedded mode, when HBase is backed by file
-security_enabled = False if not is_hbase_distributed else config['configurations']['cluster-env']['security_enabled']
-
-# this is "hadoop-metrics.properties" for 1.x stacks
-metric_prop_file_name = "hadoop-metrics2-hbase.properties"
-
-# not supporting 32 bit jdk.
-java64_home = config['hostLevelParams']['java_home']
-java_version = int(config['hostLevelParams']['java_version'])
-
-metrics_collector_heapsize = default('/configurations/ams-env/metrics_collector_heapsize', "512")
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
-metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
-metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 60)
-
-hbase_log_dir = config['configurations']['ams-hbase-env']['hbase_log_dir']
-hbase_classpath_additional = default("/configurations/ams-hbase-env/hbase_classpath_additional", None)
-master_heapsize = config['configurations']['ams-hbase-env']['hbase_master_heapsize']
-regionserver_heapsize = config['configurations']['ams-hbase-env']['hbase_regionserver_heapsize']
-
-# Check if hbase java options already have appended "m". If Yes, remove the trailing m.
-metrics_collector_heapsize = check_append_heap_property(str(metrics_collector_heapsize), "m")
-master_heapsize = check_append_heap_property(str(master_heapsize), "m")
-regionserver_heapsize = check_append_heap_property(str(regionserver_heapsize), "m")
-
-regionserver_xmn_max = default('/configurations/ams-hbase-env/hbase_regionserver_xmn_max', None)
-if regionserver_xmn_max:
- regionserver_xmn_max = int(trim_heap_property(str(regionserver_xmn_max), "m"))
- regionserver_xmn_percent = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_ratio']
- regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
-else:
- regionserver_xmn_size = config['configurations']['ams-hbase-env']['regionserver_xmn_size']
-pass
-
-hbase_master_xmn_size = config['configurations']['ams-hbase-env']['hbase_master_xmn_size']
-hbase_master_maxperm_size = config['configurations']['ams-hbase-env']['hbase_master_maxperm_size']
-
-# Check if hbase java options already have appended "m". If Yes, remove the trailing m.
-hbase_master_maxperm_size = check_append_heap_property(str(hbase_master_maxperm_size), "m")
-hbase_master_xmn_size = check_append_heap_property(str(hbase_master_xmn_size), "m")
-regionserver_xmn_size = check_append_heap_property(str(regionserver_xmn_size), "m")
-
-# Choose heap size for embedded mode as sum of master + regionserver
-if not is_hbase_distributed:
- hbase_heapsize = get_combined_memory_mb(master_heapsize, regionserver_heapsize)
- if hbase_heapsize is None:
- hbase_heapsize = master_heapsize
-else:
- hbase_heapsize = master_heapsize
-
-max_open_files_limit = default("/configurations/ams-hbase-env/max_open_files_limit", "32768")
-
-if not is_hbase_distributed:
- zookeeper_quorum_hosts = 'localhost'
- zookeeper_clientPort = '61181'
-else:
- zookeeper_quorum_hosts = default("/hostname", 'localhost')
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
-
-ams_checkpoint_dir = config['configurations']['ams-site']['timeline.metrics.aggregator.checkpoint.dir']
-hbase_pid_dir = status_params.hbase_pid_dir
-_hbase_tmp_dir = config['configurations']['ams-hbase-site']['hbase.tmp.dir']
-hbase_tmp_dir = substitute_vars(_hbase_tmp_dir, config['configurations']['ams-hbase-site'])
-_zookeeper_data_dir = config['configurations']['ams-hbase-site']['hbase.zookeeper.property.dataDir']
-zookeeper_data_dir = substitute_vars(_zookeeper_data_dir, config['configurations']['ams-hbase-site'])
-# TODO UPGRADE default, update site during upgrade
-_local_dir_conf = default('/configurations/ams-hbase-site/hbase.local.dir', "${hbase.tmp.dir}/local")
-local_dir = substitute_vars(_local_dir_conf, config['configurations']['ams-hbase-site'])
-
-phoenix_max_global_mem_percent = default('/configurations/ams-site/phoenix.query.maxGlobalMemoryPercentage', '20')
-phoenix_client_spool_dir = default('/configurations/ams-site/phoenix.spool.directory', '/tmp')
-phoenix_server_spool_dir = default('/configurations/ams-hbase-site/phoenix.spool.directory', '/tmp')
-# Substitute vars if present
-phoenix_client_spool_dir = substitute_vars(phoenix_client_spool_dir, config['configurations']['ams-hbase-site'])
-phoenix_server_spool_dir = substitute_vars(phoenix_server_spool_dir, config['configurations']['ams-hbase-site'])
-
-client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf")
-master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf")
-regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf")
-
-rs_hosts = ["localhost"]
-
-smoke_test_user = config['configurations']['cluster-env']['smokeuser']
-smokeuser_permissions = "RWXCA"
-service_check_data = functions.get_unique_id_and_date()
-user_group = config['configurations']['cluster-env']["user_group"]
-hadoop_user = "hadoop"
-
-kinit_cmd = ""
-
-if security_enabled:
- _hostname_lowercase = config['hostname'].lower()
- client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf")
- smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
- hbase_user_keytab = config['configurations']['ams-hbase-env']['hbase_user_keytab']
-
- ams_collector_jaas_config_file = format("{hbase_conf_dir}/ams_collector_jaas.conf")
- ams_collector_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.myclient.keytab']
- ams_collector_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.myclient.principal'].replace('_HOST',_hostname_lowercase)
-
- ams_zookeeper_jaas_config_file = format("{hbase_conf_dir}/ams_zookeeper_jaas.conf")
- ams_zookeeper_keytab = config['configurations']['ams-hbase-security-site']['ams.zookeeper.keytab']
- ams_zookeeper_principal_name = config['configurations']['ams-hbase-security-site']['ams.zookeeper.principal'].replace('_HOST',_hostname_lowercase)
-
- master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf")
- master_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.master.keytab.file']
- master_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
-
- regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf")
- regionserver_keytab_path = config['configurations']['ams-hbase-security-site']['hbase.regionserver.keytab.file']
- regionserver_jaas_princ = config['configurations']['ams-hbase-security-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
-
- zk_servicename = ams_zookeeper_principal_name.rpartition('/')[0]
-
-#log4j.properties
-if (('ams-hbase-log4j' in config['configurations']) and ('content' in config['configurations']['ams-hbase-log4j'])):
- hbase_log4j_props = config['configurations']['ams-hbase-log4j']['content']
-else:
- hbase_log4j_props = None
-
-if (('ams-log4j' in config['configurations']) and ('content' in config['configurations']['ams-log4j'])):
- log4j_props = config['configurations']['ams-log4j']['content']
-else:
- log4j_props = None
-
-hbase_env_sh_template = config['configurations']['ams-hbase-env']['content']
-ams_env_sh_template = config['configurations']['ams-env']['content']
-
-hbase_staging_dir = default("/configurations/ams-hbase-site/hbase.bulkload.staging.dir", "/amshbase/staging")
-
-#for create_hdfs_directory
-hostname = config["hostname"]
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-
-
-
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-
-import functools
-#create partial functions with common arguments for every HdfsResource call
-#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
- user=hdfs_user,
- security_enabled = security_enabled,
- keytab = hdfs_user_keytab,
- kinit_path_local = kinit_path_local,
- hadoop_bin_dir = hadoop_bin_dir,
- hadoop_conf_dir = hadoop_conf_dir,
- principal_name = hdfs_principal_name,
- hdfs_site = hdfs_site,
- default_fs = default_fs
- )
-
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params_linux.py
deleted file mode 100755
index 838e987..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params_linux.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from resource_management.libraries.functions import conf_select
-from ambari_commons import OSCheck
-from ambari_commons.constants import AMBARI_SUDO_BINARY
-
-config = Script.get_config()
-
-ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf"
-ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf/"
-ams_user = config['configurations']['ams-env']['ambari_metrics_user']
-#RPM versioning support
-rpm_version = default("/configurations/hadoop-env/rpm_version", None)
-
-#hadoop params
-if rpm_version is not None:
- #RPM versioning support
- rpm_version = default("/configurations/hadoop-env/rpm_version", None)
-
-hadoop_native_lib = format("/usr/lib/ams-hbase/lib/hadoop-native")
-hadoop_bin_dir = "/usr/bin"
-daemon_script = "/usr/lib/ams-hbase/bin/hbase-daemon.sh"
-region_mover = "/usr/lib/ams-hbase/bin/region_mover.rb"
-region_drainer = "/usr/lib/ams-hbase/bin/draining_servers.rb"
-hbase_cmd = "/usr/lib/ams-hbase/bin/hbase"
-
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-hbase_conf_dir = "/etc/ams-hbase/conf"
-
-limits_conf_dir = "/etc/security/limits.d"
-sudo = AMBARI_SUDO_BINARY
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params_windows.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params_windows.py
deleted file mode 100755
index acb5bba..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/params_windows.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-
-from resource_management.libraries.script.script import Script
-
-
-config = Script.get_config()
-
-hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
-ams_user = hadoop_user
-
-try:
- ams_collector_conf_dir = os.environ["COLLECTOR_CONF_DIR"]
- ams_collector_home_dir = os.environ["COLLECTOR_HOME"]
- hbase_cmd = os.path.join(os.environ["COLLECTOR_HOME"], "hbase", "bin", "hbase.cmd")
- hbase_conf_dir = os.path.join(os.environ["COLLECTOR_HOME"], "hbase", "conf")
-except:
- ams_collector_conf_dir = None
- ams_collector_home_dir = None
- hbase_cmd = None
- hbase_conf_dir = None
-
-try:
- ams_monitor_conf_dir = os.environ["MONITOR_CONF_DIR"]
- ams_monitor_home_dir = os.environ["MONITOR_HOME"]
-except:
- ams_monitor_conf_dir = None
- ams_monitor_home_dir = None
-
-hadoop_native_lib = os.path.join(os.environ["HADOOP_HOME"], "bin")
-hadoop_bin_dir = os.path.join(os.environ["HADOOP_HOME"], "bin")
-hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf")
-
-from service_mapping import *
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/service_check.py
deleted file mode 100755
index f19c823..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/service_check.py
+++ /dev/null
@@ -1,166 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.logger import Logger
-from resource_management.core.base import Fail
-from resource_management import Script
-from resource_management import Template
-
-from ambari_commons import OSConst
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-import httplib
-import urllib
-import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
-import os
-import random
-import time
-import socket
-
-
-class AMSServiceCheck(Script):
- AMS_METRICS_POST_URL = "/ws/v1/timeline/metrics/"
- AMS_METRICS_GET_URL = "/ws/v1/timeline/metrics?%s"
- AMS_CONNECT_TRIES = 30
- AMS_CONNECT_TIMEOUT = 15
-
- @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
- def service_check(self, env):
- from resource_management.libraries.functions.windows_service_utils import check_windows_service_exists
- import params
-
- env.set_params(params)
-
- #Just check that the services were correctly installed
- #Check the monitor on all hosts
- Logger.info("Metrics Monitor service check was started.")
- if not check_windows_service_exists(params.ams_monitor_win_service_name):
- raise Fail("Metrics Monitor service was not properly installed. Check the logs and retry the installation.")
- #Check the collector only where installed
- if params.ams_collector_home_dir and os.path.isdir(params.ams_collector_home_dir):
- Logger.info("Metrics Collector service check was started.")
- if not check_windows_service_exists(params.ams_collector_win_service_name):
- raise Fail("Metrics Collector service was not properly installed. Check the logs and retry the installation.")
-
- @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
- def service_check(self, env):
- import params
-
- Logger.info("Ambari Metrics service check was started.")
- env.set_params(params)
-
- random_value1 = random.random()
- headers = {"Content-type": "application/json"}
-
- for i in xrange(0, self.AMS_CONNECT_TRIES):
- try:
- current_time = int(time.time()) * 1000
- metric_json = Template('smoketest_metrics.json.j2', hostname=params.hostname, random1=random_value1,
- current_time=current_time).get_content()
- Logger.info("Generated metrics:\n%s" % metric_json)
-
- Logger.info("Connecting (POST) to %s:%s%s" % (params.metric_collector_host,
- params.metric_collector_port,
- self.AMS_METRICS_POST_URL))
- conn = httplib.HTTPConnection(params.metric_collector_host,
- int(params.metric_collector_port))
- conn.request("POST", self.AMS_METRICS_POST_URL, metric_json, headers)
-
- response = conn.getresponse()
- Logger.info("Http response: %s %s" % (response.status, response.reason))
- except (httplib.HTTPException, socket.error) as ex:
- if i < self.AMS_CONNECT_TRIES - 1: #range/xrange returns items from start to end-1
- time.sleep(self.AMS_CONNECT_TIMEOUT)
- Logger.info("Connection failed. Next retry in %s seconds."
- % (self.AMS_CONNECT_TIMEOUT))
- continue
- else:
- raise Fail("Metrics were not saved. Service check has failed. "
- "\nConnection failed.")
-
- data = response.read()
- Logger.info("Http data: %s" % data)
- conn.close()
-
- if response.status == 200:
- Logger.info("Metrics were saved.")
- break
- else:
- Logger.info("Metrics were not saved. Service check has failed.")
- if i < self.AMS_CONNECT_TRIES - 1: #range/xrange returns items from start to end-1
- time.sleep(self.AMS_CONNECT_TIMEOUT)
- Logger.info("Next retry in %s seconds."
- % (self.AMS_CONNECT_TIMEOUT))
- else:
- raise Fail("Metrics were not saved. Service check has failed. POST request status: %s %s \n%s" %
- (response.status, response.reason, data))
-
- get_metrics_parameters = {
- "metricNames": "AMBARI_METRICS.SmokeTest.FakeMetric",
- "appId": "amssmoketestfake",
- "hostname": params.hostname,
- "startTime": current_time - 60000,
- "endTime": current_time + 61000,
- "precision": "seconds",
- "grouped": "false",
- }
- encoded_get_metrics_parameters = urllib.urlencode(get_metrics_parameters)
-
- Logger.info("Connecting (GET) to %s:%s%s" % (params.metric_collector_host,
- params.metric_collector_port,
- self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters))
-
- conn = httplib.HTTPConnection(params.metric_collector_host,
- int(params.metric_collector_port))
- conn.request("GET", self.AMS_METRICS_GET_URL % encoded_get_metrics_parameters)
- response = conn.getresponse()
- Logger.info("Http response: %s %s" % (response.status, response.reason))
-
- data = response.read()
- Logger.info("Http data: %s" % data)
- conn.close()
-
- if response.status == 200:
- Logger.info("Metrics were retrieved.")
- else:
- Logger.info("Metrics were not retrieved. Service check has failed.")
- raise Fail("Metrics were not retrieved. Service check has failed. GET request status: %s %s \n%s" %
- (response.status, response.reason, data))
- data_json = json.loads(data)
-
- def floats_eq(f1, f2, delta):
- return abs(f1-f2) < delta
-
- for metrics_data in data_json["metrics"]:
- if (str(current_time) in metrics_data["metrics"] and str(current_time + 1000) in metrics_data["metrics"]
- and floats_eq(metrics_data["metrics"][str(current_time)], random_value1, 0.0000001)
- and floats_eq(metrics_data["metrics"][str(current_time + 1000)], current_time, 1)):
- Logger.info("Values %s and %s were found in the response." % (random_value1, current_time))
- break
- pass
- else:
- Logger.info("Values %s and %s were not found in the response." % (random_value1, current_time))
- raise Fail("Values %s and %s were not found in the response." % (random_value1, current_time))
-
- Logger.info("Ambari Metrics service check is finished.")
-
-if __name__ == "__main__":
- AMSServiceCheck().execute()
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/service_mapping.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/service_mapping.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/service_mapping.py
deleted file mode 100755
index 2eeb427..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/service_mapping.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-ams_collector_win_service_name = "AmbariMetricsCollector"
-ams_monitor_win_service_name = "AmbariMetricsHostMonitoring"
-ams_embedded_hbase_win_service_name = "ams_hbase_master"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/split_points.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/split_points.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/split_points.py
deleted file mode 100755
index fa4deaf..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/split_points.py
+++ /dev/null
@@ -1,236 +0,0 @@
-# !/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import sys
-import re
-import math
-import collections
-import ast
-
-metric_filename_ext = '.txt'
-# 5 regions for higher order aggregate tables
-other_region_static_count = 6
-# Max equidistant points to return per service
-max_equidistant_points = 50
-
-b_bytes = 1
-k_bytes = 1 << 10 # 1024
-m_bytes = 1 << 20 # 1024^2
-g_bytes = 1 << 30 # 1024^3
-t_bytes = 1 << 40 # 1024^4
-p_bytes = 1 << 50 # 1024^5
-
-def to_number(s):
- try:
- return int(re.sub("\D", "", s))
- except ValueError:
- return None
-
-def format_Xmx_size_to_bytes(value, default='b'):
- strvalue = str(value).lower()
- if len(strvalue) == 0:
- return 0
- modifier = strvalue[-1]
-
- if modifier == ' ' or modifier in "0123456789":
- modifier = default
-
- m = {
- modifier == 'b': b_bytes,
- modifier == 'k': k_bytes,
- modifier == 'm': m_bytes,
- modifier == 'g': g_bytes,
- modifier == 't': t_bytes,
- modifier == 'p': p_bytes
- } [1]
- return to_number(strvalue) * m
-
-# Class that takes AMS HBase configs as input and determines the Region
-# pre-splits based on selected services also passed as a parameter to the class.
-class FindSplitPointsForAMSRegions():
-
- def __init__(self, ams_hbase_site, ams_hbase_env, serviceMetricsDir,
- operation_mode = 'embedded', services = None):
- self.ams_hbase_site = ams_hbase_site
- self.ams_hbase_env = ams_hbase_env
- self.serviceMetricsDir = serviceMetricsDir
- self.services = services
- self.mode = operation_mode
- # Add host metrics if not present as input
- if self.services and 'HOST' not in self.services:
- self.services.append('HOST')
-
- # Initialize before user
- self.initialize()
-
- def initialize(self):
- # calculate regions based on available memory
- self.initialize_region_counts()
- self.initialize_ordered_set_of_metrics()
-
- def initialize_region_counts(self):
- try:
- xmx_master_bytes = format_Xmx_size_to_bytes(self.ams_hbase_env['hbase_master_heapsize'], 'm')
- xmx_region_bytes = 0
- if "hbase_regionserver_heapsize" in self.ams_hbase_env:
- xmx_region_bytes = format_Xmx_size_to_bytes(self.ams_hbase_env['hbase_regionserver_heapsize'], 'm')
- xmx_bytes = xmx_master_bytes + xmx_region_bytes
- if self.mode == 'distributed':
- xmx_bytes = xmx_region_bytes
-
- memstore_max_mem = float(self.ams_hbase_site['hbase.regionserver.global.memstore.lowerLimit']) * xmx_bytes
- memstore_flush_size = format_Xmx_size_to_bytes(self.ams_hbase_site['hbase.hregion.memstore.flush.size'])
-
- max_inmemory_regions = (memstore_max_mem / memstore_flush_size) - other_region_static_count
- print 'max_inmemory_regions: %s' % max_inmemory_regions
-
- if max_inmemory_regions > 2:
- # Lets say total = 12, so we have 7 regions to allocate between
- # METRIC_RECORD and METRIC_AGGREGATE tables, desired = (5, 2)
- self.desired_precision_region_count = int(math.floor(0.8 * max_inmemory_regions))
- self.desired_aggregate_region_count = int(max_inmemory_regions - self.desired_precision_region_count)
- else:
- self.desired_precision_region_count = 1
- self.desired_aggregate_region_count = 1
-
- except:
- print('Bad config settings, could not calculate max regions available.')
- pass
-
- def initialize_ordered_set_of_metrics(self):
- onlyServicefiles = [ f for f in os.listdir(self.serviceMetricsDir) if
- os.path.isfile(os.path.join(self.serviceMetricsDir, f)) ]
-
- metrics = set()
-
- for file in onlyServicefiles:
- # Process for services selected at deploy time or all services if
- # services arg is not passed
- if self.services is None or file.rstrip(metric_filename_ext) in self.services:
- print 'Processing file: %s' % os.path.join(self.serviceMetricsDir, file)
- service_metrics = set()
- with open(os.path.join(self.serviceMetricsDir, file), 'r') as f:
- for metric in f:
- service_metrics.add(metric.strip())
- pass
- pass
- metrics.update(self.find_equidistant_metrics(list(sorted(service_metrics))))
- pass
- pass
-
- self.metrics = sorted(metrics)
- print 'metrics length: %s' % len(self.metrics)
-
- # Pick 50 metric points for each service that are equidistant from
- # each other for a service
- def find_equidistant_metrics(self, service_metrics):
- equi_metrics = []
- idx = len(service_metrics) / max_equidistant_points
- if idx == 0:
- return service_metrics
- pass
-
- index = idx
- for i in range(0, max_equidistant_points - 1):
- equi_metrics.append(service_metrics[index - 1])
- index += idx
- pass
-
- return equi_metrics
-
- def get_split_points(self):
- split_points = collections.namedtuple('SplitPoints', [ 'precision', 'aggregate' ])
- split_points.precision = []
- split_points.aggregate = []
-
- metric_list = list(self.metrics)
- metrics_total = len(metric_list)
-
- if self.desired_precision_region_count > 1:
- idx = int(math.ceil(metrics_total / self.desired_precision_region_count))
- index = idx
- for i in range(0, self.desired_precision_region_count - 1):
- if index < metrics_total - 1:
- split_points.precision.append(metric_list[index])
- index += idx
-
- if self.desired_aggregate_region_count > 1:
- idx = int(math.ceil(metrics_total / self.desired_aggregate_region_count))
- index = idx
- for i in range(0, self.desired_aggregate_region_count - 1):
- if index < metrics_total - 1:
- split_points.aggregate.append(metric_list[index])
- index += idx
-
- return split_points
- pass
-
-def main(argv = None):
- scriptDir = os.path.realpath(os.path.dirname(argv[0]))
- serviceMetricsDir = os.path.join(scriptDir, os.pardir, 'files', 'service-metrics')
-
- if os.path.exists(serviceMetricsDir):
- onlyargs = argv[1:]
- if len(onlyargs) < 3:
- sys.stderr.write("Usage: dict(ams-hbase-site) dict(ams-hbase-env) list(services)\n")
- sys.exit(2)
- pass
-
- ams_hbase_site = None
- ams_hbase_env = None
- services = None
- try:
- ams_hbase_site = ast.literal_eval(str(onlyargs[0]))
- ams_hbase_env = ast.literal_eval(str(onlyargs[1]))
- services = onlyargs[2]
- if services:
- services = str(services).split(',')
- pass
- except Exception, ex:
- sys.stderr.write(str(ex))
- sys.stderr.write("\nUsage: Expected items not found in input. Found "
- " ams-hbase-site => {0}, ams-hbase-env => {1},"
- " services => {2}".format(ams_hbase_site, ams_hbase_env, services))
- sys.exit(2)
-
- print '--------- AMS Regions Split point finder ---------'
- print 'Services: %s' % services
-
- mode = 'distributed' if 'hbase.rootdir' in ams_hbase_site and \
- 'hdfs' in ams_hbase_site['hbase.rootdir'] else \
- 'embedded'
-
- split_point_finder = FindSplitPointsForAMSRegions(
- ams_hbase_site, ams_hbase_env, serviceMetricsDir, mode, services)
-
- result = split_point_finder.get_split_points()
- print 'Split points for precision table : %s' % len(result.precision)
- print 'precision: %s' % str(result.precision)
- print 'Split points for aggregate table : %s' % len(result.aggregate)
- print 'aggregate: %s' % str(result.aggregate)
-
- return 0
-
- else:
- print 'Cannot find service metrics dir in %s' % scriptDir
-
-if __name__ == '__main__':
- main(sys.argv)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0aab3803/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/status.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/status.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/status.py
deleted file mode 100755
index 59466ad..0000000
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/AMBARI_METRICS/package/scripts/status.py
+++ /dev/null
@@ -1,46 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from ambari_commons import OSConst
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-import os
-
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def check_service_status(name):
- if name=='collector':
- pid_file = format("{ams_collector_pid_dir}/ambari-metrics-collector.pid")
- check_process_status(pid_file)
- pid_file = format("{hbase_pid_dir}/hbase-{hbase_user}-master.pid")
- check_process_status(pid_file)
- if os.path.exists(format("{hbase_pid_dir}/distributed_mode")):
- pid_file = format("{hbase_pid_dir}/hbase-{hbase_user}-regionserver.pid")
- check_process_status(pid_file)
-
- elif name == 'monitor':
- pid_file = format("{ams_monitor_pid_dir}/ambari-metrics-monitor.pid")
- check_process_status(pid_file)
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def check_service_status(name):
- import service_mapping
- if name=='collector':
- check_windows_service_status(service_mapping.ams_collector_win_service_name)
- elif name == 'monitor':
- check_windows_service_status(service_mapping.ams_monitor_win_service_name)