You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/03/15 18:18:01 UTC
ambari git commit: AMBARI-10075 Additional configuration properies
for AMS based on cluster size (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 9c22eafba -> c12ffd18f
AMBARI-10075 Additional configuration properies for AMS based on cluster size (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c12ffd18
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c12ffd18
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c12ffd18
Branch: refs/heads/trunk
Commit: c12ffd18fc74633d721842a9aa327c22654cca8d
Parents: 9c22eaf
Author: Dmytro Sen <ds...@apache.org>
Authored: Sun Mar 15 19:15:28 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Sun Mar 15 19:15:54 2015 +0200
----------------------------------------------------------------------
.../0.1.0/configuration/ams-env.xml | 11 ++-
.../AMBARI_METRICS/0.1.0/metainfo.xml | 1 +
.../0.1.0/package/scripts/params.py | 2 +
.../stacks/HDP/2.0.6/services/stack_advisor.py | 92 ++++++++++++--------
.../stacks/2.2/common/test_stack_advisor.py | 5 ++
5 files changed, 72 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c12ffd18/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
index a294b0f..a3ddb6a 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
@@ -45,7 +45,11 @@
<value>/var/log/ambari-metrics-monitor</value>
<description>Monitor log directory.</description>
</property>
-
+ <property>
+ <name>metrics_collector_heapsize</name>
+ <value>512m</value>
+ <description>Metrics Collector Heap Size</description>
+ </property>
<property>
<name>content</name>
<value>
@@ -69,8 +73,11 @@ export AMS_MONITOR_PID_DIR={{ams_monitor_pid_dir}}
# AMS HBase pid directory
export AMS_HBASE_PID_DIR={{hbase_pid_dir}}
+# AMS Collector heapsize
+export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}
+
# AMS Collector options
-export AMS_COLLECTOR_OPTS="-Djava.library.path=/usr/lib/ams-hbase/lib/hadoop-native"
+export AMS_COLLECTOR_OPTS="-Djava.library.path=/usr/lib/ams-hbase/lib/hadoop-native -Xmx$AMS_COLLECTOR_HEAPSIZE "
{% if security_enabled %}
export AMS_COLLECTOR_OPTS="$AMS_COLLECTOR_OPTS -Djava.security.auth.login.config={{ams_collector_jaas_config_file}}"
{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c12ffd18/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
index f5f51da..ac954e6 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
@@ -121,6 +121,7 @@
<configuration-dependencies>
<config-type>ams-site</config-type>
<config-type>ams-log4j</config-type>
+ <config-type>ams-env</config-type>
<config-type>ams-hbase-policy</config-type>
<config-type>ams-hbase-site</config-type>
<config-type>ams-hbase-security-site</config-type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c12ffd18/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 4a8b90a..77b5ec1 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -73,6 +73,8 @@ metric_prop_file_name = "hadoop-metrics2-hbase.properties"
# not supporting 32 bit jdk.
java64_home = config['hostLevelParams']['java_home']
+metrics_collector_heapsize = default('/configurations/ams-env/metrics_collector_heapsize', "512m")
+
hbase_log_dir = config['configurations']['ams-hbase-env']['hbase_log_dir']
master_heapsize = config['configurations']['ams-hbase-env']['hbase_master_heapsize']
http://git-wip-us.apache.org/repos/asf/ambari/blob/c12ffd18/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 91b274a..6ee60cf 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -127,6 +127,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putHbaseProperty('hbase_master_heapsize', int(clusterData['hbaseRam']) * 1024)
def recommendAmsConfigurations(self, configurations, clusterData, services, hosts):
+ putAmsEnvProperty = self.putProperty(configurations, "ams-env")
+ putAmsEnvProperty = self.putProperty(configurations, "ams-env")
putAmsHbaseSiteProperty = self.putProperty(configurations, "ams-hbase-site")
putTimelineServiceProperty = self.putProperty(configurations, "ams-site")
putHbaseEnvProperty = self.putProperty(configurations, "ams-hbase-env")
@@ -145,12 +147,24 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
totalHostsCount = len(hosts["items"])
if totalHostsCount > 400:
putHbaseEnvProperty("hbase_master_heapsize", "12288m")
+ putAmsEnvProperty("metrics_collector_heapsize", "4096m")
+ putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
+ putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
+ putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
+ putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 268435456)
elif totalHostsCount > 100:
putHbaseEnvProperty("hbase_master_heapsize", "6144m")
+ putAmsEnvProperty("metrics_collector_heapsize", "2048m")
+ putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
+ putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
+ putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
+ putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 268435456)
elif totalHostsCount > 50:
putHbaseEnvProperty("hbase_master_heapsize", "2048m")
+ putAmsEnvProperty("metrics_collector_heapsize", "1024m")
else:
putHbaseEnvProperty("hbase_master_heapsize", "1024m")
+ putAmsEnvProperty("metrics_collector_heapsize", "512m")
def getConfigurationClusterSummary(self, servicesList, hosts, components):
@@ -648,59 +662,63 @@ def getMountPointForDir(dir, mountPoints):
return bestMountFound
def getHeapsizeProperties():
- return { "NAMENODE": { "config-name": "hadoop-env",
+ return { "NAMENODE": [{"config-name": "hadoop-env",
"property": "namenode_heapsize",
- "default": "1024m"},
- "DATANODE": { "config-name": "hadoop-env",
+ "default": "1024m"}],
+ "DATANODE": [{"config-name": "hadoop-env",
"property": "dtnode_heapsize",
- "default": "1024m"},
- "REGIONSERVER": { "config-name": "hbase-env",
+ "default": "1024m"}],
+ "REGIONSERVER": [{"config-name": "hbase-env",
"property": "hbase_regionserver_heapsize",
- "default": "1024m"},
- "HBASE_MASTER": { "config-name": "hbase-env",
+ "default": "1024m"}],
+ "HBASE_MASTER": [{"config-name": "hbase-env",
"property": "hbase_master_heapsize",
- "default": "1024m"},
- "HIVE_CLIENT": { "config-name": "hive-site",
+ "default": "1024m"}],
+ "HIVE_CLIENT": [{"config-name": "hive-site",
"property": "hive.heapsize",
- "default": "1024m"},
- "HISTORYSERVER": { "config-name": "mapred-env",
+ "default": "1024m"}],
+ "HISTORYSERVER": [{"config-name": "mapred-env",
"property": "jobhistory_heapsize",
- "default": "1024m"},
- "OOZIE_SERVER": { "config-name": "oozie-env",
+ "default": "1024m"}],
+ "OOZIE_SERVER": [{"config-name": "oozie-env",
"property": "oozie_heapsize",
- "default": "1024m"},
- "RESOURCEMANAGER": { "config-name": "yarn-env",
+ "default": "1024m"}],
+ "RESOURCEMANAGER": [{"config-name": "yarn-env",
"property": "resourcemanager_heapsize",
- "default": "1024m"},
- "NODEMANAGER": { "config-name": "yarn-env",
+ "default": "1024m"}],
+ "NODEMANAGER": [{"config-name": "yarn-env",
"property": "nodemanager_heapsize",
- "default": "1024m"},
- "APP_TIMELINE_SERVER": { "config-name": "yarn-env",
+ "default": "1024m"}],
+ "APP_TIMELINE_SERVER": [{"config-name": "yarn-env",
"property": "apptimelineserver_heapsize",
- "default": "1024m"},
- "ZOOKEEPER_SERVER": { "config-name": "zookeeper-env",
+ "default": "1024m"}],
+ "ZOOKEEPER_SERVER": [{"config-name": "zookeeper-env",
"property": "zookeeper_heapsize",
- "default": "1024m"},
- "METRICS_COLLECTOR": { "config-name": "ams-hbase-env",
- "property": "hbase_master_heapsize",
- "default": "1024m"},
+ "default": "1024m"}],
+ "METRICS_COLLECTOR": [{"config-name": "ams-hbase-env",
+ "property": "hbase_master_heapsize",
+ "default": "1024m"},
+ {"config-name": "ams-env",
+ "property": "metrics_collector_heapsize",
+ "default": "512m"}],
}
def getMemorySizeRequired(components, configurations):
totalMemoryRequired = 512*1024*1024 # 512Mb for OS needs
for component in components:
if component in getHeapsizeProperties().keys():
- heapSizeProperty = getHeapsizeProperties()[component]
- try:
- properties = configurations[heapSizeProperty["config-name"]]["properties"]
- heapsize = properties[heapSizeProperty["property"]]
- except KeyError:
- heapsize = heapSizeProperty["default"]
-
- # Assume Mb if no modifier
- if len(heapsize) > 1 and heapsize[-1] in '0123456789':
- heapsize = str(heapsize) + "m"
-
- totalMemoryRequired += formatXmxSizeToBytes(heapsize)
+ heapSizeProperties = getHeapsizeProperties()[component]
+ for heapSizeProperty in heapSizeProperties:
+ try:
+ properties = configurations[heapSizeProperty["config-name"]]["properties"]
+ heapsize = properties[heapSizeProperty["property"]]
+ except KeyError:
+ heapsize = heapSizeProperty["default"]
+
+ # Assume Mb if no modifier
+ if len(heapsize) > 1 and heapsize[-1] in '0123456789':
+ heapsize = str(heapsize) + "m"
+
+ totalMemoryRequired += formatXmxSizeToBytes(heapsize)
return totalMemoryRequired
http://git-wip-us.apache.org/repos/asf/ambari/blob/c12ffd18/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index c5beeb3..2dafb67 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -675,6 +675,11 @@ class TestHDP22StackAdvisor(TestCase):
"properties": {
"hbase_master_heapsize": "1024m",
"hbase_regionserver_heapsize": "1024m",
+ }
+ },
+ "ams-env": {
+ "properties": {
+ "metrics_collector_heapsize": "512m",
}
},
"ams-hbase-site": {