You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/09/21 13:29:34 UTC
[12/21] ambari git commit: AMBARI-18390 : Configuration warnings
during Add Service Wizard includes seemingly nonsensical heapsize
recommendations for AMS heap sizes. (avijayan)
AMBARI-18390 : Configuration warnings during Add Service Wizard includes seemingly nonsensical heapsize recommendations for AMS heap sizes. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/382eba15
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/382eba15
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/382eba15
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 382eba15e351df86da96e8d63cbdaf2fe8f4d867
Parents: b4320b5
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Sep 20 13:56:57 2016 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Sep 20 13:56:57 2016 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 106 ++++++++++++++-----
1 file changed, 81 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/382eba15/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index fd0dfed..800bfa2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -949,7 +949,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"STORM": {"storm-site": self.validateStormConfigurations},
"AMBARI_METRICS": {"ams-hbase-site": self.validateAmsHbaseSiteConfigurations,
"ams-hbase-env": self.validateAmsHbaseEnvConfigurations,
- "ams-site": self.validateAmsSiteConfigurations}
+ "ams-site": self.validateAmsSiteConfigurations,
+ "ams-env": self.validateAmsEnvConfigurations}
}
def validateMinMax(self, items, recommendedDefaults, configurations):
@@ -1139,7 +1140,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if logDirItem:
validationItems.extend([{"config-name": "hbase_log_dir", "item": logDirItem}])
- collector_heapsize = to_number(ams_env.get("metrics_collector_heapsize"))
hbase_master_heapsize = to_number(properties["hbase_master_heapsize"])
hbase_master_xmn_size = to_number(properties["hbase_master_xmn_size"])
hbase_regionserver_heapsize = to_number(properties["hbase_regionserver_heapsize"])
@@ -1230,27 +1230,25 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
requiredMemory = getMemorySizeRequired(hostComponents, configurations)
unusedMemory = host["Hosts"]["total_mem"] * 1024 - requiredMemory # in bytes
- if unusedMemory > 4*gb: # warn user, if more than 4GB RAM is unused
- heapPropertyToIncrease = "hbase_regionserver_heapsize" if is_hbase_distributed else "hbase_master_heapsize"
- xmnPropertyToIncrease = "regionserver_xmn_size" if is_hbase_distributed else "hbase_master_xmn_size"
- recommended_collector_heapsize = int((unusedMemory - 4*gb)/5) + collector_heapsize*mb
+
+ heapPropertyToIncrease = "hbase_regionserver_heapsize" if is_hbase_distributed else "hbase_master_heapsize"
+ xmnPropertyToIncrease = "regionserver_xmn_size" if is_hbase_distributed else "hbase_master_xmn_size"
+ hbase_needs_increase = to_number(properties[heapPropertyToIncrease]) * mb < 32 * gb
+
+ if unusedMemory > 4*gb and hbase_needs_increase: # warn user, if more than 4GB RAM is unused
+
recommended_hbase_heapsize = int((unusedMemory - 4*gb)*4/5) + to_number(properties.get(heapPropertyToIncrease))*mb
recommended_hbase_heapsize = min(32*gb, recommended_hbase_heapsize) #Make sure heapsize <= 32GB
- recommended_xmn_size = round_to_n(0.12*recommended_hbase_heapsize/mb,128)
-
- if collector_heapsize < recommended_collector_heapsize or \
- to_number(properties[heapPropertyToIncrease]) < recommended_hbase_heapsize:
- collectorHeapsizeItem = self.getWarnItem("{0} MB RAM is unused on the host {1} based on components " \
- "assigned. Consider allocating {2} MB to " \
- "metrics_collector_heapsize in ams-env, " \
- "{3} MB to {4} in ams-hbase-env"
- .format(unusedMemory/mb, collectorHostName,
- recommended_collector_heapsize/mb,
- recommended_hbase_heapsize/mb,
+ recommended_hbase_heapsize = round_to_n(recommended_hbase_heapsize/mb,128) # Round to 128m multiple
+ if to_number(properties[heapPropertyToIncrease]) < recommended_hbase_heapsize:
+ hbaseHeapsizeItem = self.getWarnItem("Consider allocating {0} MB to {1} in ams-hbase-env to use up some "
+ "unused memory on host"
+ .format(recommended_hbase_heapsize,
heapPropertyToIncrease))
- validationItems.extend([{"config-name": heapPropertyToIncrease, "item": collectorHeapsizeItem}])
+ validationItems.extend([{"config-name": heapPropertyToIncrease, "item": hbaseHeapsizeItem}])
- if to_number(properties[xmnPropertyToIncrease]) < recommended_hbase_heapsize:
+ recommended_xmn_size = round_to_n(0.15*recommended_hbase_heapsize,128)
+ if to_number(properties[xmnPropertyToIncrease]) < recommended_xmn_size:
xmnPropertyToIncreaseItem = self.getWarnItem("Consider allocating {0} MB to use up some unused memory "
"on host".format(recommended_xmn_size))
validationItems.extend([{"config-name": xmnPropertyToIncrease, "item": xmnPropertyToIncreaseItem}])
@@ -1258,6 +1256,38 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return self.toConfigurationValidationProblems(validationItems, "ams-hbase-env")
+ def validateAmsEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+
+ ams_env = getSiteProperties(configurations, "ams-env")
+ mb = 1024 * 1024
+ gb = 1024 * mb
+ validationItems = []
+ collector_heapsize = to_number(ams_env.get("metrics_collector_heapsize"))
+ amsCollectorHosts = self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR")
+ for collectorHostName in amsCollectorHosts:
+ for host in hosts["items"]:
+ if host["Hosts"]["host_name"] == collectorHostName:
+ hostComponents = []
+ for service in services["services"]:
+ for component in service["components"]:
+ if component["StackServiceComponents"]["hostnames"] is not None:
+ if collectorHostName in component["StackServiceComponents"]["hostnames"]:
+ hostComponents.append(component["StackServiceComponents"]["component_name"])
+
+ requiredMemory = getMemorySizeRequired(hostComponents, configurations)
+ unusedMemory = host["Hosts"]["total_mem"] * 1024 - requiredMemory # in bytes
+ collector_needs_increase = collector_heapsize * mb < 16 * gb
+
+ if unusedMemory > 4*gb and collector_needs_increase: # warn user, if more than 4GB RAM is unused
+ recommended_collector_heapsize = int((unusedMemory - 4*gb)/5) + collector_heapsize * mb
+ recommended_collector_heapsize = round_to_n(recommended_collector_heapsize/mb,128) # Round to 128m multiple
+ if collector_heapsize < recommended_collector_heapsize:
+ validation_msg = "Consider allocating {0} MB to metrics_collector_heapsize in ams-env to use up some " \
+ "unused memory on host"
+ collectorHeapsizeItem = self.getWarnItem(validation_msg.format(recommended_collector_heapsize))
+ validationItems.extend([{"config-name": "metrics_collector_heapsize", "item": collectorHeapsizeItem}])
+ pass
+ return self.toConfigurationValidationProblems(validationItems, "ams-env")
def getPreferredMountPoints(self, hostInfo):
@@ -1883,6 +1913,9 @@ def getHeapsizeProperties():
return { "NAMENODE": [{"config-name": "hadoop-env",
"property": "namenode_heapsize",
"default": "1024m"}],
+ "SECONDARY_NAMENODE": [{"config-name": "hadoop-env",
+ "property": "namenode_heapsize",
+ "default": "1024m"}],
"DATANODE": [{"config-name": "hadoop-env",
"property": "dtnode_heapsize",
"default": "1024m"}],
@@ -1892,9 +1925,15 @@ def getHeapsizeProperties():
"HBASE_MASTER": [{"config-name": "hbase-env",
"property": "hbase_master_heapsize",
"default": "1024m"}],
- "HIVE_CLIENT": [{"config-name": "hive-site",
- "property": "hive.heapsize",
- "default": "1024m"}],
+ "HIVE_CLIENT": [{"config-name": "hive-env",
+ "property": "hive.client.heapsize",
+ "default": "1024"}],
+ "HIVE_METASTORE": [{"config-name": "hive-env",
+ "property": "hive.metastore.heapsize",
+ "default": "1024"}],
+ "HIVE_SERVER": [{"config-name": "hive-env",
+ "property": "hive.heapsize",
+ "default": "1024"}],
"HISTORYSERVER": [{"config-name": "mapred-env",
"property": "jobhistory_heapsize",
"default": "1024m"}],
@@ -1911,7 +1950,7 @@ def getHeapsizeProperties():
"property": "apptimelineserver_heapsize",
"default": "1024m"}],
"ZOOKEEPER_SERVER": [{"config-name": "zookeeper-env",
- "property": "zookeeper_heapsize",
+ "property": "zk_server_heapsize",
"default": "1024m"}],
"METRICS_COLLECTOR": [{"config-name": "ams-hbase-env",
"property": "hbase_master_heapsize",
@@ -1924,7 +1963,19 @@ def getHeapsizeProperties():
"default": "512"}],
"ATLAS_SERVER": [{"config-name": "atlas-env",
"property": "atlas_server_xmx",
- "default": "2048"}]
+ "default": "2048"}],
+ "LOGSEARCH_SERVER": [{"config-name": "logsearch-env",
+ "property": "logsearch_app_max_memory",
+ "default": "1024"}],
+ "LOGSEARCH_LOGFEEDER": [{"config-name": "logfeeder-env",
+ "property": "logfeeder_max_mem",
+ "default": "512"}],
+ "SPARK_JOBHISTORYSERVER": [{"config-name": "spark-env",
+ "property": "spark_daemon_memory",
+ "default": "1024"}],
+ "SPARK2_JOBHISTORYSERVER": [{"config-name": "spark2-env",
+ "property": "spark_daemon_memory",
+ "default": "1024"}]
}
def getMemorySizeRequired(components, configurations):
@@ -1944,7 +1995,12 @@ def getMemorySizeRequired(components, configurations):
heapsize = str(heapsize) + "m"
totalMemoryRequired += formatXmxSizeToBytes(heapsize)
-
+ else:
+ if component == "METRICS_MONITOR" or "CLIENT" in component:
+ heapsize = '512m'
+ else:
+ heapsize = '1024m'
+ totalMemoryRequired += formatXmxSizeToBytes(heapsize)
return totalMemoryRequired
def round_to_n(mem_size, n=128):