You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/06/16 01:45:56 UTC
ambari git commit: AMBARI-11937. 'tez.task.launch.cmd-opts' and
'tez.am.launch.cmd-opts' should have JDK8 specific GC params (srimanth)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 01931acd3 -> 24795d2e3
AMBARI-11937. 'tez.task.launch.cmd-opts' and 'tez.am.launch.cmd-opts' should have JDK8 specific GC params (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/24795d2e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/24795d2e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/24795d2e
Branch: refs/heads/branch-2.1
Commit: 24795d2e38080cb075d703980958c852f9f0ab3e
Parents: 01931ac
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Mon Jun 15 16:37:26 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Mon Jun 15 16:37:35 2015 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.3/services/stack_advisor.py | 17 +-
.../stacks/2.3/common/test_stack_advisor.py | 187 ++++++++++++++++++-
2 files changed, 201 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/24795d2e/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index e97dcce..d65d711 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -76,12 +76,25 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
pass
pass
-
if latest_tez_jar_version:
tez_url = 'http://{0}:{1}/views/TEZ/{2}/TEZ_CLUSTER_INSTANCE'.format(server_host, server_port, latest_tez_jar_version)
putTezProperty("tez.tez-ui.history-url.base", tez_url)
pass
+ # TEZ JVM options
+ jvmGCParams = "-XX:+UseParallelGC"
+ if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]:
+ # JDK8 needs different parameters
+ match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"])
+ if match and len(match.groups()) > 0:
+ # Is version >= 1.8
+ versionSplits = re.split("\.", match.group(1))
+ if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7:
+ jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB"
+ putTezProperty('tez.am.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams)
+ putTezProperty('tez.task.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams)
+
+
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP23StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts)
putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
@@ -155,7 +168,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
if match and len(match.groups()) > 0:
# Is version >= 1.8
versionSplits = re.split("\.", match.group(1))
- if versionSplits and len(versionSplits)>1 and int(versionSplits[0])>0 and int(versionSplits[1])>7:
+ if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7:
jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB"
putHiveSiteProperty('hive.tez.java.opts', "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA " + jvmGCParams + " -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps")
http://git-wip-us.apache.org/repos/asf/ambari/blob/24795d2e/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 614644f..3089dfa 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -475,12 +475,197 @@ class TestHDP23StackAdvisor(TestCase):
self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
- # Test JDK1.8
+ # Test JDK1.9
services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.9.2_44'}
expected['hive-site']['properties']['hive.tez.java.opts'] = "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps"
self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+ def test_recommendTezConfigurations(self):
+ self.maxDiff = None
+ configurations = {
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "256",
+ "yarn.scheduler.maximum-allocation-mb": "8192",
+ },
+ },
+ "capacity-scheduler": {
+ "properties": {
+ "yarn.scheduler.capacity.root.queues": "queue1,queue2"
+ }
+ }
+ }
+ clusterData = {
+ "cpu": 4,
+ "mapMemory": 3000,
+ "amMemory": 2000,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ expected = {
+ "capacity-scheduler": {
+ "properties": {
+ "yarn.scheduler.capacity.root.queues": "queue1,queue2"
+ }
+ },
+ "tez-site": {
+ "properties": {
+ "tez.task.resource.memory.mb": "768",
+ "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.runtime.io.sort.mb": "307",
+ "tez.session.am.dag.submit.timeout.secs": "600",
+ "tez.runtime.unordered.output.buffer.size-mb": "57",
+ "tez.am.resource.memory.mb": "4000"
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "256",
+ "yarn.scheduler.maximum-allocation-mb": "8192"
+ }
+ }
+ }
+ services = {
+ "services": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
+ "StackServices": {
+ "service_name": "YARN",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1",
+ "component_category": "MASTER",
+ "component_name": "APP_TIMELINE_SERVER",
+ "display_name": "App Timeline Server",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "hostnames": [
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1-2",
+ "component_category": "MASTER",
+ "component_name": "RESOURCEMANAGER",
+ "display_name": "ResourceManager",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "CLIENT",
+ "component_name": "YARN_CLIENT",
+ "display_name": "YARN Client",
+ "is_client": "true",
+ "is_master": "false",
+ "hostnames": []
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ ],
+ "configurations": configurations,
+ "changed-configurations": [ ],
+ "ambari-server-properties": {}
+ }
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6402.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6402.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6402.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6403.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6403.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6403.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+
+ self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+ # Test JDK1.7
+ services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
+ self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+ # Test JDK1.8
+ services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.8_44'}
+ expected['tez-site']['properties']['tez.am.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
+ expected['tez-site']['properties']['tez.task.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
+ self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+ # Test JDK1.9
+ services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.9.2_44'}
+ expected['tez-site']['properties']['tez.am.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
+ expected['tez-site']['properties']['tez.task.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
+ self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
def test_validateHiveConfigurations(self):
properties = {"hive_security_authorization": "None",
"hive.exec.orc.default.stripe.size": "8388608",