You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/04/17 20:18:15 UTC
[21/34] ambari git commit: AMBARI-20763. Update YARN's ATS configs
'apptimelineserver_heapsize' and
'yarn.timeline-service.entity-group-fs-store.app-cache-size' logic in 2.6.
AMBARI-20763. Update YARN's ATS configs 'apptimelineserver_heapsize' and 'yarn.timeline-service.entity-group-fs-store.app-cache-size' logic in 2.6.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/38f84bf1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/38f84bf1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/38f84bf1
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 38f84bf12d04981017bbcfc3d44c243f062ed055
Parents: f894e48
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Thu Apr 13 15:57:10 2017 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Thu Apr 13 15:57:10 2017 -0700
----------------------------------------------------------------------
.../YARN/3.0.0.3.0/service_advisor.py | 76 ++++
.../services/YARN/configuration/yarn-env.xml | 18 +
.../stacks/HDP/2.6/services/stack_advisor.py | 80 ++++
.../stacks/2.5/common/test_stack_advisor.py | 7 +
.../stacks/2.6/common/test_stack_advisor.py | 452 ++++++++++++++++++-
5 files changed, 627 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
index fc32001..1ac7849 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
@@ -420,6 +420,7 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
def recommendYARNConfigurationsFromHDP26(self, configurations, clusterData, services, hosts):
putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
+ putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services)
if "yarn-site" in services["configurations"] and \
"yarn.resourcemanager.scheduler.monitor.enable" in services["configurations"]["yarn-site"]["properties"]:
@@ -466,6 +467,81 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
else:
self.logger.info("Not setting Yarn Repo user for Ranger.")
+ yarn_timeline_app_cache_size = None
+ host_mem = None
+ for host in hosts["items"]:
+ host_mem = host["Hosts"]["total_mem"]
+ break
+ # Check if 'yarn.timeline-service.entity-group-fs-store.app-cache-size' in changed configs.
+ changed_configs_has_ats_cache_size = self.isConfigPropertiesChanged(
+ services, "yarn-site", ['yarn.timeline-service.entity-group-fs-store.app-cache-size'], False)
+ # Check if it's : 1. 'apptimelineserver_heapsize' changed detected in changed-configurations)
+ # OR 2. cluster initialization (services['changed-configurations'] should be empty in this case)
+ if changed_configs_has_ats_cache_size:
+ yarn_timeline_app_cache_size = self.read_yarn_apptimelineserver_cache_size(services)
+ elif 0 == len(services['changed-configurations']):
+ # Fetch host memory from 1st host, to be used for ATS config calculations below.
+ if host_mem is not None:
+ yarn_timeline_app_cache_size = self.calculate_yarn_apptimelineserver_cache_size(host_mem)
+ putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.app-cache-size', yarn_timeline_app_cache_size)
+ self.logger.info("Updated YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size' as : {0}, "
+ "using 'host_mem' = {1}".format(yarn_timeline_app_cache_size, host_mem))
+ else:
+ self.logger.info("Couldn't update YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size' as "
+ "'host_mem' read = {0}".format(host_mem))
+
+ if yarn_timeline_app_cache_size is not None:
+ # Calculation for 'ats_heapsize' is in MB.
+ ats_heapsize = self.calculate_yarn_apptimelineserver_heapsize(host_mem, yarn_timeline_app_cache_size)
+ putYarnEnvProperty('apptimelineserver_heapsize', ats_heapsize) # Value in MB
+ self.logger.info("Updated YARN config 'apptimelineserver_heapsize' as : {0}, ".format(ats_heapsize))
+
+ """
+ Calculate YARN config 'apptimelineserver_heapsize' in MB.
+ """
+ def calculate_yarn_apptimelineserver_heapsize(self, host_mem, yarn_timeline_app_cache_size):
+ ats_heapsize = None
+ if host_mem < 4096:
+ ats_heapsize = 1024
+ else:
+ ats_heapsize = long(min(math.floor(host_mem/2), long(yarn_timeline_app_cache_size) * 500 + 3072))
+ return ats_heapsize
+
+ """
+ Calculates for YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size', based on YARN's NodeManager size.
+ """
+ def calculate_yarn_apptimelineserver_cache_size(self, host_mem):
+ yarn_timeline_app_cache_size = None
+ if host_mem < 4096:
+ yarn_timeline_app_cache_size = 3
+ elif host_mem >= 4096 and host_mem < 8192:
+ yarn_timeline_app_cache_size = 7
+ elif host_mem >= 8192:
+ yarn_timeline_app_cache_size = 10
+ self.logger.info("Calculated and returning 'yarn_timeline_app_cache_size' : {0}".format(yarn_timeline_app_cache_size))
+ return yarn_timeline_app_cache_size
+
+
+ """
+ Reads YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size'.
+ """
+ def read_yarn_apptimelineserver_cache_size(self, services):
+ """
+ :type services dict
+ :rtype str
+ """
+ yarn_ats_app_cache_size = None
+ yarn_ats_app_cache_size_config = "yarn.timeline-service.entity-group-fs-store.app-cache-size"
+ yarn_site_in_services = self.getServicesSiteProperties(services, "yarn-site")
+
+ if yarn_site_in_services and yarn_ats_app_cache_size_config in yarn_site_in_services:
+ yarn_ats_app_cache_size = yarn_site_in_services[yarn_ats_app_cache_size_config]
+ self.logger.info("'yarn.scheduler.minimum-allocation-mb' read from services as : {0}".format(yarn_ats_app_cache_size))
+
+ if not yarn_ats_app_cache_size:
+ self.logger.error("'{0}' was not found in the services".format(yarn_ats_app_cache_size_config))
+
+ return yarn_ats_app_cache_size
#region LLAP
def updateLlapConfigs(self, configurations, services, hosts, llap_queue_name):
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
index d04c3c5..1b2ca68 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml
@@ -25,6 +25,24 @@
<description>Set to false by default, needs to be set to true in stacks that use Ranger Yarn Plugin</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>apptimelineserver_heapsize</name>
+ <value>1024</value>
+ <display-name>AppTimelineServer Java heap size</display-name>
+ <description>Max heapsize for AppTimelineServer using a numerical value in the scale of MB</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <unit>MB</unit>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.timeline-service.entity-group-fs-store.app-cache-size</name>
+ </property>
+ </depends-on>
+ </property>
<!-- yarn-env.sh -->
<property>
<name>content</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 7881917..38f46d7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -16,6 +16,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+import math
+
import json
import re
from resource_management.libraries.functions import format
@@ -146,6 +148,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
super(HDP26StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
+ putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services)
if "yarn-site" in services["configurations"] and \
"yarn.resourcemanager.scheduler.monitor.enable" in services["configurations"]["yarn-site"]["properties"]:
@@ -186,6 +189,83 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
else:
self.logger.info("Not setting Yarn Repo user for Ranger.")
+
+ yarn_timeline_app_cache_size = None
+ host_mem = None
+ for host in hosts["items"]:
+ host_mem = host["Hosts"]["total_mem"]
+ break
+ # Check if 'yarn.timeline-service.entity-group-fs-store.app-cache-size' in changed configs.
+ changed_configs_has_ats_cache_size = self.isConfigPropertiesChanged(
+ services, "yarn-site", ['yarn.timeline-service.entity-group-fs-store.app-cache-size'], False)
+ # Check if it's : 1. 'apptimelineserver_heapsize' changed detected in changed-configurations)
+ # OR 2. cluster initialization (services['changed-configurations'] should be empty in this case)
+ if changed_configs_has_ats_cache_size:
+ yarn_timeline_app_cache_size = self.read_yarn_apptimelineserver_cache_size(services)
+ elif 0 == len(services['changed-configurations']):
+ # Fetch host memory from 1st host, to be used for ATS config calculations below.
+ if host_mem is not None:
+ yarn_timeline_app_cache_size = self.calculate_yarn_apptimelineserver_cache_size(host_mem)
+ putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.app-cache-size', yarn_timeline_app_cache_size)
+ self.logger.info("Updated YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size' as : {0}, "
+ "using 'host_mem' = {1}".format(yarn_timeline_app_cache_size, host_mem))
+ else:
+ self.logger.info("Couldn't update YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size' as "
+ "'host_mem' read = {0}".format(host_mem))
+
+ if yarn_timeline_app_cache_size is not None:
+ # Calculation for 'ats_heapsize' is in MB.
+ ats_heapsize = self.calculate_yarn_apptimelineserver_heapsize(host_mem, yarn_timeline_app_cache_size)
+ putYarnEnvProperty('apptimelineserver_heapsize', ats_heapsize) # Value in MB
+ self.logger.info("Updated YARN config 'apptimelineserver_heapsize' as : {0}, ".format(ats_heapsize))
+
+ """
+ Calculate YARN config 'apptimelineserver_heapsize' in MB.
+ """
+ def calculate_yarn_apptimelineserver_heapsize(self, host_mem, yarn_timeline_app_cache_size):
+ ats_heapsize = None
+ if host_mem < 4096:
+ ats_heapsize = 1024
+ else:
+ ats_heapsize = long(min(math.floor(host_mem/2), long(yarn_timeline_app_cache_size) * 500 + 3072))
+ return ats_heapsize
+
+ """
+ Calculates for YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size', based on YARN's NodeManager size.
+ """
+ def calculate_yarn_apptimelineserver_cache_size(self, host_mem):
+ yarn_timeline_app_cache_size = None
+ if host_mem < 4096:
+ yarn_timeline_app_cache_size = 3
+ elif host_mem >= 4096 and host_mem < 8192:
+ yarn_timeline_app_cache_size = 7
+ elif host_mem >= 8192:
+ yarn_timeline_app_cache_size = 10
+ self.logger.info("Calculated and returning 'yarn_timeline_app_cache_size' : {0}".format(yarn_timeline_app_cache_size))
+ return yarn_timeline_app_cache_size
+
+
+ """
+ Reads YARN config 'yarn.timeline-service.entity-group-fs-store.app-cache-size'.
+ """
+ def read_yarn_apptimelineserver_cache_size(self, services):
+ """
+ :type services dict
+ :rtype str
+ """
+ yarn_ats_app_cache_size = None
+ yarn_ats_app_cache_size_config = "yarn.timeline-service.entity-group-fs-store.app-cache-size"
+ yarn_site_in_services = self.getServicesSiteProperties(services, "yarn-site")
+
+ if yarn_site_in_services and yarn_ats_app_cache_size_config in yarn_site_in_services:
+ yarn_ats_app_cache_size = yarn_site_in_services[yarn_ats_app_cache_size_config]
+ self.logger.info("'yarn.scheduler.minimum-allocation-mb' read from services as : {0}".format(yarn_ats_app_cache_size))
+
+ if not yarn_ats_app_cache_size:
+ self.logger.error("'{0}' was not found in the services".format(yarn_ats_app_cache_size_config))
+
+ return yarn_ats_app_cache_size
+
def getMetadataConnectionString(self, database_type):
driverDict = {
'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true',
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 4250681..77a06fe 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -926,6 +926,8 @@ class TestHDP25StackAdvisor(TestCase):
]
}
],
+ "changed-configurations": [
+ ],
"configurations": {
"capacity-scheduler": {
"properties": {
@@ -4918,6 +4920,9 @@ class TestHDP25StackAdvisor(TestCase):
"capacity-scheduler":{"properties":{
"capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue,leaf\n" +
"yarn.scheduler.capacity.root.ndfqueue.queues=ndfqueue1,ndfqueue2\n"}}}
+ services["changed-configurations"]= []
+
+
hosts = self.prepareHosts([])
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
@@ -4948,6 +4953,8 @@ class TestHDP25StackAdvisor(TestCase):
"stack_versions": ["2.4", "2.3", "2.2", "2.1", "2.0.6"]
}
},
+ "changed-configurations": [
+ ],
"configurations": configurations,
"services": [],
"ambari-server-properties": {}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f84bf1/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
index 5bfa1a9..2d7322d 100644
--- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -989,7 +989,26 @@ class TestHDP26StackAdvisor(TestCase):
"yarnMinContainerSize": 256
}
- hosts = {}
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 50331648,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
services = {
"services":
@@ -1007,6 +1026,8 @@ class TestHDP26StackAdvisor(TestCase):
"stack_name" : "HDP",
"stack_version": "2.6"
},
+ "changed-configurations": [
+ ],
"configurations": configurations,
"ambari-server-properties": {"ambari-server.user":"ambari_user"}
}
@@ -1016,6 +1037,7 @@ class TestHDP26StackAdvisor(TestCase):
'yarn-env': {
'properties': {
'min_user_id': '500',
+ 'apptimelineserver_heapsize': '8072',
'service_check.queue.name': 'default'
}
},
@@ -1131,11 +1153,16 @@ class TestHDP26StackAdvisor(TestCase):
'yarn.scheduler.minimum-allocation-vcores': '1',
'yarn.scheduler.maximum-allocation-vcores': '4',
'yarn.nodemanager.resource.memory-mb': '768',
+ 'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local',
+ 'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log',
+ 'yarn.timeline-service.entity-group-fs-store.app-cache-size': '10',
'yarn.scheduler.minimum-allocation-mb': '256',
'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
'yarn.nodemanager.resource.cpu-vcores': '4',
'yarn.scheduler.maximum-allocation-mb': '768',
- 'yarn.nodemanager.linux-container-executor.group': 'hadoop'
+ 'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+ 'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline',
+ 'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline'
},
'property_attributes': {
'yarn.authorization-provider': {
@@ -1267,6 +1294,8 @@ class TestHDP26StackAdvisor(TestCase):
"components": []
}
],
+ "changed-configurations": [
+ ],
"configurations": configurations
}
@@ -1282,7 +1311,8 @@ class TestHDP26StackAdvisor(TestCase):
'properties': {
'yarn_user': 'custom_yarn',
'service_check.queue.name': 'default',
- 'min_user_id': '500'
+ 'min_user_id': '500',
+ 'apptimelineserver_heapsize': '2048'
}
},
'ranger-yarn-plugin-properties': {
@@ -1304,19 +1334,429 @@ class TestHDP26StackAdvisor(TestCase):
'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
'yarn.nodemanager.resource.cpu-vcores': '4',
'yarn.scheduler.maximum-allocation-mb': '1280',
- 'yarn.nodemanager.linux-container-executor.group': 'hadoop'
+ 'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+ 'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local',
+ 'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log',
+ 'yarn.timeline-service.entity-group-fs-store.app-cache-size': '7',
+ 'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline',
+ 'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline'
+
}
}
}
- self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None)
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 4096,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
+
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+
configurations['yarn-env']['properties']['yarn_user'] = 'yarn'
expected['yarn-env']['properties']['yarn_user'] = 'yarn'
expected['ranger-yarn-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'yarn'
- self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None)
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+
+
+ def test_recommendYARNConfigurations_for_ats_heapsize_and_cache(self):
+ configurations = {
+ "yarn-env": {
+ "properties": {
+ "yarn_user" : "custom_yarn"
+ }
+ },
+ "ranger-yarn-plugin-properties": {
+ "properties": {
+ "ranger-yarn-plugin-enabled" : "Yes",
+ "REPOSITORY_CONFIG_USERNAME":"yarn"
+ }
+ }
+ }
+ services = {
+ "services" : [{
+ "StackServices": {
+ "service_name" : "YARN",
+ "service_version" : "2.7.3.2.6"
+ },
+ "components": []
+ }
+ ],
+ "changed-configurations": [
+ ],
+ "configurations": configurations
+ }
+
+
+ clusterData = {
+ "cpu": 4,
+ "containers" : 5,
+ "ramPerContainer": 256,
+ "yarnMinContainerSize": 256
+ }
+ expected = {
+ 'yarn-env': {
+ 'properties': {
+ 'yarn_user': 'custom_yarn',
+ 'service_check.queue.name': 'default',
+ 'min_user_id': '500',
+ 'apptimelineserver_heapsize': '1024'
+ }
+ },
+ 'ranger-yarn-plugin-properties': {
+ 'properties': {
+ 'ranger-yarn-plugin-enabled': 'Yes',
+ 'REPOSITORY_CONFIG_USERNAME': 'custom_yarn'
+ }
+ },
+ 'yarn-site': {
+ 'properties': {
+ 'hadoop.registry.rm.enabled': 'false',
+ 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes': '',
+ 'yarn.authorization-provider': 'org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer',
+ 'yarn.acl.enable': 'true',
+ 'yarn.scheduler.minimum-allocation-vcores': '1',
+ 'yarn.scheduler.maximum-allocation-vcores': '4',
+ 'yarn.nodemanager.resource.memory-mb': '1280',
+ 'yarn.scheduler.minimum-allocation-mb': '256',
+ 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '',
+ 'yarn.nodemanager.resource.cpu-vcores': '4',
+ 'yarn.scheduler.maximum-allocation-mb': '1280',
+ 'yarn.nodemanager.linux-container-executor.group': 'hadoop',
+ 'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local',
+ 'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log',
+ 'yarn.timeline-service.entity-group-fs-store.app-cache-size': '3',
+ 'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline',
+ 'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline'
+
+ }
+ }
+ }
+
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 2048,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
+
+
+
+
+ '''
+ Test 1 :
+ I/P:
+ - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size')
+ - 'host_mem' = 2048
+ O/P :
+ - Config value recommended for:
+ - yarn.timeline-service.entity-group-fs-store.app-cache-size = 3
+ - apptimelineserver_heapsize = 1024
+ '''
+
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+
+
+ '''
+ Test 2 :
+ I/P:
+ - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size')
+ - 'host_mem' = 4096
+ O/P :
+ - Config value recommended for:
+ - yarn.timeline-service.entity-group-fs-store.app-cache-size = 7
+ - apptimelineserver_heapsize = 2048
+ '''
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 4096,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
+
+ expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '2048'
+ expected['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.app-cache-size'] = '7'
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+
+
+ '''
+ Test 3 :
+ I/P:
+ - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size')
+ - 'host_mem' = 8192
+ O/P :
+ - Config value recommended for:
+ - yarn.timeline-service.entity-group-fs-store.app-cache-size = 10
+ - apptimelineserver_heapsize = 4096
+ '''
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 8192,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
+
+ expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '4096'
+ expected['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.app-cache-size'] = '10'
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+
+
+ '''
+ Test 4 :
+ I/P:
+ - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+ - 'host_mem' = 2048
+ O/P :
+ - Config value recommended for:
+ - apptimelineserver_heapsize = 4096
+ '''
+
+ services["changed-configurations"] = [
+ {
+ u'old_value': u'10',
+ u'type': u'yarn-site',
+ u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+ }
+ ]
+
+ services["configurations"] = {
+ "yarn-env": {
+ "properties": {
+ "yarn_user" : "custom_yarn",
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "7"
+ }
+ },
+ "ranger-yarn-plugin-properties": {
+ "properties": {
+ "ranger-yarn-plugin-enabled" : "Yes",
+ "REPOSITORY_CONFIG_USERNAME":"yarn"
+ }
+ }
+ }
+
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 4096,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
+
+
+
+ '''
+ Test 5 :
+ I/P:
+ - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+ - 'host_mem' = 4096
+ O/P :
+ - Config value recommended for:
+ - apptimelineserver_heapsize = 2048
+ '''
+
+ services["changed-configurations"] = [
+ {
+ u'old_value': u'10',
+ u'type': u'yarn-site',
+ u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+ }
+ ]
+
+ services["configurations"] = {
+ "yarn-env": {
+ "properties": {
+ "yarn_user" : "custom_yarn",
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "7"
+ }
+ },
+ "ranger-yarn-plugin-properties": {
+ "properties": {
+ "ranger-yarn-plugin-enabled" : "Yes",
+ "REPOSITORY_CONFIG_USERNAME":"yarn"
+ }
+ }
+ }
+
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 4096,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
+
+
+ expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '2048'
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+
+
+ '''
+ Test 6 :
+ I/P:
+ - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+ - 'host_mem' = 8196
+ O/P :
+ - Config value recommended for:
+ - Shouldn't have yarn.timeline-service.entity-group-fs-store.app-cache-size
+ - apptimelineserver_heapsize = 4572
+ '''
+
+ services["changed-configurations"] = [
+ {
+ u'old_value': u'10',
+ u'type': u'yarn-site',
+ u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size'
+ }
+ ]
+
+ services["configurations"] = {
+ "yarn-env": {
+ "properties": {
+ "yarn_user" : "custom_yarn",
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "3"
+ }
+ },
+ "ranger-yarn-plugin-properties": {
+ "properties": {
+ "ranger-yarn-plugin-enabled" : "Yes",
+ "REPOSITORY_CONFIG_USERNAME":"yarn"
+ }
+ }
+ }
+
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 6,
+ "total_mem": 16392,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
+
+
+ expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '4572'
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
+
def test_recommendKAFKAConfigurations(self):
configurations = {
"kafka-env": {