You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2017/01/06 17:56:34 UTC
ambari git commit: AMBARI-19404 Warning by default by stack advisor
(dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk ebabac069 -> 9bcd3f378
AMBARI-19404 Warning by default by stack advisor (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9bcd3f37
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9bcd3f37
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9bcd3f37
Branch: refs/heads/trunk
Commit: 9bcd3f3784d985da6b5515aa6ec7ece37a3329b0
Parents: ebabac0
Author: Dmytro Sen <ds...@apache.org>
Authored: Fri Jan 6 19:56:27 2017 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Fri Jan 6 19:56:27 2017 +0200
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 10 ++--
.../stacks/HDP/2.2/services/stack_advisor.py | 11 ----
.../stacks/2.0.6/common/test_stack_advisor.py | 54 ++++++++++++++++++++
.../stacks/2.2/common/test_stack_advisor.py | 27 ++++++++--
4 files changed, 83 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bcd3f37/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index b85ddf8..62f4e3e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -456,12 +456,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
self.updateMountProperties("hdfs-site", hdfs_mount_properties, configurations, services, hosts)
- if hdfsSiteProperties and "dfs.datanode.data.dir" in hdfsSiteProperties and\
- hdfsSiteProperties["dfs.datanode.data.dir"] is not None:
+ if configurations and "hdfs-site" in configurations and \
+ "dfs.datanode.data.dir" in configurations["hdfs-site"]["properties"] and \
+ configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"] is not None:
+ dataDirs = configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"].split(",")
+ elif hdfsSiteProperties and "dfs.datanode.data.dir" in hdfsSiteProperties and \
+ hdfsSiteProperties["dfs.datanode.data.dir"] is not None:
dataDirs = hdfsSiteProperties["dfs.datanode.data.dir"].split(",")
- else:
- dataDirs = configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"].split(",")
# dfs.datanode.du.reserved should be set to 10-15% of volume size
# For each host selects maximum size of the volume. Then gets minimum for all hosts.
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bcd3f37/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index b61b0ab..63e4b20 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -100,17 +100,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
- # Above is the default calculated 'maximum' values derived purely from hosts.
- # However, there are 'maximum' and other attributes that actually change based on the values
- # of other configs. We need to update those values.
- if ("yarn-site" in services["configurations"]):
- if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]):
- putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
- putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
- if ("yarn.nodemanager.resource.cpu-vcores" in services["configurations"]["yarn-site"]["properties"]):
- putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
- putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
-
kerberos_authentication_enabled = self.isSecurityEnabled(services)
if kerberos_authentication_enabled:
putYarnProperty('yarn.nodemanager.container-executor.class',
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bcd3f37/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index ca697c0..ab7d53b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -1513,6 +1513,60 @@ class TestHDP206StackAdvisor(TestCase):
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts)
self.assertEquals(configurations, expected)
+ hosts = {
+ "items": [
+ {
+ "href": "/api/v1/hosts/host1",
+ "Hosts": {
+ "cpu_count": 1,
+ "host_name": "c6401.ambari.apache.org",
+ "os_arch": "x86_64",
+ "os_type": "centos6",
+ "ph_cpu_count": 1,
+ "public_host_name": "public.c6401.ambari.apache.org",
+ "rack_info": "/default-rack",
+ "total_mem": 2097152,
+ "disk_info": [{
+ "available" : "21039512",
+ "device" : "/dev/vda1",
+ "used" : "3316924",
+ "percent" : "14%",
+ "size" : "25666616",
+ "type" : "ext4",
+ "mountpoint" : "/"
+ },
+ {
+ "available" : "244732200",
+ "device" : "/dev/vdb",
+ "used" : "60508",
+ "percent" : "1%",
+ "size" : "257899908",
+ "type" : "ext4",
+ "mountpoint" : "/grid/0"
+ }]
+ }
+ }
+ ]}
+
+ expected["hdfs-site"] = {
+ 'properties': {
+ 'dfs.datanode.du.reserved': '33011188224',
+ 'dfs.internal.nameservices': 'mycluster',
+ 'dfs.ha.namenodes.mycluster': 'nn1,nn2',
+ 'dfs.datanode.data.dir': '/hadoop/hdfs/data,/grid/0/hadoop/hdfs/data',
+ 'dfs.namenode.name.dir': '/hadoop/hdfs/namenode,/grid/0/hadoop/hdfs/namenode',
+ 'dfs.namenode.checkpoint.dir': '/hadoop/hdfs/namesecondary',
+ },
+ 'property_attributes': {
+ 'dfs.namenode.rpc-address': {
+ 'delete': 'true'
+ }
+ }
+ }
+ self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts)
+ self.assertEquals(configurations, expected)
+
+
def test_recommendAmsConfigurations(self):
configurations = {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bcd3f37/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 1c9238e..2a61b34 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1168,17 +1168,36 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
- # Test - with no 'changed-configurations', we should get updated 'maximum's.
services.pop("changed-configurations", None)
+ services["changed-configurations"] = [{
+ "type": "yarn-site",
+ "name": "yarn.nodemanager.resource.memory-mb",
+ "old_value": "1280"
+ }]
services.pop("configurations", None)
- services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}}
- expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9'
- expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9'
+ services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321'}}}
+
+ expected["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] = '4321'
+ expected["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"] = '4321'
expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321'
expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-mb"]["maximum"] = '4321'
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+ services["changed-configurations"].append({
+ "type": "yarn-site",
+ "name": "yarn.nodemanager.resource.cpu-vcores",
+ "old_value": "7"
+ })
+ services.pop("configurations", None)
+ services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.cpu-vcores": '9', "yarn.nodemanager.resource.memory-mb": '4321'}}}
+ expected["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"] = '9'
+ expected["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-vcores"] = '9'
+ expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9'
+ expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9'
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
def test_multipleDependsOn(self):
configurations = {
"yarn-env": {