You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/06/18 17:02:09 UTC
[1/2] ambari git commit: AMBARI-11993. Tez, YARN,
MR2 checks failed on 2.2 cluster (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 158f2e13d -> 73b7d7273
refs/heads/trunk 4fd35351b -> 47f925eff
AMBARI-11993. Tez, YARN, MR2 checks failed on 2.2 cluster (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/47f925ef
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/47f925ef
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/47f925ef
Branch: refs/heads/trunk
Commit: 47f925efff2ddfa1d5483aeaf8b25d042ea43879
Parents: 4fd3535
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Jun 18 18:01:56 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Jun 18 18:01:56 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 2 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 2 +-
.../stacks/HDPWIN/2.1/services/stack_advisor.py | 2 +-
.../stacks/HDPWIN/2.2/services/stack_advisor.py | 4 +--
.../stacks/2.0.6/common/test_stack_advisor.py | 26 ++++++++++----------
.../stacks/2.2/common/test_stack_advisor.py | 3 ++-
6 files changed, 20 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index d9da407..f12524a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -306,7 +306,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
totalAvailableRam = cluster["ram"] - cluster["reservedRam"]
if cluster["hBaseInstalled"]:
totalAvailableRam -= cluster["hbaseRam"]
- cluster["totalAvailableRam"] = max(2048, totalAvailableRam * 1024)
+ cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024)
'''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))'''
cluster["containers"] = round(max(3,
min(2 * cluster["cpu"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index bdbe9b1..9de8421 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -638,7 +638,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
min_mapreduce_map_memory_mb = 0
min_mapreduce_reduce_memory_mb = 0
min_mapreduce_map_java_opts = 0
- if ("PIG" in servicesList):
+ if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 4096:
min_mapreduce_map_memory_mb = 1536
min_mapreduce_reduce_memory_mb = 1536
min_mapreduce_map_java_opts = 1024
http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
index b8334e3..75b427e 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
@@ -292,7 +292,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
totalAvailableRam = cluster["ram"] - cluster["reservedRam"]
if cluster["hBaseInstalled"]:
totalAvailableRam -= cluster["hbaseRam"]
- cluster["totalAvailableRam"] = max(2048, totalAvailableRam * 1024)
+ cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024)
'''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))'''
cluster["containers"] = round(max(3,
min(2 * cluster["cpu"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
index 6d527ea..c6941f0 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
@@ -236,7 +236,7 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
min_mapreduce_map_memory_mb = 0
min_mapreduce_reduce_memory_mb = 0
min_mapreduce_map_java_opts = 0
- if ("PIG" in servicesList):
+ if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 8192:
min_mapreduce_map_memory_mb = 1536
min_mapreduce_reduce_memory_mb = 1536
min_mapreduce_map_java_opts = 1024
@@ -641,7 +641,7 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
min_mapreduce_map_memory_mb = 0
min_mapreduce_map_java_opts = 0
- if ("PIG" in servicesList):
+ if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 8192:
min_mapreduce_map_memory_mb = 1500
min_mapreduce_map_java_opts = 1024
putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 00622a7..65fa78a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -36,7 +36,7 @@ class TestHDP206StackAdvisor(TestCase):
stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
self.stackAdvisor = clazz()
-
+ self.maxDiff = None
# substitute method in the instance
self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid
self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic
@@ -201,7 +201,7 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {"message": "Value is less than the recommended default of 2048", "level": "WARN"},
+ {"message": "Value is less than the recommended default of 512", "level": "WARN"},
{"message": "Value should be integer", "level": "ERROR"},
{"message": "Value should be set", "level": "ERROR"}
]
@@ -407,15 +407,15 @@ class TestHDP206StackAdvisor(TestCase):
})
expected["referenceHost"] = hosts["items"][1]["Hosts"]
expected["referenceNodeManagerHost"] = hosts["items"][1]["Hosts"]
- expected["amMemory"] = 256
- expected["containers"] = 8
+ expected["amMemory"] = 170.66666666666666
+ expected["containers"] = 3.0
expected["cpu"] = 4
- expected["totalAvailableRam"] = 2048
- expected["mapMemory"] = 256
+ expected["totalAvailableRam"] = 512
+ expected["mapMemory"] = 170
expected["minContainerSize"] = 256
- expected["reduceMemory"] = 256
+ expected["reduceMemory"] = 170.66666666666666
expected["ram"] = 0
- expected["ramPerContainer"] = 256
+ expected["ramPerContainer"] = 170.66666666666666
expected["reservedRam"] = 1
result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
self.assertEquals(result, expected)
@@ -529,12 +529,12 @@ class TestHDP206StackAdvisor(TestCase):
"reservedRam": 1,
"hbaseRam": 1,
"minContainerSize": 256,
- "totalAvailableRam": 2048,
+ "totalAvailableRam": 512,
"containers": 3,
- "ramPerContainer": 682.6666666666666,
- "mapMemory": 682,
- "reduceMemory": 682.6666666666666,
- "amMemory": 682.6666666666666
+ "ramPerContainer": 170.66666666666666,
+ "mapMemory": 170,
+ "reduceMemory": 170.66666666666666,
+ "amMemory": 170.66666666666666
}
self.assertEquals(result, expected)
http://git-wip-us.apache.org/repos/asf/ambari/blob/47f925ef/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index a984a7a..78acce3 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1265,7 +1265,8 @@ class TestHDP22StackAdvisor(TestCase):
clusterData = {
"cpu": 4,
"containers" : 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "totalAvailableRam": 4096,
}
expected = {
"yarn-env": {
[2/2] ambari git commit: AMBARI-11993. Tez, YARN,
MR2 checks failed on 2.2 cluster (aonishuk)
Posted by ao...@apache.org.
AMBARI-11993. Tez, YARN, MR2 checks failed on 2.2 cluster (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/73b7d727
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/73b7d727
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/73b7d727
Branch: refs/heads/branch-2.1
Commit: 73b7d72734858becc5426defbfdc660ff532a44b
Parents: 158f2e1
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Jun 18 18:01:58 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Jun 18 18:01:58 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 2 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 2 +-
.../stacks/HDPWIN/2.1/services/stack_advisor.py | 2 +-
.../stacks/HDPWIN/2.2/services/stack_advisor.py | 4 +--
.../stacks/2.0.6/common/test_stack_advisor.py | 26 ++++++++++----------
.../stacks/2.2/common/test_stack_advisor.py | 3 ++-
6 files changed, 20 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/73b7d727/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index d9da407..f12524a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -306,7 +306,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
totalAvailableRam = cluster["ram"] - cluster["reservedRam"]
if cluster["hBaseInstalled"]:
totalAvailableRam -= cluster["hbaseRam"]
- cluster["totalAvailableRam"] = max(2048, totalAvailableRam * 1024)
+ cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024)
'''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))'''
cluster["containers"] = round(max(3,
min(2 * cluster["cpu"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/73b7d727/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index bdbe9b1..9de8421 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -638,7 +638,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
min_mapreduce_map_memory_mb = 0
min_mapreduce_reduce_memory_mb = 0
min_mapreduce_map_java_opts = 0
- if ("PIG" in servicesList):
+ if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 4096:
min_mapreduce_map_memory_mb = 1536
min_mapreduce_reduce_memory_mb = 1536
min_mapreduce_map_java_opts = 1024
http://git-wip-us.apache.org/repos/asf/ambari/blob/73b7d727/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
index b8334e3..75b427e 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
@@ -292,7 +292,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
totalAvailableRam = cluster["ram"] - cluster["reservedRam"]
if cluster["hBaseInstalled"]:
totalAvailableRam -= cluster["hbaseRam"]
- cluster["totalAvailableRam"] = max(2048, totalAvailableRam * 1024)
+ cluster["totalAvailableRam"] = max(512, totalAvailableRam * 1024)
'''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))'''
cluster["containers"] = round(max(3,
min(2 * cluster["cpu"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/73b7d727/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
index 6d527ea..c6941f0 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
@@ -236,7 +236,7 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
min_mapreduce_map_memory_mb = 0
min_mapreduce_reduce_memory_mb = 0
min_mapreduce_map_java_opts = 0
- if ("PIG" in servicesList):
+ if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 8192:
min_mapreduce_map_memory_mb = 1536
min_mapreduce_reduce_memory_mb = 1536
min_mapreduce_map_java_opts = 1024
@@ -641,7 +641,7 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
min_mapreduce_map_memory_mb = 0
min_mapreduce_map_java_opts = 0
- if ("PIG" in servicesList):
+ if ("PIG" in servicesList) and clusterData["totalAvailableRam"] >= 8192:
min_mapreduce_map_memory_mb = 1500
min_mapreduce_map_java_opts = 1024
putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
http://git-wip-us.apache.org/repos/asf/ambari/blob/73b7d727/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 00622a7..65fa78a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -36,7 +36,7 @@ class TestHDP206StackAdvisor(TestCase):
stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
self.stackAdvisor = clazz()
-
+ self.maxDiff = None
# substitute method in the instance
self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid
self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic
@@ -201,7 +201,7 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {"message": "Value is less than the recommended default of 2048", "level": "WARN"},
+ {"message": "Value is less than the recommended default of 512", "level": "WARN"},
{"message": "Value should be integer", "level": "ERROR"},
{"message": "Value should be set", "level": "ERROR"}
]
@@ -407,15 +407,15 @@ class TestHDP206StackAdvisor(TestCase):
})
expected["referenceHost"] = hosts["items"][1]["Hosts"]
expected["referenceNodeManagerHost"] = hosts["items"][1]["Hosts"]
- expected["amMemory"] = 256
- expected["containers"] = 8
+ expected["amMemory"] = 170.66666666666666
+ expected["containers"] = 3.0
expected["cpu"] = 4
- expected["totalAvailableRam"] = 2048
- expected["mapMemory"] = 256
+ expected["totalAvailableRam"] = 512
+ expected["mapMemory"] = 170
expected["minContainerSize"] = 256
- expected["reduceMemory"] = 256
+ expected["reduceMemory"] = 170.66666666666666
expected["ram"] = 0
- expected["ramPerContainer"] = 256
+ expected["ramPerContainer"] = 170.66666666666666
expected["reservedRam"] = 1
result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
self.assertEquals(result, expected)
@@ -529,12 +529,12 @@ class TestHDP206StackAdvisor(TestCase):
"reservedRam": 1,
"hbaseRam": 1,
"minContainerSize": 256,
- "totalAvailableRam": 2048,
+ "totalAvailableRam": 512,
"containers": 3,
- "ramPerContainer": 682.6666666666666,
- "mapMemory": 682,
- "reduceMemory": 682.6666666666666,
- "amMemory": 682.6666666666666
+ "ramPerContainer": 170.66666666666666,
+ "mapMemory": 170,
+ "reduceMemory": 170.66666666666666,
+ "amMemory": 170.66666666666666
}
self.assertEquals(result, expected)
http://git-wip-us.apache.org/repos/asf/ambari/blob/73b7d727/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index a984a7a..78acce3 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1265,7 +1265,8 @@ class TestHDP22StackAdvisor(TestCase):
clusterData = {
"cpu": 4,
"containers" : 5,
- "ramPerContainer": 256
+ "ramPerContainer": 256,
+ "totalAvailableRam": 4096,
}
expected = {
"yarn-env": {