You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2015/12/01 19:19:59 UTC
[09/50] ambari git commit: AMBARI-14099. Tez config relationship
dependency not taking effect (aonishuk)
AMBARI-14099. Tez config relationship dependency not taking effect (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/01f7d709
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/01f7d709
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/01f7d709
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 01f7d709e8f49b3d2095ebb4de6c269e4aad1230
Parents: ebdda3b
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Nov 27 16:51:28 2015 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Nov 27 16:51:28 2015 +0200
----------------------------------------------------------------------
.../HDP/2.2/services/TEZ/configuration/tez-site.xml | 12 ++++++++++++
.../resources/stacks/HDP/2.2/services/stack_advisor.py | 3 ++-
.../resources/stacks/HDP/2.3/services/stack_advisor.py | 3 +--
3 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/01f7d709/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
index 33920d6..3a04f70 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
@@ -304,12 +304,24 @@
<name>tez.runtime.io.sort.mb</name>
<value>512</value>
<description>The size of the sort buffer when output needs to be sorted</description>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.task.resource.memory.mb</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>tez.runtime.unordered.output.buffer.size-mb</name>
<value>100</value>
<description>The size of the buffer when output does not require to be sorted</description>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.task.resource.memory.mb</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/01f7d709/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index c9b3aa3..a33be1e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -744,12 +744,13 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
#properties below should be always present as they are provided in HDP206 stack advisor
yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
- putTezProperty = self.putProperty(configurations, "tez-site")
+ putTezProperty = self.putProperty(configurations, "tez-site", services)
putTezProperty("tez.am.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory'])))
taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory, yarnMaxAllocationSize)
putTezProperty("tez.task.resource.memory.mb", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), taskResourceMemory))
+ taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory * 0.4), 2047))
putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075))
putTezProperty("tez.session.am.dag.submit.timeout.secs", "600")
http://git-wip-us.apache.org/repos/asf/ambari/blob/01f7d709/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 419fb3a..a5e419a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -128,8 +128,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
putTezProperty = self.putProperty(configurations, "tez-site")
# remove 2gb limit for tez.runtime.io.sort.mb
# in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case comment calculation code below
- taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
- taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory)
+ taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4))
if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]: