You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/06/07 16:07:31 UTC
ambari git commit: AMBARI-11767 Configs: hive warning when using the
default values during install (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 2fc88fa76 -> 21ddf8b86
AMBARI-11767 Configs: hive warning when using the default values during install (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/21ddf8b8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/21ddf8b8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/21ddf8b8
Branch: refs/heads/trunk
Commit: 21ddf8b86af187a1fc8d27b11c500604dc61de55
Parents: 2fc88fa
Author: Dmytro Sen <ds...@apache.org>
Authored: Sun Jun 7 17:06:59 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Sun Jun 7 17:07:08 2015 +0300
----------------------------------------------------------------------
.../common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml | 2 +-
.../stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml | 2 +-
.../src/main/resources/stacks/HDP/2.1/services/stack_advisor.py | 3 ++-
.../stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml | 2 +-
.../src/main/resources/stacks/HDP/2.2/services/stack_advisor.py | 2 +-
.../src/test/python/stacks/2.1/common/test_stack_advisor.py | 5 +++--
6 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/21ddf8b8/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index 82eafde..f4e4b25 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -255,7 +255,7 @@ limitations under the License.
<value>2147483648</value>
<description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it
is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly
- converted to a mapjoin(there is no conditional task). The default is 10MB.
+ converted to a mapjoin(there is no conditional task).
</description>
<display-name>For Map Join, per Map memory threshold</display-name>
<value-attributes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/21ddf8b8/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
index f0b5a04..182acfe 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
@@ -253,7 +253,7 @@ limitations under the License.
<value>1000000000</value>
<description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it
is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly
- converted to a mapjoin(there is no conditional task). The default is 10MB.
+ converted to a mapjoin(there is no conditional task).
</description>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/21ddf8b8/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 2c80263..44595be 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -40,8 +40,9 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
def recommendHiveConfigurations(self, configurations, clusterData, services, hosts):
containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
+ container_size_bytes = int(containerSize)*1024*1024
putHiveProperty = self.putProperty(configurations, "hive-site", services)
- putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(round(containerSize / 3)) * 1048576)
+ putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(round(container_size_bytes / 3)))
putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(round((0.8 * containerSize) + 0.5)))
+ "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps")
putHiveProperty('hive.tez.container.size', containerSize)
http://git-wip-us.apache.org/repos/asf/ambari/blob/21ddf8b8/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
index 5e5c64f..5f24e7c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
@@ -1407,7 +1407,7 @@ limitations under the License.
<value>52428800</value>
<description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it
is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly
- converted to a mapjoin(there is no conditional task). The default is 10MB.
+ converted to a mapjoin(there is no conditional task).
</description>
<display-name>For Map Join, per Map memory threshold</display-name>
<value-attributes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/21ddf8b8/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 5839149..36a6b0b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -330,7 +330,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
container_size = configurations["hive-site"]["properties"]["hive.tez.container.size"]
container_size_bytes = int(container_size)*1024*1024
# Memory
- putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(container_size_bytes/3))
+ putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(round(container_size_bytes/3)))
putHiveSitePropertyAttribute("hive.auto.convert.join.noconditionaltask.size", "maximum", container_size_bytes)
putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864")
http://git-wip-us.apache.org/repos/asf/ambari/blob/21ddf8b8/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
index 9473785..3728d8e 100644
--- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
@@ -79,7 +79,7 @@ class TestHDP21StackAdvisor(TestCase):
expected = {
"hive-site": {
"properties": {
- "hive.auto.convert.join.noconditionaltask.size": "718274560",
+ "hive.auto.convert.join.noconditionaltask.size": "718624085",
"hive.tez.java.opts": "-server -Xmx1645m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps",
"hive.tez.container.size": "2056"
}
@@ -87,6 +87,7 @@ class TestHDP21StackAdvisor(TestCase):
}
self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, None, None)
+ self.maxDiff = None
self.assertEquals(configurations, expected)
def test_recommendHiveConfigurations_mapMemoryMoreThan2048(self):
@@ -272,4 +273,4 @@ class TestHDP21StackAdvisor(TestCase):
]
res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations, '', '')
- self.assertEquals(res, res_expected)
\ No newline at end of file
+ self.assertEquals(res, res_expected)