You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/04/29 02:31:13 UTC
ambari git commit: AMBARI-10810. Make Hive theme changes to reduce
number of configs (srimanth)
Repository: ambari
Updated Branches:
refs/heads/trunk d0290c095 -> 132cf31bc
AMBARI-10810. Make Hive theme changes to reduce number of configs (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/132cf31b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/132cf31b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/132cf31b
Branch: refs/heads/trunk
Commit: 132cf31bc4eb797f708b09c388ce06860d598dd6
Parents: d0290c0
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Tue Apr 28 16:35:14 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Tue Apr 28 17:00:37 2015 -0700
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/configuration/hive-site.xml | 10 +-
.../services/HIVE/configuration/hive-site.xml | 16 +-
.../HDP/2.2/services/HIVE/themes/theme.json | 370 ++++---------------
.../stacks/HDP/2.2/services/stack_advisor.py | 2 +-
.../stacks/2.2/common/test_stack_advisor.py | 2 +-
5 files changed, 82 insertions(+), 318 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index 6cfb4ec..8715be2 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -356,21 +356,13 @@ limitations under the License.
<type>value-list</type>
<entries>
<entry>
- <value>ORCFile</value>
+ <value>ORC</value>
<description>The Optimized Row Columnar (ORC) file format provides a highly efficient way to store Hive data. It was designed to overcome limitations of the other Hive file formats. Using ORC files improves performance when Hive is reading, writing, and processing data.</description>
</entry>
<entry>
<value>TextFile</value>
<description>Text file format saves Hive data as normal text.</description>
</entry>
- <entry>
- <value>SequenceFile</value>
- <description>Compressed text file format saves Hive data in CompressedStorage.</description>
- </entry>
- <entry>
- <value>RCfile</value>
- <description>Short of Record Columnar File, are flat files consisting of binary key/value pairs.</description>
- </entry>
</entries>
</value-attributes>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
index 95fd405..4525345 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
@@ -175,7 +175,7 @@ limitations under the License.
<name>hive.exec.reducers.bytes.per.reducer</name>
<value>67108864</value>
<description>size per reducer.The default is 256Mb, i.e if the input size is 1G, it will use 4 reducers.</description>
- <display-name>Memory</display-name>
+ <display-name>Data per Reducer</display-name>
<value-attributes>
<type>int</type>
<minimum>64</minimum>
@@ -1096,7 +1096,7 @@ limitations under the License.
</entry>
<entry>
<value>LDAP</value>
- <label>Lightweight Directory Access Protocol</label>
+ <label>LDAP</label>
</entry>
<entry>
<value>KERBEROS</value>
@@ -1104,7 +1104,11 @@ limitations under the License.
</entry>
<entry>
<value>PAM</value>
- <label>Pluggable Authentication Modules</label>
+ <label>PAM</label>
+ </entry>
+ <entry>
+ <value>CUSTOM</value>
+ <label>Custom</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
@@ -1399,6 +1403,12 @@ limitations under the License.
<unit>B</unit>
<step-increment></step-increment>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-site</type>
+ <name>hive.tez.container.size</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
index efbe83b..c810f6d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
@@ -13,121 +13,83 @@
"tab-rows": 6,
"tab-columns": 3,
"sections": [
- {
- "name": "optimization",
- "display-name": "Optimization",
+ {
+ "name": "acid-transactions",
+ "display-name": "ACID Transactions",
"row-index": "0",
"column-index": "0",
- "row-span": "3",
- "column-span": "2",
- "section-columns": "2",
- "section-rows": "4",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
"subsections": [
{
- "name": "optimization-row1-col1",
- "display-name": "Storage",
- "row-index": "0",
- "column-index": "0",
- "row-span": "2",
- "column-span": "1"
- },
- {
- "name": "optimization-row1-col2-1",
- "display-name": "",
+ "name": "acid-transactions-row1-col1-1",
"row-index": "0",
- "column-index": "1",
- "row-span": "1",
- "column-span": "1"
- },
- {
- "name": "optimization-row1-col2-2",
- "display-name": "",
- "row-index": "1",
- "column-index": "1",
- "row-span": "1",
- "column-span": "1",
- "border": true
- },
- {
- "name": "optimization-row2-col1",
- "display-name": "Optimization Vectorization",
- "row-index": "2",
"column-index": "0",
"row-span": "1",
"column-span": "1"
- },
- {
- "name": "optimization-row2-col2",
- "display-name": "",
- "row-index": "2",
- "column-index": "1",
- "row-span": "1",
- "column-span": "1"
- },
+ }
+ ]
+ },
+ {
+ "name": "interactive-query",
+ "display-name": "Interactive Query",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
{
- "name": "optimization-row3-col1",
- "display-name": "Memory",
- "row-index": "3",
+ "name": "interactive-query-row1-col1",
+ "row-index": "0",
"column-index": "0",
"row-span": "1",
"column-span": "1"
- },
- {
- "name": "optimization-row3-col2",
- "display-name": "",
- "row-index": "3",
- "column-index": "1",
- "row-span": "1",
- "column-span": "1"
}
]
},
{
- "name": "acid-transactions",
- "display-name": "Acid Transactions",
+ "name": "security",
+ "display-name": "Security",
"row-index": "0",
"column-index": "2",
- "row-span": "2",
+ "row-span": "1",
"column-span": "1",
"section-columns": "1",
- "section-rows": "2",
+ "section-rows": "1",
"subsections": [
{
- "name": "acid-transactions-row1-col1-1",
+ "name": "security-row1-col1",
"row-index": "0",
"column-index": "0",
"row-span": "1",
"column-span": "1"
- },
- {
- "name": "acid-transactions-row1-col1-2",
- "row-index": "1",
- "column-index": "0",
- "row-span": "1",
- "column-span": "1"
}
]
},
{
- "name": "performance",
- "display-name": "Performance",
- "row-index": "3",
+ "name": "optimization",
+ "display-name": "Optimization",
+ "row-index": "1",
"column-index": "0",
- "row-span": "3",
- "column-span": "2",
- "section-columns": "2",
+ "row-span": "1",
+ "column-span": "3",
+ "section-columns": "3",
"section-rows": "2",
"subsections": [
{
- "name": "performance-row1-col1",
- "display-name": "TEZ Engine",
+ "name": "optimization-row1-col1",
+ "display-name": "Tez",
"row-index": "0",
"column-index": "0",
"row-span": "1",
"column-span": "1"
},
{
- "name": "performance-row1-col2",
+ "name": "optimization-row1-col2",
"display-name": "",
"row-index": "0",
"column-index": "1",
@@ -135,55 +97,35 @@
"column-span": "1"
},
{
- "name": "performance-row2-col1",
+ "name": "optimization-row1-col3",
+ "display-name": "CBO",
+ "row-index": "0",
+ "column-index": "2",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "optimization-row2-col1",
+ "display-name": "Storage",
"row-index": "1",
"column-index": "0",
"row-span": "1",
"column-span": "1"
},
{
- "name": "performance-row2-col2",
+ "name": "optimization-row2-col2",
+ "display-name": "",
"row-index": "1",
"column-index": "1",
"row-span": "1",
"column-span": "1"
- }
- ]
- },
- {
- "name": "interactive-query",
- "display-name": "Interactive Query",
- "row-index": "2",
- "column-index": "2",
- "row-span": "2",
- "column-span": "1",
- "section-columns": "1",
- "section-rows": "2",
- "subsections": [
- {
- "name": "interactive-query-row1-col1",
- "row-index": "0",
- "column-index": "0",
- "row-span": "2",
- "column-span": "1"
- }
- ]
- },
- {
- "name": "security",
- "display-name": "Security",
- "row-index": "4",
- "column-index": "2",
- "row-span": "2",
- "column-span": "1",
- "section-columns": "1",
- "section-rows": "2",
- "subsections": [
+ },
{
- "name": "security-row1-col1",
- "row-index": "0",
- "column-index": "0",
- "row-span": "2",
+ "name": "optimization-row2-col3",
+ "display-name": "Memory",
+ "row-index": "1",
+ "column-index": "2",
+ "row-span": "1",
"column-span": "1"
}
]
@@ -198,66 +140,30 @@
"configuration-layout": "default",
"configs": [
{
- "config": "hive-site/hive.default.fileformat",
- "subsection-name": "optimization-row1-col1"
- },
- {
"config": "hive-site/hive.exec.orc.default.stripe.size",
- "subsection-name": "optimization-row1-col1"
+ "subsection-name": "optimization-row2-col1"
},
{
"config": "hive-site/hive.exec.orc.default.compress",
- "subsection-name": "optimization-row1-col2-1"
- },
- {
- "config": "hive-env/hive_exec_orc_storage_strategy",
- "subsection-name": "optimization-row1-col2-2"
- },
- {
- "config": "hive-site/hive.optimize.index.filter",
- "subsection-name": "optimization-row1-col1"
- },
- {
- "config": "hive-site/hive.optimize.sort.dynamic.partition",
- "subsection-name": "optimization-row1-col1"
- },
- {
- "config": "hive-site/hive.exec.orc.encoding.strategy",
- "subsection-name": "optimization-row1-col2-2"
- },
- {
- "config": "hive-site/hive.exec.orc.compression.strategy",
- "subsection-name": "optimization-row1-col2-2"
- },
- {
- "config": "hive-site/hive.vectorized.execution.enabled",
"subsection-name": "optimization-row2-col1"
},
{
- "config": "hive-site/hive.vectorized.execution.reduce.enabled",
+ "config": "hive-env/hive_exec_orc_storage_strategy",
"subsection-name": "optimization-row2-col2"
},
{
"config": "hive-site/hive.auto.convert.join.noconditionaltask.size",
- "subsection-name": "optimization-row3-col1"
+ "subsection-name": "optimization-row2-col3"
},
{
"config": "hive-site/hive.exec.reducers.bytes.per.reducer",
- "subsection-name": "optimization-row3-col2"
+ "subsection-name": "optimization-row2-col3"
},
{
"config": "hive-env/hive_txn_acid",
"subsection-name": "acid-transactions-row1-col1-1"
},
{
- "config": "hive-site/hive.txn.manager",
- "subsection-name": "acid-transactions-row1-col1-1"
- },
- {
- "config": "hive-site/hive.support.concurrency",
- "subsection-name": "acid-transactions-row1-col1-1"
- },
- {
"config": "hive-site/hive.compactor.initiator.on",
"subsection-name": "acid-transactions-row1-col1-1"
},
@@ -266,56 +172,28 @@
"subsection-name": "acid-transactions-row1-col1-1"
},
{
- "config": "hive-site/hive.enforce.bucketing",
- "subsection-name": "acid-transactions-row1-col1-2"
- },
- {
- "config": "hive-site/hive.exec.dynamic.partition.mode",
- "subsection-name": "acid-transactions-row1-col1-2"
- },
- {
"config": "hive-site/hive.execution.engine",
- "subsection-name": "performance-row1-col1"
- },
- {
- "config": "hive-env/hive_timeline_logging_enabled",
- "subsection-name": "performance-row1-col1"
+ "subsection-name": "optimization-row1-col1"
},
{
"config": "hive-site/hive.tez.container.size",
- "subsection-name": "performance-row1-col1"
+ "subsection-name": "optimization-row1-col1"
},
{
"config": "hive-site/hive.prewarm.enabled",
- "subsection-name": "performance-row1-col1"
+ "subsection-name": "optimization-row1-col2"
},
{
"config": "hive-site/hive.prewarm.numcontainers",
- "subsection-name": "performance-row1-col2"
- },
- {
- "config": "hive-site/hive.tez.auto.reducer.parallelism",
- "subsection-name": "performance-row1-col2"
- },
- {
- "config": "hive-site/hive.tez.dynamic.partition.pruning",
- "subsection-name": "performance-row1-col2"
+ "subsection-name": "optimization-row1-col2"
},
{
"config": "hive-site/hive.cbo.enable",
- "subsection-name": "performance-row2-col1"
- },
- {
- "config": "hive-site/hive.stats.fetch.partition.stats",
- "subsection-name": "performance-row2-col1"
+ "subsection-name": "optimization-row1-col3"
},
{
"config": "hive-site/hive.stats.fetch.column.stats",
- "subsection-name": "performance-row2-col1"
- },
- {
- "config": "hive-site/hive.compute.query.using.stats",
- "subsection-name": "performance-row2-col2"
+ "subsection-name": "optimization-row1-col3"
},
{
"config": "hiveserver2-site/hive.server2.tez.default.queues",
@@ -330,10 +208,6 @@
"subsection-name": "interactive-query-row1-col1"
},
{
- "config": "hiveserver2-site/hive.server2.enable.doAs",
- "subsection-name": "interactive-query-row1-col1"
- },
- {
"config": "hiveserver2-site/tez.session.am.dag.submit.timeout.secs",
"subsection-name": "interactive-query-row1-col1"
},
@@ -342,11 +216,7 @@
"subsection-name": "security-row1-col1"
},
{
- "config": "hive-site/hive.security.authorization.enabled",
- "subsection-name": "security-row1-col1"
- },
- {
- "config": "hive-site/hive.security.metastore.authorization.manager",
+ "config": "hiveserver2-site/hive.server2.enable.doAs",
"subsection-name": "security-row1-col1"
},
{
@@ -361,12 +231,6 @@
},
"widgets": [
{
- "config": "hive-site/hive.default.fileformat",
- "widget": {
- "type": "combo"
- }
- },
- {
"config": "hive-site/hive.exec.orc.default.stripe.size",
"widget": {
"type": "slider",
@@ -384,48 +248,12 @@
}
},
{
- "config": "hive-site/hive.optimize.index.filter",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "hive-site/hive.optimize.sort.dynamic.partition",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "hive-site/hive.exec.orc.encoding.strategy",
- "widget": {
- "type": "toggle"
- }
- },
- {
"config": "hive-env/hive_exec_orc_storage_strategy",
"widget": {
"type": "toggle"
}
},
{
- "config": "hive-site/hive.exec.orc.compression.strategy",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "hive-site/hive.vectorized.execution.enabled",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "hive-site/hive.vectorized.execution.reduce.enabled",
- "widget": {
- "type": "toggle"
- }
- },
- {
"config": "hive-site/hive.auto.convert.join.noconditionaltask.size",
"widget": {
"type": "slider",
@@ -454,24 +282,6 @@
}
},
{
- "config": "hive-site/hive.txn.manager",
- "widget": {
- "type": "combo"
- }
- },
- {
- "config": "hive-site/hive.security.metastore.authorization.manager",
- "widget": {
- "type": "string"
- }
- },
- {
- "config": "hive-site/hive.support.concurrency",
- "widget": {
- "type": "toggle"
- }
- },
- {
"config": "hive-site/hive.compactor.initiator.on",
"widget": {
"type": "toggle"
@@ -489,18 +299,6 @@
}
},
{
- "config": "hive-site/hive.enforce.bucketing",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "hive-site/hive.exec.dynamic.partition.mode",
- "widget": {
- "type": "toggle"
- }
- },
- {
"config": "hive-site/hive.execution.engine",
"widget": {
"type": "combo"
@@ -535,42 +333,18 @@
}
},
{
- "config": "hive-site/hive.tez.auto.reducer.parallelism",
- "widget": {
- "type": "toggle"
- }
- },
- {
- "config": "hive-site/hive.tez.dynamic.partition.pruning",
- "widget": {
- "type": "toggle"
- }
- },
- {
"config": "hive-site/hive.cbo.enable",
"widget": {
"type": "toggle"
}
},
{
- "config": "hive-site/hive.stats.fetch.partition.stats",
- "widget": {
- "type": "toggle"
- }
- },
- {
"config": "hive-site/hive.stats.fetch.column.stats",
"widget": {
"type": "toggle"
}
},
{
- "config": "hive-site/hive.compute.query.using.stats",
- "widget": {
- "type": "toggle"
- }
- },
- {
"config": "hiveserver2-site/hive.server2.tez.initialize.default.sessions",
"widget": {
"type": "toggle"
@@ -611,12 +385,6 @@
}
},
{
- "config": "hive-site/hive.security.authorization.enabled",
- "widget": {
- "type": "toggle"
- }
- },
- {
"config": "hive-site/hive.server2.authentication",
"widget": {
"type": "combo"
@@ -633,12 +401,6 @@
"widget": {
"type": "list"
}
- },
- {
- "config": "hive-env/hive_timeline_logging_enabled",
- "widget": {
- "type": "toggle"
- }
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 3700a2d..d5e1534 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -233,7 +233,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveSiteProperty("hive.vectorized.execution.reduce.enabled", "false")
# Memory
- putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", "2147483648")
putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864")
# Transactions
@@ -297,6 +296,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
"yarn.scheduler.minimum-allocation-mb" in configurations["yarn-site"]["properties"]:
container_size = configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]
putHiveSiteProperty("hive.tez.container.size", container_size)
+ putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", str(int(int(container_size)/3)*1024*1024))
putHiveSiteProperty("hive.prewarm.enabled", "false")
putHiveSiteProperty("hive.prewarm.numcontainers", "3")
putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true")
http://git-wip-us.apache.org/repos/asf/ambari/blob/132cf31b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 1253781..e0a5006 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -874,7 +874,7 @@ class TestHDP22StackAdvisor(TestCase):
},
'hive-site': {
'properties': {
- 'hive.auto.convert.join.noconditionaltask.size': '2147483648',
+ 'hive.auto.convert.join.noconditionaltask.size': '89128960',
'hive.cbo.enable': 'true',
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',