You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2013/05/14 22:21:34 UTC
svn commit: r1482557 - in /incubator/ambari/trunk: CHANGES.txt
ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml
ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml
Author: swagle
Date: Tue May 14 20:21:34 2013
New Revision: 1482557
URL: http://svn.apache.org/r1482557
Log:
AMBARI-2116. Add parameters to improve HIVE performance. With descriptions. (swagle)
Modified:
incubator/ambari/trunk/CHANGES.txt
incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml
incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml
Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1482557&r1=1482556&r2=1482557&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Tue May 14 20:21:34 2013
@@ -850,6 +850,9 @@ Trunk (unreleased changes):
BUG FIXES
+ AMBARI-2116. Add parameters to improve HIVE performance. With decriptions.
+ (swagle)
+
AMBARI-2127. Service check should not emit log with Exceptions considering
its a positive scenario. (smohanty)
Modified: incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml?rev=1482557&r1=1482556&r2=1482557&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml (original)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml Tue May 14 20:21:34 2013
@@ -143,16 +143,19 @@ limitations under the License.
<property>
<name>hive.enforce.bucketing</name>
<value>true</value>
+ <description>Whether bucketing is enforced. If true, while inserting into the table, bucketing is enforced.</description>
</property>
<property>
<name>hive.enforce.sorting</name>
<value>true</value>
+ <description>Whether sorting is enforced. If true, while inserting into the table, sorting is enforced.</description>
</property>
<property>
<name>hive.map.aggr</name>
<value>true</value>
+ <description>Whether to use map-side aggregation in Hive Group By queries.</description>
</property>
<property>
@@ -168,16 +171,22 @@ limitations under the License.
<property>
<name>hive.mapred.reduce.tasks.speculative.execution</name>
<value>false</value>
+ <description>Whether speculative execution for reducers should be turned on.</description>
</property>
<property>
<name>hive.auto.convert.join</name>
<value>true</value>
+ <description>Whether Hive enable the optimization about converting common
+ join into mapjoin based on the input file size.</description>
</property>
<property>
<name>hive.auto.convert.sortmerge.join</name>
<value>true</value>
+ <description>Will the join be automatically converted to a sort-merge join, if the joined tables pass
+ the criteria for sort-merge join.
+ </description>
</property>
<property>
@@ -188,21 +197,38 @@ limitations under the License.
<property>
<name>hive.auto.convert.join.noconditionaltask</name>
<value>true</value>
+ <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file
+ size. If this paramater is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than the
+ specified size, the join is directly converted to a mapjoin (there is no conditional task).
+ </description>
</property>
<property>
<name>hive.auto.convert.join.noconditionaltask.size</name>
<value>1000000000</value>
+ <description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it
+ is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly
+ converted to a mapjoin(there is no conditional task). The default is 10MB.
+ </description>
</property>
<property>
<name>hive.optimize.reducededuplication.min.reducer</name>
<value>1</value>
+ <description>Reduce deduplication merges two RSs by moving key/parts/reducer-num of the child RS to parent RS.
+ That means if reducer-num of the child RS is fixed (order by or forced bucketing) and small, it can make very slow, single MR.
+ The optimization will be disabled if number of reducers is less than specified value.
+ </description>
</property>
<property>
<name>hive.optimize.mapjoin.mapreduce</name>
<value>true</value>
+ <description>If hive.auto.convert.join is off, this parameter does not take
+ affect. If it is on, and if there are map-join jobs followed by a map-reduce
+ job (for e.g a group by), each map-only job is merged with the following
+ map-reduce job.
+ </description>
</property>
</configuration>
Modified: incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml?rev=1482557&r1=1482556&r2=1482557&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml (original)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml Tue May 14 20:21:34 2013
@@ -143,16 +143,19 @@ limitations under the License.
<property>
<name>hive.enforce.bucketing</name>
<value>true</value>
+ <description>Whether bucketing is enforced. If true, while inserting into the table, bucketing is enforced.</description>
</property>
<property>
<name>hive.enforce.sorting</name>
<value>true</value>
+ <description>Whether sorting is enforced. If true, while inserting into the table, sorting is enforced.</description>
</property>
<property>
<name>hive.map.aggr</name>
<value>true</value>
+ <description>Whether to use map-side aggregation in Hive Group By queries.</description>
</property>
<property>
@@ -168,16 +171,22 @@ limitations under the License.
<property>
<name>hive.mapred.reduce.tasks.speculative.execution</name>
<value>false</value>
+ <description>Whether speculative execution for reducers should be turned on.</description>
</property>
<property>
<name>hive.auto.convert.join</name>
<value>true</value>
+ <description>Whether Hive enable the optimization about converting common
+ join into mapjoin based on the input file size.</description>
</property>
<property>
<name>hive.auto.convert.sortmerge.join</name>
<value>true</value>
+ <description>Will the join be automatically converted to a sort-merge join, if the joined tables pass
+ the criteria for sort-merge join.
+ </description>
</property>
<property>
@@ -188,21 +197,38 @@ limitations under the License.
<property>
<name>hive.auto.convert.join.noconditionaltask</name>
<value>true</value>
+ <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file
+ size. If this paramater is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than the
+ specified size, the join is directly converted to a mapjoin (there is no conditional task).
+ </description>
</property>
<property>
<name>hive.auto.convert.join.noconditionaltask.size</name>
<value>1000000000</value>
+ <description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it
+ is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly
+ converted to a mapjoin(there is no conditional task). The default is 10MB.
+ </description>
</property>
<property>
<name>hive.optimize.reducededuplication.min.reducer</name>
<value>1</value>
+ <description>Reduce deduplication merges two RSs by moving key/parts/reducer-num of the child RS to parent RS.
+ That means if reducer-num of the child RS is fixed (order by or forced bucketing) and small, it can make very slow, single MR.
+ The optimization will be disabled if number of reducers is less than specified value.
+ </description>
</property>
<property>
<name>hive.optimize.mapjoin.mapreduce</name>
<value>true</value>
+ <description>If hive.auto.convert.join is off, this parameter does not take
+ affect. If it is on, and if there are map-join jobs followed by a map-reduce
+ job (for e.g a group by), each map-only job is merged with the following
+ map-reduce job.
+ </description>
</property>
</configuration>