You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/01/08 22:22:13 UTC
svn commit: r1556641 - in /hive/branches/tez:
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/conf/tez/
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/jav...
Author: gunther
Date: Wed Jan 8 21:22:12 2014
New Revision: 1556641
URL: http://svn.apache.org/r1556641
Log:
HIVE-6103: Change hive.optimize.tez to hive.execution.engine with [mr, tez] values (Gunther Hagleitner)
Modified:
hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/tez/conf/hive-default.xml.template
hive/branches/tez/data/conf/tez/hive-site.xml
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredContext.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompilerFactory.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q
hive/branches/tez/ql/src/test/queries/clientpositive/tez_dml.q
hive/branches/tez/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
hive/branches/tez/ql/src/test/queries/clientpositive/tez_join_tests.q
hive/branches/tez/ql/src/test/queries/clientpositive/tez_joins_explain.q
Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java Wed Jan 8 21:22:12 2014
@@ -47,7 +47,7 @@ public class StatsSetupConst {
public String getPublisher(Configuration conf) {
return "org.apache.hadoop.hive.ql.stats.CounterStatsPublisher"; }
public String getAggregator(Configuration conf) {
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
return "org.apache.hadoop.hive.ql.stats.CounterStatsAggregatorTez";
}
return "org.apache.hadoop.hive.ql.stats.CounterStatsAggregator"; }
Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Jan 8 21:22:12 2014
@@ -864,7 +864,8 @@ public class HiveConf extends Configurat
// Whether to show the unquoted partition names in query results.
HIVE_DECODE_PARTITION_NAME("hive.decode.partition.name", false),
- HIVE_OPTIMIZE_TEZ("hive.optimize.tez", false),
+ HIVE_EXECUTION_ENGINE("hive.execution.engine", "mr",
+ new StringsValidator("mr", "tez")),
HIVE_JAR_DIRECTORY("hive.jar.directory", "hdfs:///user/hive/"),
HIVE_USER_INSTALL_DIR("hive.user.install.directory", "hdfs:///user/"),
Modified: hive/branches/tez/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/tez/conf/hive-default.xml.template?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/conf/hive-default.xml.template (original)
+++ hive/branches/tez/conf/hive-default.xml.template Wed Jan 8 21:22:12 2014
@@ -2037,11 +2037,10 @@
</property>
<property>
- <name>hive.optimize.tez</name>
- <value>false</value>
+ <name>hive.execution.engine</name>
+ <value>mr</value>
<description>
- Setting this property turns on Tez execution. Needs tez installed on the
- cluster. (Only availble on hadoop 2)
+ Chooses execution engine. Options are: mr (Map reduce, default) or tez (hadoop 2 only)
</description>
</property>
Modified: hive/branches/tez/data/conf/tez/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/data/conf/tez/hive-site.xml?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
Binary files - no diff available.
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java Wed Jan 8 21:22:12 2014
@@ -32,7 +32,7 @@ public class HashTableLoaderFactory {
}
public static HashTableLoader getLoader(Configuration hconf) {
- if (HiveConf.getBoolVar(hconf, ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ if (HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
return new org.apache.hadoop.hive.ql.exec.tez.HashTableLoader();
} else {
return new org.apache.hadoop.hive.ql.exec.mr.HashTableLoader();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredContext.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredContext.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredContext.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredContext.java Wed Jan 8 21:22:12 2014
@@ -49,7 +49,7 @@ public class MapredContext {
public static MapredContext init(boolean isMap, JobConf jobConf) {
MapredContext context =
- HiveConf.getBoolVar(jobConf, ConfVars.HIVE_OPTIMIZE_TEZ) ?
+ HiveConf.getVar(jobConf, ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") ?
new TezContext(isMap, jobConf) : new MapredContext(isMap, jobConf);
contexts.set(context);
return context;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java Wed Jan 8 21:22:12 2014
@@ -35,7 +35,7 @@ public class ObjectCacheFactory {
* Returns the appropriate cache
*/
public static ObjectCache getCache(Configuration conf) {
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
return new org.apache.hadoop.hive.ql.exec.tez.ObjectCache();
} else {
return new org.apache.hadoop.hive.ql.exec.mr.ObjectCache();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Jan 8 21:22:12 2014
@@ -2937,7 +2937,8 @@ public final class Utilities {
pathsProcessed.add(path);
LOG.info("Adding input file " + path);
- if (!HiveConf.getBoolVar(job, ConfVars.HIVE_OPTIMIZE_TEZ) && isEmptyPath(job, path, ctx)) {
+ if (!HiveConf.getVar(job, ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")
+ && isEmptyPath(job, path, ctx)) {
path = createDummyFileForEmptyPartition(path, job, work,
hiveScratchDir, alias, sequenceNumber++);
@@ -2954,7 +2955,8 @@ public final class Utilities {
// T2) x;
// If T is empty and T2 contains 100 rows, the user expects: 0, 100 (2
// rows)
- if (path == null && !HiveConf.getBoolVar(job, ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ if (path == null
+ && !HiveConf.getVar(job, ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
path = createDummyFileForEmptyTable(job, work, hiveScratchDir,
alias, sequenceNumber++);
pathsToAdd.add(path);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java Wed Jan 8 21:22:12 2014
@@ -138,7 +138,7 @@ public class HadoopJobExecHelper {
if (job != null) {
// even with tez on some jobs are run as MR. disable the flag in
// the conf, so that the backend runs fully as MR.
- HiveConf.setBoolVar(job, HiveConf.ConfVars.HIVE_OPTIMIZE_TEZ, false);
+ HiveConf.setVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr");
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Wed Jan 8 21:22:12 2014
@@ -1263,7 +1263,7 @@ public final class GenMapRedUtils {
} else {
cplan = createMRWorkForMergingFiles(conf, tsMerge, fsInputDesc);
- if (conf.getBoolVar(ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ if (conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
work = new TezWork();
cplan.setName("Merge");
((TezWork)work).add(cplan);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Wed Jan 8 21:22:12 2014
@@ -116,7 +116,7 @@ public class Optimizer {
transformations.add(new StatsOptimizer());
}
if (pctx.getContext().getExplain() ||
- HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
transformations.add(new AnnotateWithStatistics());
}
transformations.add(new SimpleFetchOptimizer()); // must be called last
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Jan 8 21:22:12 2014
@@ -6646,7 +6646,7 @@ public class SemanticAnalyzer extends Ba
if (((ASTNode) hint.getChild(0)).getToken().getType() == HiveParser.TOK_MAPJOIN) {
// the user has specified to ignore mapjoin hint
if (!conf.getBoolVar(HiveConf.ConfVars.HIVEIGNOREMAPJOINHINT)
- && !conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ && !conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
ASTNode hintTblNames = (ASTNode) hint.getChild(1);
int numCh = hintTblNames.getChildCount();
for (int tblPos = 0; tblPos < numCh; tblPos++) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompilerFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompilerFactory.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompilerFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompilerFactory.java Wed Jan 8 21:22:12 2014
@@ -35,7 +35,7 @@ public class TaskCompilerFactory {
* into executable units.
*/
public static TaskCompiler getCompiler(HiveConf conf, ParseContext parseContext) {
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_OPTIMIZE_TEZ)
+ if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")
&& !parseContext.getQB().getParseInfo().isAnalyzeCommand()) {
return new TezCompiler();
} else {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Wed Jan 8 21:22:12 2014
@@ -390,7 +390,7 @@ public final class PlanUtils {
*/
public static TableDesc getMapJoinKeyTableDesc(Configuration conf,
List<FieldSchema> fieldSchemas) {
- if (HiveConf.getBoolVar(conf, ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ if (HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
// In tez we use a different way of transmitting the hash table.
// We basically use ReduceSinkOperators and set the transfer to
// be broadcast (instead of partitioned). As a consequence we use
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Wed Jan 8 21:22:12 2014
@@ -310,7 +310,8 @@ public class SessionState {
throw new RuntimeException(e);
}
- if (HiveConf.getBoolVar(startSs.getConf(), HiveConf.ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ if (HiveConf.getVar(startSs.getConf(), HiveConf.ConfVars.HIVE_EXECUTION_ENGINE)
+ .equals("tez")) {
try {
if (startSs.tezSessionState == null) {
startSs.tezSessionState = new TezSessionState();
@@ -320,7 +321,7 @@ public class SessionState {
throw new RuntimeException(e);
}
} else {
- LOG.info("No Tez session required at this point. hive.optimize.tez is false.");
+ LOG.info("No Tez session required at this point. hive.execution.engine=mr.");
}
return startSs;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/mrr.q Wed Jan 8 21:22:12 2014
@@ -1,5 +1,3 @@
-set hive.optimize.tez=true;
-
-- simple query with multiple reduce stages
EXPLAIN SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt;
SELECT key, count(value) as cnt FROM src GROUP BY key ORDER BY cnt;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/tez_dml.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/tez_dml.q?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/tez_dml.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/tez_dml.q Wed Jan 8 21:22:12 2014
@@ -1,4 +1,3 @@
-set hive.optimize.tez=true;
set hive.exec.dynamic.partition.mode=nonstrict;
-- CTAS
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q Wed Jan 8 21:22:12 2014
@@ -1,5 +1,3 @@
-set hive.optimize.tez=true;
-
insert overwrite local directory '${system:test.tmp.dir}/tez_local_src_table_1'
select * from src order by key limit 10 ;
dfs -cat file:${system:test.tmp.dir}/tez_local_src_table_1/000000_0 ;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/tez_join_tests.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/tez_join_tests.q?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/tez_join_tests.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/tez_join_tests.q Wed Jan 8 21:22:12 2014
@@ -1,4 +1,3 @@
-set hive.optimize.tez=true;
explain
select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key) order by b.key) x right outer join src c on (x.value = c.value) order by x.key;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/tez_joins_explain.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/tez_joins_explain.q?rev=1556641&r1=1556640&r2=1556641&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/tez_joins_explain.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/tez_joins_explain.q Wed Jan 8 21:22:12 2014
@@ -1,4 +1,3 @@
-set hive.optimize.tez=true;
explain
select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key) order by b.key) x right outer join src c on (x.value = c.value) order by x.key;