You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/11/07 09:54:20 UTC
svn commit: r1539560 - in
/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql:
exec/tez/TezSessionState.java optimizer/GenMapRedUtils.java
plan/ConditionalResolverMergeFiles.java
Author: gunther
Date: Thu Nov 7 08:54:19 2013
New Revision: 1539560
URL: http://svn.apache.org/r1539560
Log:
HIVE-5770: Switch merge tasks to use tez (Gunther Hagleitner)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java?rev=1539560&r1=1539559&r2=1539560&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java Thu Nov 7 08:54:19 2013
@@ -115,7 +115,7 @@ public class TezSessionState {
// In case we need to run some MR jobs, we'll run them under tez MR emulation. The session
// id is used for tez to reuse the current session rather than start a new one.
conf.set("mapreduce.framework.name", "yarn-tez");
- conf.set("tez.session.id", session.getApplicationId().toString());
+ conf.set("mapreduce.tez.session.tokill-application-id", session.getApplicationId().toString());
}
/**
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1539560&r1=1539559&r2=1539560&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Thu Nov 7 08:54:19 2013
@@ -1245,10 +1245,16 @@ public final class GenMapRedUtils {
} else {
cplan = createMRWorkForMergingFiles(conf, tsMerge, fsInputDesc);
- work = new MapredWork();
- ((MapredWork)work).setMapWork(cplan);
- // use CombineHiveInputFormat for map-only merging
+ if (conf.getBoolVar(ConfVars.HIVE_OPTIMIZE_TEZ)) {
+ work = new TezWork();
+ cplan.setName("Merge");
+ ((TezWork)work).add(cplan);
+ } else {
+ work = new MapredWork();
+ ((MapredWork)work).setMapWork(cplan);
+ }
}
+ // use CombineHiveInputFormat for map-only merging
cplan.setInputformat("org.apache.hadoop.hive.ql.io.CombineHiveInputFormat");
// NOTE: we should gather stats in MR1 rather than MR2 at merge job since we don't
// know if merge MR2 will be triggered at execution time
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java?rev=1539560&r1=1539559&r2=1539560&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java Thu Nov 7 08:54:19 2013
@@ -147,6 +147,8 @@ public class ConditionalResolverMergeFil
MapWork work;
if (mrTask.getWork() instanceof MapredWork) {
work = ((MapredWork) mrTask.getWork()).getMapWork();
+ } else if (mrTask.getWork() instanceof TezWork){
+ work = (MapWork) ((TezWork) mrTask.getWork()).getAllWork().get(0);
} else {
work = (MapWork) mrTask.getWork();
}