svn commit: r995276 [1/3] - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/

[na...@apache.org]

Author: namit
Date: Wed Sep  8 22:46:04 2010
New Revision: 995276

URL: http://svn.apache.org/viewvc?rev=995276&view=rev
Log:
HIVE-1622. Use map-only task to merge if available
(Ning Zhang via namit)


Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge3.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/merge3.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/multi_insert.q.out

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=995276&r1=995275&r2=995276&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Sep  8 22:46:04 2010
@@ -241,6 +241,9 @@ Trunk -  Unreleased
     in HIVE-675
     (Carl Steinbach via Ning Zhang)
 
+    HIVE-1622. Use map-only tak to merge if available
+    (Ning Zhang via namit)
+
   TESTS
 
     HIVE-1464. improve  test query performance

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=995276&r1=995275&r2=995276&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Wed Sep  8 22:46:04 2010
@@ -222,7 +222,9 @@ public class GenMRFileSink1 implements N
     // if the hadoop version support CombineFileInputFormat (version >= 0.20),
     // create a Map-only job for merge, otherwise create a MapReduce merge job.
     ParseContext parseCtx = ctx.getParseCtx();
-    if (parseCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEMERGEMAPFILES) &&
+    HiveConf conf = parseCtx.getConf();
+    if ((conf.getBoolVar(HiveConf.ConfVars.HIVEMERGEMAPFILES) ||
+         conf.getBoolVar(HiveConf.ConfVars.HIVEMERGEMAPREDFILES)) &&
         Utilities.supportCombineFileInputFormat()) {
       // create Map-only merge job
       createMap4Merge(fsOp, ctx, finalName);

Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge3.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge3.q?rev=995276&r1=995275&r2=995276&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge3.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge3.q Wed Sep  8 22:46:04 2010
@@ -37,3 +37,20 @@ where ds is not null;
 show partitions merge_src_part2;
 
 select * from merge_src_part2 where ds is not null;
+
+drop table merge_src_part2;
+
+create table merge_src_part2 like merge_src_part;
+
+explain extended
+from (select * from merge_src_part where ds is not null distribute by ds) s
+insert overwrite table merge_src_part2 partition(ds)
+select key, value, ds;
+
+from (select * from merge_src_part where ds is not null distribute by ds) s
+insert overwrite table merge_src_part2 partition(ds)
+select key, value, ds;
+
+show partitions merge_src_part2;
+
+select * from merge_src_part2 where ds is not null;