You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/07/16 02:01:57 UTC

svn commit: r794468 - in /hadoop/hive/trunk: ./ common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/

Author: zshao
Date: Thu Jul 16 00:01:57 2009
New Revision: 794468

URL: http://svn.apache.org/viewvc?rev=794468&view=rev
Log:
HIVE-629. Concat files for map-reduce jobs. (Namit Jain via zshao)

Added:
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge1.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/merge1.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hadoop/hive/trunk/conf/hive-default.xml
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=794468&r1=794467&r2=794468&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Jul 16 00:01:57 2009
@@ -126,6 +126,8 @@
     HIVE-628. Remove stray lines from hive-default.xml.
     (Prasad Chakka via zshao)
 
+    HIVE-629. Concat files for map-reduce jobs. (Namit Jain via zshao)
+
   OPTIMIZATIONS
 
     HIVE-279. Predicate Pushdown support (Prasad Chakka via athusoo).

Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=794468&r1=794467&r2=794468&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jul 16 00:01:57 2009
@@ -145,7 +145,8 @@
     HIVETESTMODENOSAMPLE("hive.test.mode.nosamplelist", ""),
 
     HIVEMERGEMAPFILES("hive.merge.mapfiles", true),
-    HIVEMERGEMAPFILESSIZE("hive.merge.size.per.mapper", (long)(1000*1000*1000)),
+    HIVEMERGEMAPREDFILES("hive.merge.mapredfiles", false),
+    HIVEMERGEMAPFILESSIZE("hive.merge.size.per.task", (long)(1000*1000*1000)),
       
     HIVESENDHEARTBEAT("hive.heartbeat.interval", 1000),
 

Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=794468&r1=794467&r2=794468&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Thu Jul 16 00:01:57 2009
@@ -305,7 +305,13 @@
 <property>
   <name>hive.merge.mapfiles</name>
   <value>true</value>
-  <description>Merge small files at the end of the job</description>
+  <description>Merge small files at the end of a map-only job</description>
+</property>
+
+<property>
+  <name>hive.merge.mapredfiles</name>
+  <value>false</value>
+  <description>Merge small files at the end of any job(map only or map-reduce)</description>
 </property>
 
 <property>
@@ -315,7 +321,7 @@
 </property>
 
 <property>
-  <name>hive.merge.size.per.mapper</name>
+  <name>hive.merge.size.per.task</name>
   <value>1000000000</value>
   <description>Size of merged files at the end of the job</description>
 </property>

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=794468&r1=794467&r2=794468&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Thu Jul 16 00:01:57 2009
@@ -83,11 +83,13 @@
     boolean chDir = false;
     Task<? extends Serializable> currTask = ctx.getCurrTask();
 
-    if ((parseCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEMERGEMAPFILES)) &&
-        (ctx.getMvTask() != null) && (!ctx.getMvTask().isEmpty())) {
-      if (((ctx.getSeenFileSinkOps() == null) ||
-           (!ctx.getSeenFileSinkOps().contains((FileSinkOperator)nd))) &&
-          (((mapredWork)currTask.getWork()).getReducer() == null))
+    // Has the user enabled merging of files for map-only jobs or for all jobs
+    if (((ctx.getMvTask() != null) && (!ctx.getMvTask().isEmpty())) &&
+        ((ctx.getSeenFileSinkOps() == null) ||
+           (!ctx.getSeenFileSinkOps().contains((FileSinkOperator)nd)))) {
+      if ((parseCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEMERGEMAPFILES) &&
+          (((mapredWork)currTask.getWork()).getReducer() == null)) ||
+          parseCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVEMERGEMAPREDFILES))
         chDir = true;
     }
 

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge1.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge1.q?rev=794468&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge1.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/merge1.q Thu Jul 16 00:01:57 2009
@@ -0,0 +1,15 @@
+drop table dest1;
+set hive.merge.mapredfiles=true;
+
+create table dest1(key int, val int);
+
+explain
+insert overwrite table dest1
+select key, count(1) from src group by key;
+
+insert overwrite table dest1
+select key, count(1) from src group by key;
+
+select * from dest1;
+
+drop table dest1;

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/merge1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/merge1.q.out?rev=794468&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/merge1.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/merge1.q.out Thu Jul 16 00:01:57 2009
@@ -0,0 +1,430 @@
+query: drop table dest1
+query: create table dest1(key int, val int)
+query: explain
+insert overwrite table dest1
+select key, count(1) from src group by key
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-4 depends on stages: Stage-1
+  Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        src 
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+              Group By Operator
+                aggregations:
+                      expr: count(1)
+                keys:
+                      expr: key
+                      type: string
+                mode: hash
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col0
+                        type: string
+                  sort order: +
+                  Map-reduce partition columns:
+                        expr: _col0
+                        type: string
+                  tag: -1
+                  value expressions:
+                        expr: _col1
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          keys:
+                expr: KEY._col0
+                type: string
+          mode: mergepartial
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: string
+                  expr: _col1
+                  type: bigint
+            Select Operator
+              expressions:
+                    expr: UDFToInteger(_col0)
+                    type: int
+                    expr: UDFToInteger(_col1)
+                    type: int
+              File Output Operator
+                compressed: false
+                GlobalTableId: 1
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    name: dest1
+
+  Stage: Stage-4
+    Conditional Operator
+      list of dependent Tasks:
+          Move Operator
+            files:
+                hdfs directory: true
+                destination: file:/data/users/njain/hive1/hive1/build/ql/tmp/261244854/10000
+          Map Reduce
+            Alias -> Map Operator Tree:
+              file:/data/users/njain/hive1/hive1/build/ql/tmp/856923111/10002 
+                  Reduce Output Operator
+                    sort order: 
+                    Map-reduce partition columns:
+                          expr: rand()
+                          type: double
+                    tag: -1
+                    value expressions:
+                          expr: key
+                          type: int
+                          expr: val
+                          type: int
+            Reduce Operator Tree:
+              Extract
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: dest1
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: dest1
+
+
+query: insert overwrite table dest1
+select key, count(1) from src group by key
+Input: default/src
+Output: default/dest1
+query: select * from dest1
+Input: default/dest1
+Output: file:/data/users/njain/hive1/hive1/build/ql/tmp/710265692/10000
+0	3
+10	1
+100	2
+103	2
+104	2
+105	1
+11	1
+111	1
+113	2
+114	1
+116	1
+118	2
+119	3
+12	2
+120	2
+125	2
+126	1
+128	3
+129	2
+131	1
+133	1
+134	2
+136	1
+137	2
+138	4
+143	1
+145	1
+146	2
+149	2
+15	2
+150	1
+152	2
+153	1
+155	1
+156	1
+157	1
+158	1
+160	1
+162	1
+163	1
+164	2
+165	2
+166	1
+167	3
+168	1
+169	4
+17	1
+170	1
+172	2
+174	2
+175	2
+176	2
+177	1
+178	1
+179	2
+18	2
+180	1
+181	1
+183	1
+186	1
+187	3
+189	1
+19	1
+190	1
+191	2
+192	1
+193	3
+194	1
+195	2
+196	1
+197	2
+199	3
+2	1
+20	1
+200	2
+201	1
+202	1
+203	2
+205	2
+207	2
+208	3
+209	2
+213	2
+214	1
+216	2
+217	2
+218	1
+219	2
+221	2
+222	1
+223	2
+224	2
+226	1
+228	1
+229	2
+230	5
+233	2
+235	1
+237	2
+238	2
+239	2
+24	2
+241	1
+242	2
+244	1
+247	1
+248	1
+249	1
+252	1
+255	2
+256	2
+257	1
+258	1
+26	2
+260	1
+262	1
+263	1
+265	2
+266	1
+27	1
+272	2
+273	3
+274	1
+275	1
+277	4
+278	2
+28	1
+280	2
+281	2
+282	2
+283	1
+284	1
+285	1
+286	1
+287	1
+288	2
+289	1
+291	1
+292	1
+296	1
+298	3
+30	1
+302	1
+305	1
+306	1
+307	2
+308	1
+309	2
+310	1
+311	3
+315	1
+316	3
+317	2
+318	3
+321	2
+322	2
+323	1
+325	2
+327	3
+33	1
+331	2
+332	1
+333	2
+335	1
+336	1
+338	1
+339	1
+34	1
+341	1
+342	2
+344	2
+345	1
+348	5
+35	3
+351	1
+353	2
+356	1
+360	1
+362	1
+364	1
+365	1
+366	1
+367	2
+368	1
+369	3
+37	2
+373	1
+374	1
+375	1
+377	1
+378	1
+379	1
+382	2
+384	3
+386	1
+389	1
+392	1
+393	1
+394	1
+395	2
+396	3
+397	2
+399	2
+4	1
+400	1
+401	5
+402	1
+403	3
+404	2
+406	4
+407	1
+409	3
+41	1
+411	1
+413	2
+414	2
+417	3
+418	1
+419	1
+42	2
+421	1
+424	2
+427	1
+429	2
+43	1
+430	3
+431	3
+432	1
+435	1
+436	1
+437	1
+438	3
+439	2
+44	1
+443	1
+444	1
+446	1
+448	1
+449	1
+452	1
+453	1
+454	3
+455	1
+457	1
+458	2
+459	2
+460	1
+462	2
+463	2
+466	3
+467	1
+468	4
+469	5
+47	1
+470	1
+472	1
+475	1
+477	1
+478	2
+479	1
+480	3
+481	1
+482	1
+483	1
+484	1
+485	1
+487	1
+489	4
+490	1
+491	1
+492	2
+493	1
+494	1
+495	1
+496	1
+497	1
+498	3
+5	3
+51	2
+53	1
+54	1
+57	1
+58	2
+64	1
+65	1
+66	1
+67	2
+69	1
+70	3
+72	2
+74	1
+76	2
+77	1
+78	1
+8	1
+80	1
+82	1
+83	2
+84	2
+85	1
+86	1
+87	1
+9	1
+90	3
+92	1
+95	2
+96	1
+97	2
+98	2
+query: drop table dest1