You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/12/17 22:53:30 UTC
svn commit: r891936 - in /hadoop/hive/trunk: CHANGES.txt
ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
ql/src/test/queries/clientpositive/join_map_ppr.q
ql/src/test/results/clientpositive/join_map_ppr.q.out
Author: namit
Date: Thu Dec 17 21:53:29 2009
New Revision: 891936
URL: http://svn.apache.org/viewvc?rev=891936&view=rev
Log:
HIVE-993 Remove duplicate conversion for MapJoin
(He Yongqiang via namit)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=891936&r1=891935&r2=891936&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Dec 17 21:53:29 2009
@@ -324,6 +324,9 @@
HIVE-991 union with 200 kids fail
(namit via He Yongqiang)
+ HIVE-993 Remove duplicate conversion for MapJoin
+ (He Yongqiang via namit)
+
Release 0.4.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=891936&r1=891935&r2=891936&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Thu Dec 17 21:53:29 2009
@@ -145,8 +145,6 @@
pos++;
}
- int keyLength = 0;
-
//get the join keys from old parent ReduceSink operators
for (pos = 0; pos < newParentOps.size(); pos++) {
ReduceSinkOperator oldPar = (ReduceSinkOperator)oldReduceSinkParentOps.get(pos);
@@ -191,27 +189,6 @@
valueExprMap.put(new Byte((byte)pos), values);
}
- // implicit type conversion hierarchy
- for (int k = 0; k < keyLength; k++) {
- // Find the common class for type conversion
- TypeInfo commonType = keyExprMap.get(new Byte((byte)0)).get(k).getTypeInfo();
- for (int i=1; i < newParentOps.size(); i++) {
- TypeInfo a = commonType;
- TypeInfo b = keyExprMap.get(new Byte((byte)i)).get(k).getTypeInfo();
- commonType = FunctionRegistry.getCommonClassForComparison(a, b);
- if (commonType == null) {
- throw new SemanticException("Cannot do equality join on different types: " + a.getTypeName() + " and " + b.getTypeName());
- }
- }
-
- // Add implicit type conversion if necessary
- for (int i=0; i < newParentOps.size(); i++) {
- if (!commonType.equals(keyExprMap.get(new Byte((byte)i)).get(k).getTypeInfo())) {
- keyExprMap.get(new Byte((byte)i)).set(k, TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(commonType.getTypeName(), keyExprMap.get(new Byte((byte)i)).get(k)));
- }
- }
- }
-
org.apache.hadoop.hive.ql.plan.joinCond[] joinCondns = op.getConf().getConds();
Operator[] newPar = new Operator[newParentOps.size()];
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q?rev=891936&r1=891935&r2=891936&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q Thu Dec 17 21:53:29 2009
@@ -15,5 +15,27 @@
select * from dest_j1 x order by x.key;
+CREATE TABLE src_copy(key int, value string);
+CREATE TABLE src1_copy(key string, value string);
+INSERT OVERWRITE TABLE src_copy select key, value from src;
+INSERT OVERWRITE TABLE src1_copy select key, value from src1;
+
+EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key)
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11;
+
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key)
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11;
+
+select * from dest_j1 x order by x.key;
+
+drop table src_copy;
+drop table src1_copy;
drop table dest_j1;
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out?rev=891936&r1=891935&r2=891936&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out Thu Dec 17 21:53:29 2009
@@ -81,7 +81,7 @@
File Output Operator
compressed: false
GlobalTableId: 1
- directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
+ directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -96,7 +96,7 @@
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
- transient_lastDdlTime 1258011142
+ transient_lastDdlTime 1260991796
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest_j1
Local Work:
@@ -156,7 +156,7 @@
File Output Operator
compressed: false
GlobalTableId: 1
- directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
+ directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -171,7 +171,7 @@
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
- transient_lastDdlTime 1258011142
+ transient_lastDdlTime 1260991796
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest_j1
x
@@ -221,7 +221,7 @@
File Output Operator
compressed: false
GlobalTableId: 1
- directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
+ directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -236,7 +236,7 @@
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
- transient_lastDdlTime 1258011142
+ transient_lastDdlTime 1260991796
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest_j1
Needs Tagging: false
@@ -261,8 +261,8 @@
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11
- transient_lastDdlTime 1258011140
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart
+ transient_lastDdlTime 1260991793
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -279,7 +279,7 @@
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart
- transient_lastDdlTime 1258011139
+ transient_lastDdlTime 1260991793
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: srcpart
name: srcpart
@@ -290,11 +290,11 @@
Move Operator
files:
hdfs directory: true
- source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
- destination: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/379610489/10000
+ source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
+ destination: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/209835084/10000
Map Reduce
Alias -> Map Operator Tree:
- file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
+ file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
Reduce Output Operator
sort order:
Map-reduce partition columns:
@@ -310,9 +310,9 @@
type: string
Needs Tagging: false
Path -> Alias:
- file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002 [file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002]
+ file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002 [file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002]
Path -> Partition:
- file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
+ file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
Partition
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -327,7 +327,7 @@
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
- transient_lastDdlTime 1258011142
+ transient_lastDdlTime 1260991796
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -343,7 +343,7 @@
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
- transient_lastDdlTime 1258011142
+ transient_lastDdlTime 1260991796
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest_j1
name: dest_j1
@@ -352,7 +352,7 @@
File Output Operator
compressed: false
GlobalTableId: 0
- directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/379610489/10000
+ directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/209835084/10000
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -367,7 +367,7 @@
file.inputformat org.apache.hadoop.mapred.TextInputFormat
location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- transient_lastDdlTime 1258011142
+ transient_lastDdlTime 1260991796
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest_j1
@@ -375,7 +375,7 @@
Move Operator
tables:
replace: true
- source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/379610489/10000
+ source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/209835084/10000
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -390,10 +390,10 @@
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
- transient_lastDdlTime 1258011142
+ transient_lastDdlTime 1260991796
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest_j1
- tmp directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/379610489/10001
+ tmp directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/209835084/10001
PREHOOK: query: INSERT OVERWRITE TABLE dest_j1
@@ -419,11 +419,11 @@
PREHOOK: query: select * from dest_j1 x order by x.key
PREHOOK: type: QUERY
PREHOOK: Input: default@dest_j1
-PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/18999706/10000
+PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1227206168/10000
POSTHOOK: query: select * from dest_j1 x order by x.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest_j1
-POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/18999706/10000
+POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1227206168/10000
128 val_128 val_128
128 val_128 val_128
128 val_128 val_128
@@ -531,6 +531,570 @@
98 val_98 val_98
98 val_98 val_98
98 val_98 val_98
+PREHOOK: query: CREATE TABLE src_copy(key int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE src_copy(key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@src_copy
+PREHOOK: query: CREATE TABLE src1_copy(key string, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE src1_copy(key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@src1_copy
+PREHOOK: query: INSERT OVERWRITE TABLE src_copy select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src_copy
+POSTHOOK: query: INSERT OVERWRITE TABLE src_copy select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@src_copy
+PREHOOK: query: INSERT OVERWRITE TABLE src1_copy select key, value from src1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@src1_copy
+POSTHOOK: query: INSERT OVERWRITE TABLE src1_copy select key, value from src1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@src1_copy
+PREHOOK: query: EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key)
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key)
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF src1_copy x) (TOK_TABREF src_copy y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key))) (TOK_TABREF srcpart z) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL z) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest_j1)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x y))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL z) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) value))) (TOK_WHERE (and (= (. (TOK_TABLE_OR_COL z) ds) '2008-04-08') (= (. (TOK_TABLE_OR_COL z) hr) 11)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-4 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ z
+ TableScan
+ alias: z
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: ((ds = '2008-04-08') and (hr = 11))
+ type: boolean
+ Common Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ 2 {value} {ds} {hr}
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ 2 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ outputColumnNames: _col0, _col3, _col5, _col6, _col7
+ Position of Big Table: 2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ expr: _col5
+ type: string
+ expr: _col6
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col3, _col5, _col6, _col7
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: ((_col6 = '2008-04-08') and (_col7 = 11))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ name dest_j1
+ columns.types string:string:string
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ columns key,value,val2
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+ transient_lastDdlTime 1260991796
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+ Local Work:
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ y
+ Fetch Operator
+ limit: -1
+ x
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ y
+ TableScan
+ alias: y
+ Common Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ 2 {value} {ds} {hr}
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ 2 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ outputColumnNames: _col0, _col3, _col5, _col6, _col7
+ Position of Big Table: 2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ expr: _col5
+ type: string
+ expr: _col6
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col3, _col5, _col6, _col7
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: ((_col6 = '2008-04-08') and (_col7 = 11))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ name dest_j1
+ columns.types string:string:string
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ columns key,value,val2
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+ transient_lastDdlTime 1260991796
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+ x
+ TableScan
+ alias: x
+ Common Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ 2 {value} {ds} {hr}
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ 2 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ outputColumnNames: _col0, _col3, _col5, _col6, _col7
+ Position of Big Table: 2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ expr: _col5
+ type: string
+ expr: _col6
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col3, _col5, _col6, _col7
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: ((_col6 = '2008-04-08') and (_col7 = 11))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ name dest_j1
+ columns.types string:string:string
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ columns key,value,val2
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+ transient_lastDdlTime 1260991796
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+ Needs Tagging: false
+ Path -> Alias:
+ file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z]
+ Path -> Partition:
+ file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11
+ Partition
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ name srcpart
+ columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ columns key,value
+ partition_columns ds/hr
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart
+ transient_lastDdlTime 1260991793
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ name srcpart
+ columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ columns key,value
+ partition_columns ds/hr
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart
+ transient_lastDdlTime 1260991793
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: srcpart
+ name: srcpart
+
+ Stage: Stage-4
+ Conditional Operator
+ list of dependent Tasks:
+ Move Operator
+ files:
+ hdfs directory: true
+ source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+ destination: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/612668114/10000
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns:
+ expr: rand()
+ type: double
+ tag: -1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ expr: val2
+ type: string
+ Needs Tagging: false
+ Path -> Alias:
+ file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002 [file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002]
+ Path -> Partition:
+ file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+ Partition
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ name dest_j1
+ columns.types string:string:string
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ columns key,value,val2
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+ transient_lastDdlTime 1260991796
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ name dest_j1
+ columns.types string:string:string
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ columns key,value,val2
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+ transient_lastDdlTime 1260991796
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+ name: dest_j1
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/612668114/10000
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ name dest_j1
+ columns.types string:string:string
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ columns key,value,val2
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ transient_lastDdlTime 1260991796
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/612668114/10000
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ name dest_j1
+ columns.types string:string:string
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ columns key,value,val2
+ bucket_count -1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ file.inputformat org.apache.hadoop.mapred.TextInputFormat
+ file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+ transient_lastDdlTime 1260991796
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+ tmp directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/612668114/10001
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key)
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@src_copy
+PREHOOK: Input: default@src1_copy
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key)
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@src_copy
+POSTHOOK: Input: default@src1_copy
+POSTHOOK: Output: default@dest_j1
+PREHOOK: query: select * from dest_j1 x order by x.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j1
+PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1242228089/10000
+POSTHOOK: query: select * from dest_j1 x order by x.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j1
+POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1242228089/10000
+128 val_128 val_128
+128 val_128 val_128
+128 val_128 val_128
+128 val_128 val_128
+128 val_128 val_128
+128 val_128 val_128
+128 val_128 val_128
+128 val_128 val_128
+128 val_128 val_128
+146 val_146 val_146
+146 val_146 val_146
+146 val_146 val_146
+146 val_146 val_146
+150 val_150 val_150
+213 val_213 val_213
+213 val_213 val_213
+213 val_213 val_213
+213 val_213 val_213
+224 val_224 val_224
+224 val_224 val_224
+224 val_224 val_224
+224 val_224 val_224
+238 val_238 val_238
+238 val_238 val_238
+238 val_238 val_238
+238 val_238 val_238
+255 val_255 val_255
+255 val_255 val_255
+255 val_255 val_255
+255 val_255 val_255
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+278 val_278 val_278
+278 val_278 val_278
+278 val_278 val_278
+278 val_278 val_278
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+369 val_369 val_369
+369 val_369 val_369
+369 val_369 val_369
+369 val_369 val_369
+369 val_369 val_369
+369 val_369 val_369
+369 val_369 val_369
+369 val_369 val_369
+369 val_369 val_369
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+66 val_66 val_66
+98 val_98 val_98
+98 val_98 val_98
+98 val_98 val_98
+98 val_98 val_98
+PREHOOK: query: drop table src_copy
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table src_copy
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@src_copy
+PREHOOK: query: drop table src1_copy
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table src1_copy
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@src1_copy
PREHOOK: query: drop table dest_j1
PREHOOK: type: DROPTABLE
POSTHOOK: query: drop table dest_j1