You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/07/14 02:11:34 UTC
svn commit: r793762 - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/files/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientnegative/
ql/src/test/results/...
Author: zshao
Date: Tue Jul 14 00:11:33 2009
New Revision: 793762
URL: http://svn.apache.org/viewvc?rev=793762&view=rev
Log:
HIVE-626. Fix Column Pruner column order bug. (Yongqiang He via zshao)
Added:
hadoop/hive/trunk/data/files/hive_626_bar.txt
hadoop/hive/trunk/data/files/hive_626_count.txt
hadoop/hive/trunk/data/files/hive_626_foo.txt
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/conf/hive-default.xml
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Jul 14 00:11:33 2009
@@ -342,6 +342,8 @@
HIVE-631. Fix TestParse. (Namit Jain via zshao)
+ HIVE-626. Fix Column Pruner column order bug. (Yongqiang He via zshao)
+
Release 0.3.1 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Jul 14 00:11:33 2009
@@ -150,7 +150,8 @@
HIVESENDHEARTBEAT("hive.heartbeat.interval", 1000),
// Optimizer
- HIVEOPTPPD("hive.optimize.ppd", false); // predicate pushdown
+ HIVEOPTCP("hive.optimize.cp", true), // column pruner
+ HIVEOPTPPD("hive.optimize.ppd", true); // predicate pushdown
public final String varname;
Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Tue Jul 14 00:11:33 2009
@@ -253,6 +253,12 @@
</property>
<property>
+ <name>hive.optimize.cp</name>
+ <value>true</value>
+ <description>Whether to enable column pruner</description>
+</property>
+
+<property>
<name>hive.optimize.ppd</name>
<value>true</value>
<description>Whether to enable predicate pushdown</description>
Added: hadoop/hive/trunk/data/files/hive_626_bar.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/hive_626_bar.txt?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/hive_626_bar.txt (added)
+++ hadoop/hive/trunk/data/files/hive_626_bar.txt Tue Jul 14 00:11:33 2009
@@ -0,0 +1 @@
+10,0,1,1,bar10,a,b,c,d
Added: hadoop/hive/trunk/data/files/hive_626_count.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/hive_626_count.txt?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/hive_626_count.txt (added)
+++ hadoop/hive/trunk/data/files/hive_626_count.txt Tue Jul 14 00:11:33 2009
@@ -0,0 +1 @@
+10,2
Added: hadoop/hive/trunk/data/files/hive_626_foo.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/hive_626_foo.txt?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/hive_626_foo.txt (added)
+++ hadoop/hive/trunk/data/files/hive_626_foo.txt Tue Jul 14 00:11:33 2009
@@ -0,0 +1 @@
+1,foo1,a,b,c,d
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Tue Jul 14 00:11:33 2009
@@ -540,8 +540,8 @@
}
}
- for (int i = 0; i < childColLists.size(); i++) {
- String internalName = childColLists.get(i);
+ for (int i = 0; i < outputCols.size(); i++) {
+ String internalName = outputCols.get(i);
String[] nm = joinRR.reverseLookup(internalName);
ColumnInfo col = joinRR.get(nm[0], nm[1]);
newJoinRR.put(nm[0], nm[1], col);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Tue Jul 14 00:11:33 2009
@@ -46,9 +46,12 @@
*/
public void initialize(HiveConf hiveConf) {
transformations = new ArrayList<Transform>();
- transformations.add(new ColumnPruner());
- if (hiveConf.getBoolean("hive.optimize.ppd", false))
+ if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCP)) {
+ transformations.add(new ColumnPruner());
+ }
+ if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) {
transformations.add(new PredicatePushDown());
+ }
transformations.add(new UnionProcessor());
transformations.add(new MapJoinProcessor());
}
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q Tue Jul 14 00:11:33 2009
@@ -0,0 +1,30 @@
+drop table hive_foo;
+drop table hive_bar;
+drop table hive_count;
+
+create table hive_foo (foo_id int, foo_name string, foo_a string, foo_b string,
+foo_c string, foo_d string) row format delimited fields terminated by ','
+stored as textfile;
+
+create table hive_bar (bar_id int, bar_0 int, foo_id int, bar_1 int, bar_name
+string, bar_a string, bar_b string, bar_c string, bar_d string) row format
+delimited fields terminated by ',' stored as textfile;
+
+create table hive_count (bar_id int, n int) row format delimited fields
+terminated by ',' stored as textfile;
+
+load data local inpath '../data/files/hive_626_foo.txt' overwrite into table hive_foo;
+load data local inpath '../data/files/hive_626_bar.txt' overwrite into table hive_bar;
+load data local inpath '../data/files/hive_626_count.txt' overwrite into table hive_count;
+
+explain
+select hive_foo.foo_name, hive_bar.bar_name, n from hive_foo join hive_bar on hive_foo.foo_id =
+hive_bar.foo_id join hive_count on hive_count.bar_id = hive_bar.bar_id;
+
+select hive_foo.foo_name, hive_bar.bar_name, n from hive_foo join hive_bar on hive_foo.foo_id =
+hive_bar.foo_id join hive_count on hive_count.bar_id = hive_bar.bar_id;
+
+
+drop table hive_foo;
+drop table hive_bar;
+drop table hive_count;
Modified: hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out Tue Jul 14 00:11:33 2009
@@ -39,5 +39,5 @@
query: SELECT TRANSFORM(src.key, src.value) USING '../data/scripts/error_script' AS (tkey, tvalue)
FROM src
Input: default/src
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/35582581/85175665.10000
+Output: file:/Users/char/Documents/workspace/Hive-Clean/build/ql/tmp/1352070559/10000
FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out Tue Jul 14 00:11:33 2009
@@ -0,0 +1,140 @@
+query: drop table hive_foo
+query: drop table hive_bar
+query: drop table hive_count
+query: create table hive_foo (foo_id int, foo_name string, foo_a string, foo_b string,
+foo_c string, foo_d string) row format delimited fields terminated by ','
+stored as textfile
+query: create table hive_bar (bar_id int, bar_0 int, foo_id int, bar_1 int, bar_name
+string, bar_a string, bar_b string, bar_c string, bar_d string) row format
+delimited fields terminated by ',' stored as textfile
+query: create table hive_count (bar_id int, n int) row format delimited fields
+terminated by ',' stored as textfile
+query: load data local inpath '../data/files/hive_626_foo.txt' overwrite into table hive_foo
+query: load data local inpath '../data/files/hive_626_bar.txt' overwrite into table hive_bar
+query: load data local inpath '../data/files/hive_626_count.txt' overwrite into table hive_count
+query: explain
+select hive_foo.foo_name, hive_bar.bar_name, n from hive_foo join hive_bar on hive_foo.foo_id =
+hive_bar.foo_id join hive_count on hive_count.bar_id = hive_bar.bar_id
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF hive_foo) (TOK_TABREF hive_bar) (= (. (TOK_TABLE_OR_COL hive_foo) foo_id) (. (TOK_TABLE_OR_COL hive_bar) foo_id))) (TOK_TABREF hive_count) (= (. (TOK_TABLE_OR_COL hive_count) bar_id) (. (TOK_TABLE_OR_COL hive_bar) bar_id)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL hive_foo) foo_name)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL hive_bar) bar_name)) (TOK_SELEXPR (TOK_TABLE_OR_COL n)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ hive_foo
+ Reduce Output Operator
+ key expressions:
+ expr: foo_id
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: foo_id
+ type: int
+ tag: 0
+ value expressions:
+ expr: foo_name
+ type: string
+ hive_bar
+ Reduce Output Operator
+ key expressions:
+ expr: foo_id
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: foo_id
+ type: int
+ tag: 1
+ value expressions:
+ expr: bar_id
+ type: int
+ expr: bar_name
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col4}
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ name: binary_table
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: _col6
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col6
+ type: int
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: string
+ expr: _col10
+ type: string
+ hive_count
+ Reduce Output Operator
+ key expressions:
+ expr: bar_id
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: bar_id
+ type: int
+ tag: 1
+ value expressions:
+ expr: n
+ type: int
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col1} {VALUE._col10}
+ 1 {VALUE._col1}
+ Select Operator
+ expressions:
+ expr: _col1
+ type: string
+ expr: _col10
+ type: string
+ expr: _col16
+ type: int
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+query: select hive_foo.foo_name, hive_bar.bar_name, n from hive_foo join hive_bar on hive_foo.foo_id =
+hive_bar.foo_id join hive_count on hive_count.bar_id = hive_bar.bar_id
+Input: default/hive_foo
+Input: default/hive_count
+Input: default/hive_bar
+Output: file:/Users/char/Documents/workspace/Hive-Clean/build/ql/tmp/512847523/10000
+foo1 bar10 2
+query: drop table hive_foo
+query: drop table hive_bar
+query: drop table hive_count