You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/07/14 02:11:34 UTC

svn commit: r793762 - in /hadoop/hive/trunk: ./ common/src/java/org/apache/hadoop/hive/conf/ conf/ data/files/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientnegative/ ql/src/test/results/...

Author: zshao
Date: Tue Jul 14 00:11:33 2009
New Revision: 793762

URL: http://svn.apache.org/viewvc?rev=793762&view=rev
Log:
HIVE-626. Fix Column Pruner column order bug. (Yongqiang He via zshao)

Added:
    hadoop/hive/trunk/data/files/hive_626_bar.txt
    hadoop/hive/trunk/data/files/hive_626_count.txt
    hadoop/hive/trunk/data/files/hive_626_foo.txt
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hadoop/hive/trunk/conf/hive-default.xml
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
    hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Jul 14 00:11:33 2009
@@ -342,6 +342,8 @@
 
     HIVE-631. Fix TestParse. (Namit Jain via zshao)
 
+    HIVE-626. Fix Column Pruner column order bug. (Yongqiang He via zshao)
+
 Release 0.3.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Jul 14 00:11:33 2009
@@ -150,7 +150,8 @@
     HIVESENDHEARTBEAT("hive.heartbeat.interval", 1000),
 
     // Optimizer
-    HIVEOPTPPD("hive.optimize.ppd", false); // predicate pushdown
+    HIVEOPTCP("hive.optimize.cp", true), // column pruner
+    HIVEOPTPPD("hive.optimize.ppd", true); // predicate pushdown
     
     
     public final String varname;

Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Tue Jul 14 00:11:33 2009
@@ -253,6 +253,12 @@
 </property>
 
 <property>
+  <name>hive.optimize.cp</name>
+  <value>true</value>
+  <description>Whether to enable column pruner</description>
+</property>
+
+<property>
   <name>hive.optimize.ppd</name>
   <value>true</value>
   <description>Whether to enable predicate pushdown</description>

Added: hadoop/hive/trunk/data/files/hive_626_bar.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/hive_626_bar.txt?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/hive_626_bar.txt (added)
+++ hadoop/hive/trunk/data/files/hive_626_bar.txt Tue Jul 14 00:11:33 2009
@@ -0,0 +1 @@
+10,0,1,1,bar10,a,b,c,d

Added: hadoop/hive/trunk/data/files/hive_626_count.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/hive_626_count.txt?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/hive_626_count.txt (added)
+++ hadoop/hive/trunk/data/files/hive_626_count.txt Tue Jul 14 00:11:33 2009
@@ -0,0 +1 @@
+10,2

Added: hadoop/hive/trunk/data/files/hive_626_foo.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/hive_626_foo.txt?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/hive_626_foo.txt (added)
+++ hadoop/hive/trunk/data/files/hive_626_foo.txt Tue Jul 14 00:11:33 2009
@@ -0,0 +1 @@
+1,foo1,a,b,c,d

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Tue Jul 14 00:11:33 2009
@@ -540,8 +540,8 @@
       }
     }
 
-    for (int i = 0; i < childColLists.size(); i++) {
-      String internalName = childColLists.get(i);
+    for (int i = 0; i < outputCols.size(); i++) {
+      String internalName = outputCols.get(i);
       String[] nm = joinRR.reverseLookup(internalName);
       ColumnInfo col = joinRR.get(nm[0], nm[1]);
       newJoinRR.put(nm[0], nm[1], col);

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Tue Jul 14 00:11:33 2009
@@ -46,9 +46,12 @@
 	 */
 	public void initialize(HiveConf hiveConf) {
 		transformations = new ArrayList<Transform>();
-		transformations.add(new ColumnPruner());
-    if (hiveConf.getBoolean("hive.optimize.ppd", false))
+		if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCP)) {
+		  transformations.add(new ColumnPruner());
+		}
+    if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) {
       transformations.add(new PredicatePushDown());
+    }
     transformations.add(new UnionProcessor());
 		transformations.add(new MapJoinProcessor());
 	}

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_hive_626.q Tue Jul 14 00:11:33 2009
@@ -0,0 +1,30 @@
+drop table hive_foo;
+drop table hive_bar;
+drop table hive_count;
+
+create table hive_foo (foo_id int, foo_name string, foo_a string, foo_b string, 
+foo_c string, foo_d string) row format delimited fields terminated by ','
+stored as textfile;
+
+create table hive_bar (bar_id int, bar_0 int, foo_id int, bar_1 int, bar_name
+string, bar_a string, bar_b string, bar_c string, bar_d string) row format 
+delimited fields terminated by ',' stored as textfile;
+
+create table hive_count (bar_id int, n int) row format delimited fields 
+terminated by ',' stored as textfile;
+
+load data local inpath '../data/files/hive_626_foo.txt' overwrite into table hive_foo;
+load data local inpath '../data/files/hive_626_bar.txt' overwrite into table hive_bar;
+load data local inpath '../data/files/hive_626_count.txt' overwrite into table hive_count;
+
+explain
+select hive_foo.foo_name, hive_bar.bar_name, n from hive_foo join hive_bar on hive_foo.foo_id =
+hive_bar.foo_id join hive_count on hive_count.bar_id = hive_bar.bar_id;
+
+select hive_foo.foo_name, hive_bar.bar_name, n from hive_foo join hive_bar on hive_foo.foo_id =
+hive_bar.foo_id join hive_count on hive_count.bar_id = hive_bar.bar_id;
+
+
+drop table hive_foo;
+drop table hive_bar;
+drop table hive_count;

Modified: hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out?rev=793762&r1=793761&r2=793762&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/script_error.q.out Tue Jul 14 00:11:33 2009
@@ -39,5 +39,5 @@
 query: SELECT TRANSFORM(src.key, src.value) USING '../data/scripts/error_script' AS (tkey, tvalue)
 FROM src
 Input: default/src
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/35582581/85175665.10000
+Output: file:/Users/char/Documents/workspace/Hive-Clean/build/ql/tmp/1352070559/10000
 FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out?rev=793762&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out Tue Jul 14 00:11:33 2009
@@ -0,0 +1,140 @@
+query: drop table hive_foo
+query: drop table hive_bar
+query: drop table hive_count
+query: create table hive_foo (foo_id int, foo_name string, foo_a string, foo_b string, 
+foo_c string, foo_d string) row format delimited fields terminated by ','
+stored as textfile
+query: create table hive_bar (bar_id int, bar_0 int, foo_id int, bar_1 int, bar_name
+string, bar_a string, bar_b string, bar_c string, bar_d string) row format 
+delimited fields terminated by ',' stored as textfile
+query: create table hive_count (bar_id int, n int) row format delimited fields 
+terminated by ',' stored as textfile
+query: load data local inpath '../data/files/hive_626_foo.txt' overwrite into table hive_foo
+query: load data local inpath '../data/files/hive_626_bar.txt' overwrite into table hive_bar
+query: load data local inpath '../data/files/hive_626_count.txt' overwrite into table hive_count
+query: explain
+select hive_foo.foo_name, hive_bar.bar_name, n from hive_foo join hive_bar on hive_foo.foo_id =
+hive_bar.foo_id join hive_count on hive_count.bar_id = hive_bar.bar_id
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF hive_foo) (TOK_TABREF hive_bar) (= (. (TOK_TABLE_OR_COL hive_foo) foo_id) (. (TOK_TABLE_OR_COL hive_bar) foo_id))) (TOK_TABREF hive_count) (= (. (TOK_TABLE_OR_COL hive_count) bar_id) (. (TOK_TABLE_OR_COL hive_bar) bar_id)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL hive_foo) foo_name)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL hive_bar) bar_name)) (TOK_SELEXPR (TOK_TABLE_OR_COL n)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        hive_foo 
+            Reduce Output Operator
+              key expressions:
+                    expr: foo_id
+                    type: int
+              sort order: +
+              Map-reduce partition columns:
+                    expr: foo_id
+                    type: int
+              tag: 0
+              value expressions:
+                    expr: foo_name
+                    type: string
+        hive_bar 
+            Reduce Output Operator
+              key expressions:
+                    expr: foo_id
+                    type: int
+              sort order: +
+              Map-reduce partition columns:
+                    expr: foo_id
+                    type: int
+              tag: 1
+              value expressions:
+                    expr: bar_id
+                    type: int
+                    expr: bar_name
+                    type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 {VALUE._col1}
+            1 {VALUE._col0} {VALUE._col4}
+          File Output Operator
+            compressed: false
+            GlobalTableId: 0
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                name: binary_table
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        $INTNAME 
+            Reduce Output Operator
+              key expressions:
+                    expr: _col6
+                    type: int
+              sort order: +
+              Map-reduce partition columns:
+                    expr: _col6
+                    type: int
+              tag: 0
+              value expressions:
+                    expr: _col1
+                    type: string
+                    expr: _col10
+                    type: string
+        hive_count 
+            Reduce Output Operator
+              key expressions:
+                    expr: bar_id
+                    type: int
+              sort order: +
+              Map-reduce partition columns:
+                    expr: bar_id
+                    type: int
+              tag: 1
+              value expressions:
+                    expr: n
+                    type: int
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 {VALUE._col1} {VALUE._col10}
+            1 {VALUE._col1}
+          Select Operator
+            expressions:
+                  expr: _col1
+                  type: string
+                  expr: _col10
+                  type: string
+                  expr: _col16
+                  type: int
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+query: select hive_foo.foo_name, hive_bar.bar_name, n from hive_foo join hive_bar on hive_foo.foo_id =
+hive_bar.foo_id join hive_count on hive_count.bar_id = hive_bar.bar_id
+Input: default/hive_foo
+Input: default/hive_count
+Input: default/hive_bar
+Output: file:/Users/char/Documents/workspace/Hive-Clean/build/ql/tmp/512847523/10000
+foo1	bar10	2
+query: drop table hive_foo
+query: drop table hive_bar
+query: drop table hive_count