You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/04/18 03:24:41 UTC
svn commit: r766196 - in /hadoop/hive/trunk: CHANGES.txt
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
ql/src/test/queries/clientpositive/input4_limit.q
ql/src/test/results/clientpositive/input4_limit.q.out
Author: zshao
Date: Sat Apr 18 01:24:41 2009
New Revision: 766196
URL: http://svn.apache.org/viewvc?rev=766196&view=rev
Log:
HIVE-404. Fix ordering in "SELECT * FROM t SORT BY col1 LIMIT 100" when query is not a outer-most query. (Namit Jain via zshao)
Added:
hadoop/hive/trunk/ql/src/test/queries/clientpositive/input4_limit.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=766196&r1=766195&r2=766196&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Sat Apr 18 01:24:41 2009
@@ -64,6 +64,9 @@
HIVE-426. Fix undeterministic query plan because of aliasToWork.
(Namit Jain via zshao)
+ HIVE-404. Fix ordering in "SELECT * FROM t SORT BY col1 LIMIT 100" when
+ query is not a outer-most query. (Namit Jain via zshao)
+
Release 0.3.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=766196&r1=766195&r2=766196&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Apr 18 01:24:41 2009
@@ -2351,8 +2351,8 @@
// Add the limit operator to get the value fields
Operator curr = genLimitPlan(dest, qb, input, limit);
- // If it is a outer most query, the exact limit is applied by the fetch task
- if (isOuterQuery)
+ // If it is a outer most query and no sorting is specified, exact limit is applied by the fetch task
+ if (isOuterQuery && !sortRequired(dest, qb))
return curr;
// Create a reduceSink operator followed by another limit
@@ -2360,6 +2360,21 @@
return genLimitPlan(dest, qb, curr, limit);
}
+ /*
+ * Is sorting reuired ?
+ * If there are no cluster by/sort by keys, then an additional map-reduce job is not needed.
+ * Else, sort the output by the relevant key (via another map-reduce job).
+ */
+ private boolean sortRequired(String dest, QB qb) {
+ if (qb.getParseInfo().getClusterByForClause(dest) != null)
+ return true;
+
+ if (qb.getParseInfo().getSortByForClause(dest) != null)
+ return true;
+
+ return false;
+ }
+
@SuppressWarnings("nls")
private Operator genReduceSinkPlan(String dest, QB qb,
Operator input, int numReducers) throws SemanticException {
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input4_limit.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input4_limit.q?rev=766196&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input4_limit.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input4_limit.q Sat Apr 18 01:24:41 2009
@@ -0,0 +1,5 @@
+explain
+select * from src sort by key limit 10;
+
+
+select * from src sort by key limit 10;
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out?rev=766196&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out Sat Apr 18 01:24:41 2009
@@ -0,0 +1,81 @@
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_COLREF key))) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: string
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ name: binary_table
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ /data/users/njain/hive1/hive/build/ql/tmp/528401691/102882840.10002
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: string
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
+100 val_100
+103 val_103
+103 val_103
+104 val_104
+104 val_104