You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/07/17 06:16:05 UTC
svn commit: r1362342 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
test/queries/clientpositive/enforce_order.q
test/results/clientpositive/enforce_order.q.out
Author: namit
Date: Tue Jul 17 04:16:05 2012
New Revision: 1362342
URL: http://svn.apache.org/viewvc?rev=1362342&view=rev
Log:
HIVE-3247 Sorted by order of table not respected
(Navis via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/enforce_order.q
hive/trunk/ql/src/test/results/clientpositive/enforce_order.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1362342&r1=1362341&r2=1362342&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Jul 17 04:16:05 2012
@@ -3891,6 +3891,7 @@ public class SemanticAnalyzer extends Ba
ArrayList<ExprNodeDesc> partnCols = new ArrayList<ExprNodeDesc>();
ArrayList<ExprNodeDesc> partnColsNoConvert = new ArrayList<ExprNodeDesc>();
ArrayList<ExprNodeDesc> sortCols = new ArrayList<ExprNodeDesc>();
+ ArrayList<Integer> sortOrders = new ArrayList<Integer>();
boolean multiFileSpray = false;
int numFiles = 1;
int totalFiles = 1;
@@ -3907,6 +3908,7 @@ public class SemanticAnalyzer extends Ba
(conf.getBoolVar(HiveConf.ConfVars.HIVEENFORCESORTING))) {
enforceSorting = true;
sortCols = getSortCols(dest, qb, dest_tab, table_desc, input, true);
+ sortOrders = getSortOrders(dest, qb, dest_tab, input);
if (!enforceBucketing) {
partnCols = sortCols;
partnColsNoConvert = getSortCols(dest, qb, dest_tab, table_desc, input, false);
@@ -3932,7 +3934,8 @@ public class SemanticAnalyzer extends Ba
maxReducers = numBuckets;
}
- input = genReduceSinkPlanForSortingBucketing(dest_tab, input, sortCols, partnCols, maxReducers);
+ input = genReduceSinkPlanForSortingBucketing(dest_tab, input,
+ sortCols, sortOrders, partnCols, maxReducers);
ctx.setMultiFileSpray(multiFileSpray);
ctx.setNumFiles(numFiles);
ctx.setPartnCols(partnColsNoConvert);
@@ -4703,9 +4706,28 @@ public class SemanticAnalyzer extends Ba
return genConvertCol(dest, qb, tab, table_desc, input, posns, convert);
}
+ private ArrayList<Integer> getSortOrders(String dest, QB qb, Table tab, Operator input)
+ throws SemanticException {
+ RowResolver inputRR = opParseCtx.get(input).getRowResolver();
+ List<Order> tabSortCols = tab.getSortCols();
+ List<FieldSchema> tabCols = tab.getCols();
+
+ ArrayList<Integer> orders = new ArrayList<Integer>();
+ for (Order sortCol : tabSortCols) {
+ for (FieldSchema tabCol : tabCols) {
+ if (sortCol.getCol().equals(tabCol.getName())) {
+ orders.add(sortCol.getOrder());
+ break;
+ }
+ }
+ }
+ return orders;
+ }
+
@SuppressWarnings("nls")
private Operator genReduceSinkPlanForSortingBucketing(Table tab, Operator input,
ArrayList<ExprNodeDesc> sortCols,
+ List<Integer> sortOrders,
ArrayList<ExprNodeDesc> partitionCols,
int numReducers)
throws SemanticException {
@@ -4729,8 +4751,8 @@ public class SemanticAnalyzer extends Ba
}
StringBuilder order = new StringBuilder();
- for (int i = 0; i < sortCols.size(); i++) {
- order.append("+");
+ for (int sortOrder : sortOrders) {
+ order.append(sortOrder == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC ? '+' :'-');
}
Operator interim = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils
Added: hive/trunk/ql/src/test/queries/clientpositive/enforce_order.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/enforce_order.q?rev=1362342&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/enforce_order.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/enforce_order.q Tue Jul 17 04:16:05 2012
@@ -0,0 +1,13 @@
+drop table table_asc;
+drop table table_desc;
+
+set hive.enforce.sorting = true;
+
+create table table_asc(key string, value string) clustered by (key) sorted by (key ASC) into 1 BUCKETS;
+create table table_desc(key string, value string) clustered by (key) sorted by (key DESC) into 1 BUCKETS;
+
+insert overwrite table table_asc select key, value from src;
+insert overwrite table table_desc select key, value from src;
+
+select * from table_asc limit 10;
+select * from table_desc limit 10;
Added: hive/trunk/ql/src/test/results/clientpositive/enforce_order.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/enforce_order.q.out?rev=1362342&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/enforce_order.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/enforce_order.q.out Tue Jul 17 04:16:05 2012
@@ -0,0 +1,84 @@
+PREHOOK: query: drop table table_asc
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table table_asc
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table table_desc
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table table_desc
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table table_asc(key string, value string) clustered by (key) sorted by (key ASC) into 1 BUCKETS
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table table_asc(key string, value string) clustered by (key) sorted by (key ASC) into 1 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table_asc
+PREHOOK: query: create table table_desc(key string, value string) clustered by (key) sorted by (key DESC) into 1 BUCKETS
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table table_desc(key string, value string) clustered by (key) sorted by (key DESC) into 1 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table_desc
+PREHOOK: query: insert overwrite table table_asc select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@table_asc
+POSTHOOK: query: insert overwrite table table_asc select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@table_asc
+POSTHOOK: Lineage: table_asc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table_asc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table table_desc select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@table_desc
+POSTHOOK: query: insert overwrite table table_desc select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@table_desc
+POSTHOOK: Lineage: table_asc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table_asc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table_desc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table_desc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from table_asc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table_asc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from table_asc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table_asc
+#### A masked pattern was here ####
+POSTHOOK: Lineage: table_asc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table_asc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table_desc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table_desc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
+100 val_100
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+PREHOOK: query: select * from table_desc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table_desc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from table_desc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table_desc
+#### A masked pattern was here ####
+POSTHOOK: Lineage: table_asc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table_asc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table_desc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table_desc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+98 val_98
+98 val_98
+97 val_97
+97 val_97
+96 val_96
+95 val_95
+95 val_95
+92 val_92
+90 val_90
+90 val_90