You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/10 02:07:09 UTC
svn commit: r1466300 [6/13] - in /hive/trunk:
cli/src/java/org/apache/hadoop/hive/cli/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Modified: hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out Wed Apr 10 00:06:55 2013
@@ -11,11 +11,13 @@ PREHOOK: Output: default@t1
POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T1.txt' INTO TABLE T1
POSTHOOK: type: LOAD
POSTHOOK: Output: default@t1
-PREHOOK: query: INSERT OVERWRITE TABLE T1 select key, val from T1
+PREHOOK: query: -- perform an insert to make sure there are 2 files
+INSERT OVERWRITE TABLE T1 select key, val from T1
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
PREHOOK: Output: default@t1
-POSTHOOK: query: INSERT OVERWRITE TABLE T1 select key, val from T1
+POSTHOOK: query: -- perform an insert to make sure there are 2 files
+INSERT OVERWRITE TABLE T1 select key, val from T1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t1
POSTHOOK: Output: default@t1
@@ -28,11 +30,17 @@ POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@outputTbl1
POSTHOOK: Lineage: t1.key SIMPLE [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- The plan should be converted to a map-side group by if the group by key
+-- matches the sorted key
+-- addind a order by at the end to make the test results deterministic
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT key, count(1) FROM T1 GROUP BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- The plan should be converted to a map-side group by if the group by key
+-- matches the sorted key
+-- addind a order by at the end to make the test results deterministic
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT key, count(1) FROM T1 GROUP BY key
POSTHOOK: type: QUERY
@@ -371,11 +379,13 @@ POSTHOOK: Lineage: outputtbl1.cnt EXPRES
POSTHOOK: Lineage: outputtbl1.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t1.key SIMPLE [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- no map-side group by even if the group by key is a superset of sorted key
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl2
SELECT key, val, count(1) FROM T1 GROUP BY key, val
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- no map-side group by even if the group by key is a superset of sorted key
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl2
SELECT key, val, count(1) FROM T1 GROUP BY key, val
POSTHOOK: type: QUERY
@@ -665,11 +675,13 @@ POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1
7 17 1
8 18 1
8 28 1
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- It should work for sub-queries
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT key, count(1) FROM (SELECT key, val FROM T1) subq1 GROUP BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- It should work for sub-queries
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT key, count(1) FROM (SELECT key, val FROM T1) subq1 GROUP BY key
POSTHOOK: type: QUERY
@@ -1054,11 +1066,13 @@ POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1
3 1
7 1
8 2
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- It should work for sub-queries with column aliases
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT k, count(1) FROM (SELECT key as k, val as v FROM T1) subq1 GROUP BY k
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- It should work for sub-queries with column aliases
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT k, count(1) FROM (SELECT key as k, val as v FROM T1) subq1 GROUP BY k
POSTHOOK: type: QUERY
@@ -1465,11 +1479,15 @@ POSTHOOK: Lineage: outputtbl2.key1 EXPRE
POSTHOOK: Lineage: outputtbl2.key2 SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
POSTHOOK: Lineage: t1.key SIMPLE [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- The plan should be converted to a map-side group by if the group by key contains a constant followed
+-- by a match to the sorted key
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl3
SELECT 1, key, count(1) FROM T1 GROUP BY 1, key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- The plan should be converted to a map-side group by if the group by key contains a constant followed
+-- by a match to the sorted key
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl3
SELECT 1, key, count(1) FROM T1 GROUP BY 1, key
POSTHOOK: type: QUERY
@@ -1851,11 +1869,13 @@ POSTHOOK: Lineage: outputtbl3.key1 SIMPL
POSTHOOK: Lineage: outputtbl3.key2 EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t1.key SIMPLE [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- no map-side group by if the group by key contains a constant followed by another column
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl4
SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- no map-side group by if the group by key contains a constant followed by another column
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl4
SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val
POSTHOOK: type: QUERY
@@ -2191,11 +2211,13 @@ POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1
7 1 17 1
8 1 18 1
8 1 28 1
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- no map-side group by if the group by key contains a function
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl3
SELECT key, key + 1, count(1) FROM T1 GROUP BY key, key + 1
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- no map-side group by if the group by key contains a function
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl3
SELECT key, key + 1, count(1) FROM T1 GROUP BY key, key + 1
POSTHOOK: type: QUERY
@@ -2534,13 +2556,21 @@ POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1
3 4 1
7 8 1
8 9 2
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- it should not matter what follows the group by
+-- test various cases
+
+-- group by followed by another group by
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT key + key, sum(cnt) from
(SELECT key, count(1) as cnt FROM T1 GROUP BY key) subq1
group by key + key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- it should not matter what follows the group by
+-- test various cases
+
+-- group by followed by another group by
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT key + key, sum(cnt) from
(SELECT key, count(1) as cnt FROM T1 GROUP BY key) subq1
@@ -2894,7 +2924,8 @@ POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1
6 1
14 1
16 2
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- group by followed by a union
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT * FROM (
SELECT key, count(1) FROM T1 GROUP BY key
@@ -2902,7 +2933,8 @@ SELECT key, count(1) FROM T1 GROUP BY ke
SELECT key, count(1) FROM T1 GROUP BY key
) subq1
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- group by followed by a union
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT * FROM (
SELECT key, count(1) FROM T1 GROUP BY key
@@ -3422,7 +3454,8 @@ POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1
7 1
8 2
8 2
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- group by followed by a union where one of the sub-queries is map-side group by
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT * FROM (
SELECT key, count(1) FROM T1 GROUP BY key
@@ -3430,7 +3463,8 @@ SELECT key, count(1) FROM T1 GROUP BY ke
SELECT key + key as key, count(1) FROM T1 GROUP BY key + key
) subq1
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- group by followed by a union where one of the sub-queries is map-side group by
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT * FROM (
SELECT key, count(1) FROM T1 GROUP BY key
@@ -4135,7 +4169,8 @@ POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1
8 2
14 1
16 2
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- group by followed by a join
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT subq1.key, subq1.cnt+subq2.cnt FROM
(SELECT key, count(1) as cnt FROM T1 GROUP BY key) subq1
@@ -4143,7 +4178,8 @@ JOIN
(SELECT key, count(1) as cnt FROM T1 GROUP BY key) subq2
ON subq1.key = subq2.key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- group by followed by a join
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT subq1.key, subq1.cnt+subq2.cnt FROM
(SELECT key, count(1) as cnt FROM T1 GROUP BY key) subq1
@@ -4485,14 +4521,16 @@ POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1
3 2
7 2
8 4
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- group by followed by a join where one of the sub-queries can be performed in the mapper
+EXPLAIN EXTENDED
SELECT * FROM
(SELECT key, count(1) FROM T1 GROUP BY key) subq1
JOIN
(SELECT key, val, count(1) FROM T1 GROUP BY key, val) subq2
ON subq1.key = subq2.key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- group by followed by a join where one of the sub-queries can be performed in the mapper
+EXPLAIN EXTENDED
SELECT * FROM
(SELECT key, count(1) FROM T1 GROUP BY key) subq1
JOIN
@@ -4945,11 +4983,13 @@ POSTHOOK: Lineage: outputtbl4.key2 SIMPL
POSTHOOK: Lineage: outputtbl4.key3 SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
POSTHOOK: Lineage: t1.key SIMPLE [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
-PREHOOK: query: INSERT OVERWRITE TABLE T2 select key, val from T1
+PREHOOK: query: -- perform an insert to make sure there are 2 files
+INSERT OVERWRITE TABLE T2 select key, val from T1
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
PREHOOK: Output: default@t2
-POSTHOOK: query: INSERT OVERWRITE TABLE T2 select key, val from T1
+POSTHOOK: query: -- perform an insert to make sure there are 2 files
+INSERT OVERWRITE TABLE T2 select key, val from T1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t1
POSTHOOK: Output: default@t2
@@ -4984,11 +5024,13 @@ POSTHOOK: Lineage: t1.key SIMPLE [(t1)t1
POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
POSTHOOK: Lineage: t2.key SIMPLE [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t2.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- no mapside sort group by if the group by is a prefix of the sorted key
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT key, count(1) FROM T2 GROUP BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- no mapside sort group by if the group by is a prefix of the sorted key
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl1
SELECT key, count(1) FROM T2 GROUP BY key
POSTHOOK: type: QUERY
@@ -5350,11 +5392,15 @@ POSTHOOK: Lineage: t2.val SIMPLE [(t1)t1
3 1
7 1
8 2
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- The plan should be converted to a map-side group by if the group by key contains a constant in between the
+-- sorted keys
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl4
SELECT key, 1, val, count(1) FROM T2 GROUP BY key, 1, val
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- The plan should be converted to a map-side group by if the group by key contains a constant in between the
+-- sorted keys
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl4
SELECT key, 1, val, count(1) FROM T2 GROUP BY key, 1, val
POSTHOOK: type: QUERY
@@ -5874,11 +5920,15 @@ POSTHOOK: Lineage: t1.key SIMPLE [(t1)t1
POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
POSTHOOK: Lineage: t2.key SIMPLE [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t2.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- The plan should be converted to a map-side group by if the group by key contains a constant in between the
+-- sorted keys followed by anything
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl5
SELECT key, 1, val, 2, count(1) FROM T2 GROUP BY key, 1, val, 2
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- The plan should be converted to a map-side group by if the group by key contains a constant in between the
+-- sorted keys followed by anything
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl5
SELECT key, 1, val, 2, count(1) FROM T2 GROUP BY key, 1, val, 2
POSTHOOK: type: QUERY
@@ -6336,13 +6386,15 @@ POSTHOOK: Lineage: t2.val SIMPLE [(t1)t1
7 1 17 2 1
8 1 18 2 1
8 1 28 2 1
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- contants from sub-queries should work fine
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl4
SELECT key, constant, val, count(1) from
(SELECT key, 1 as constant, val from T2)subq
group by key, constant, val
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- contants from sub-queries should work fine
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl4
SELECT key, constant, val, count(1) from
(SELECT key, 1 as constant, val from T2)subq
@@ -6855,7 +6907,8 @@ POSTHOOK: Lineage: t2.val SIMPLE [(t1)t1
7 1 17 1
8 1 18 1
8 1 28 1
-PREHOOK: query: EXPLAIN EXTENDED
+PREHOOK: query: -- multiple levels of contants from sub-queries should work fine
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl4
select key, constant3, val, count(1) from
(
@@ -6864,7 +6917,8 @@ SELECT key, constant as constant2, val,
)subq2
group by key, constant3, val
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN EXTENDED
+POSTHOOK: query: -- multiple levels of contants from sub-queries should work fine
+EXPLAIN EXTENDED
INSERT OVERWRITE TABLE outputTbl4
select key, constant3, val, count(1) from
(
@@ -7939,12 +7993,14 @@ POSTHOOK: Lineage: t2.val SIMPLE [(t1)t1
7 17 1
8 18 1
8 28 1
-PREHOOK: query: EXPLAIN
+PREHOOK: query: -- multi-table insert with a sub-query
+EXPLAIN
FROM (select key, val from T2 where key = 8) x
INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key
INSERT OVERWRITE TABLE DEST2 SELECT key, val, count(1) GROUP BY key, val
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN
+POSTHOOK: query: -- multi-table insert with a sub-query
+EXPLAIN
FROM (select key, val from T2 where key = 8) x
INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key
INSERT OVERWRITE TABLE DEST2 SELECT key, val, count(1) GROUP BY key, val
Modified: hive/trunk/ql/src/test/results/clientpositive/groupby_sort_test_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/groupby_sort_test_1.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/groupby_sort_test_1.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/groupby_sort_test_1.q.out Wed Apr 10 00:06:55 2013
@@ -11,11 +11,13 @@ PREHOOK: Output: default@t1
POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/T1.txt' INTO TABLE T1
POSTHOOK: type: LOAD
POSTHOOK: Output: default@t1
-PREHOOK: query: INSERT OVERWRITE TABLE T1 select key, val from T1
+PREHOOK: query: -- perform an insert to make sure there are 2 files
+INSERT OVERWRITE TABLE T1 select key, val from T1
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
PREHOOK: Output: default@t1
-POSTHOOK: query: INSERT OVERWRITE TABLE T1 select key, val from T1
+POSTHOOK: query: -- perform an insert to make sure there are 2 files
+INSERT OVERWRITE TABLE T1 select key, val from T1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t1
POSTHOOK: Output: default@t1
@@ -28,11 +30,15 @@ POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@outputTbl1
POSTHOOK: Lineage: t1.key SIMPLE [(t1)t1.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: t1.val SIMPLE [(t1)t1.FieldSchema(name:val, type:string, comment:null), ]
-PREHOOK: query: EXPLAIN
+PREHOOK: query: -- The plan should be converted to a map-side group by if the group by key
+-- matches the sorted key. However, in test mode, the group by wont be converted.
+EXPLAIN
INSERT OVERWRITE TABLE outputTbl1
SELECT key, count(1) FROM T1 GROUP BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN
+POSTHOOK: query: -- The plan should be converted to a map-side group by if the group by key
+-- matches the sorted key. However, in test mode, the group by wont be converted.
+EXPLAIN
INSERT OVERWRITE TABLE outputTbl1
SELECT key, count(1) FROM T1 GROUP BY key
POSTHOOK: type: QUERY
Modified: hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_script0.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_script0.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_script0.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_script0.q.out Wed Apr 10 00:06:55 2013
@@ -1,4 +1,7 @@
-PREHOOK: query: SELECT TRANSFORM(src.key, src.value)
+PREHOOK: query: -- checking that script operator does not cause NPE
+-- Derby strangeness is causing the output collector for the Hive Profiler to not get output during DB read
+
+SELECT TRANSFORM(src.key, src.value)
USING 'testgrep' AS (tkey, tvalue)
FROM src
PREHOOK: type: QUERY
Modified: hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_union0.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_union0.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_union0.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/hiveprofiler_union0.q.out Wed Apr 10 00:06:55 2013
@@ -7,6 +7,16 @@ SELECT unioninput.*
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
+SEL_5 ==> UNION_6: 414
+FIL_10 ==> SEL_5: 414
+TS_0 ==> FIL_9: 500
+TS_3 ==> FIL_10: 500
+UNION_6 ==> SEL_7: 498
+SEL_2 ==> UNION_6: 84
+FIL_9 ==> SEL_2: 84
+main() ==> TS_3: 500
+main() ==> TS_0: 500
+SEL_7 ==> FS_8: 498
238 val_238
86 val_86
311 val_311
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out Wed Apr 10 00:06:55 2013
@@ -1,8 +1,14 @@
-PREHOOK: query: SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
+PREHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+
+-- without indexing
+SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
+POSTHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+
+-- without indexing
+SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
@@ -41,10 +47,13 @@ POSTHOOK: Output: default@default__src_s
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: -- manual indexing
#### A masked pattern was here ####
PREHOOK: type: QUERY
PREHOOK: Input: default@default__src_src_index__
#### A masked pattern was here ####
+POSTHOOK: query: -- manual indexing
+#### A masked pattern was here ####
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__src_src_index__
#### A masked pattern was here ####
@@ -138,9 +147,11 @@ POSTHOOK: Lineage: default__src_src_inde
97 val_97
98 val_98
98 val_98
-PREHOOK: query: EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
+PREHOOK: query: -- automatic indexing
+EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
+POSTHOOK: query: -- automatic indexing
+EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_empty.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_empty.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_empty.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_empty.q.out Wed Apr 10 00:06:55 2013
@@ -1,11 +1,19 @@
-PREHOOK: query: CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
+PREHOOK: query: -- Test to ensure that an empty index result is propagated correctly
+
+-- Create temp, and populate it with some values in src.
+CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
-POSTHOOK: query: CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
+POSTHOOK: query: -- Test to ensure that an empty index result is propagated correctly
+
+-- Create temp, and populate it with some values in src.
+CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@temp
-PREHOOK: query: CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- Build an index on temp.
+CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- Build an index on temp.
+CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__temp_temp_index__
PREHOOK: query: ALTER INDEX temp_index ON temp REBUILD
@@ -19,11 +27,13 @@ POSTHOOK: Output: default@default__temp_
POSTHOOK: Lineage: default__temp_temp_index__._bucketname SIMPLE [(temp)temp.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__temp_temp_index__._offsets EXPRESSION [(temp)temp.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__temp_temp_index__.key SIMPLE [(temp)temp.FieldSchema(name:key, type:string, comment:null), ]
-PREHOOK: query: SELECT * FROM default__temp_temp_index__ WHERE key = 86
+PREHOOK: query: -- query should not return any values
+SELECT * FROM default__temp_temp_index__ WHERE key = 86
PREHOOK: type: QUERY
PREHOOK: Input: default@default__temp_temp_index__
#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM default__temp_temp_index__ WHERE key = 86
+POSTHOOK: query: -- query should not return any values
+SELECT * FROM default__temp_temp_index__ WHERE key = 86
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__temp_temp_index__
#### A masked pattern was here ####
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,8 @@
-PREHOOK: query: CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- test automatic use of index on different file formats
+CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- test automatic use of index on different file formats
+CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__src_src_index__
PREHOOK: query: ALTER INDEX src_index ON src REBUILD
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,12 @@
-PREHOOK: query: EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
+PREHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+
+-- without indexing
+EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
+POSTHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+
+-- without indexing
+EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
POSTHOOK: type: QUERY
ABSTRACT SYNTAX TREE:
(TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME srcpart) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value))) (TOK_WHERE (AND (AND (AND (> (. (TOK_TABLE_OR_COL a) key) 80) (< (. (TOK_TABLE_OR_COL a) key) 100)) (> (. (TOK_TABLE_OR_COL b) key) 70)) (< (. (TOK_TABLE_OR_COL b) key) 90))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL a) key)))))
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,12 @@
-PREHOOK: query: EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
+PREHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+
+-- without indexing
+EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
+POSTHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+
+-- without indexing
+EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
POSTHOOK: type: QUERY
ABSTRACT SYNTAX TREE:
(TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME srcpart) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value))) (TOK_WHERE (AND (AND (AND (> (. (TOK_TABLE_OR_COL a) key) 80) (< (. (TOK_TABLE_OR_COL a) key) 100)) (> (. (TOK_TABLE_OR_COL b) key) 70)) (< (. (TOK_TABLE_OR_COL b) key) 90))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL a) key)))))
@@ -220,9 +226,11 @@ POSTHOOK: Lineage: default__src_src_inde
POSTHOOK: Lineage: default__srcpart_srcpart_index__ PARTITION(ds=2008-04-09,hr=12)._bucketname SIMPLE [(srcpart)srcpart.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__srcpart_srcpart_index__ PARTITION(ds=2008-04-09,hr=12)._offsets EXPRESSION [(srcpart)srcpart.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__srcpart_srcpart_index__ PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
+PREHOOK: query: -- automatic indexing
+EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
+POSTHOOK: query: -- automatic indexing
+EXPLAIN SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,10 @@
-PREHOOK: query: CREATE INDEX src_key_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- With multiple indexes, make sure we choose which to use in a consistent order
+
+CREATE INDEX src_key_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX src_key_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- With multiple indexes, make sure we choose which to use in a consistent order
+
+CREATE INDEX src_key_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__src_src_key_index__
PREHOOK: query: CREATE INDEX src_val_index ON TABLE src(value) as 'COMPACT' WITH DEFERRED REBUILD
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,8 @@
-PREHOOK: query: CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- test automatic use of index on table with partitions
+CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- test automatic use of index on table with partitions
+CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__srcpart_src_part_index__
PREHOOK: query: ALTER INDEX src_part_index ON srcpart REBUILD
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_self_join.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_self_join.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_self_join.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_self_join.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,10 @@
-PREHOOK: query: EXPLAIN SELECT a.key, b.key FROM src a JOIN src b ON (a.value = b.value) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
+PREHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+
+EXPLAIN SELECT a.key, b.key FROM src a JOIN src b ON (a.value = b.value) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT a.key, b.key FROM src a JOIN src b ON (a.value = b.value) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
+POSTHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+
+EXPLAIN SELECT a.key, b.key FROM src a JOIN src b ON (a.value = b.value) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
POSTHOOK: type: QUERY
ABSTRACT SYNTAX TREE:
(TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME src) b) (= (. (TOK_TABLE_OR_COL a) value) (. (TOK_TABLE_OR_COL b) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) key))) (TOK_WHERE (AND (AND (AND (> (. (TOK_TABLE_OR_COL a) key) 80) (< (. (TOK_TABLE_OR_COL a) key) 100)) (> (. (TOK_TABLE_OR_COL b) key) 70)) (< (. (TOK_TABLE_OR_COL b) key) 90))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL a) key)))))
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,10 @@
-PREHOOK: query: CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- test cases where the index should not be used automatically
+
+CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- test cases where the index should not be used automatically
+
+CREATE INDEX src_index ON TABLE src(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__src_src_index__
PREHOOK: query: ALTER INDEX src_index ON src REBUILD
@@ -14,9 +18,11 @@ POSTHOOK: Output: default@default__src_s
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: EXPLAIN SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
+PREHOOK: query: -- min size too large (src is less than 5G)
+EXPLAIN SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
+POSTHOOK: query: -- min size too large (src is less than 5G)
+EXPLAIN SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
@@ -104,9 +110,11 @@ POSTHOOK: Lineage: default__src_src_inde
97 val_97
98 val_98
98 val_98
-PREHOOK: query: EXPLAIN SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
+PREHOOK: query: -- max size too small
+EXPLAIN SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
+POSTHOOK: query: -- max size too small
+EXPLAIN SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
@@ -194,9 +202,11 @@ POSTHOOK: Lineage: default__src_src_inde
97 val_97
98 val_98
98 val_98
-PREHOOK: query: EXPLAIN SELECT * FROM src WHERE key < 10 OR key > 480 ORDER BY key
+PREHOOK: query: -- OR predicate not supported by compact indexes
+EXPLAIN SELECT * FROM src WHERE key < 10 OR key > 480 ORDER BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT * FROM src WHERE key < 10 OR key > 480 ORDER BY key
+POSTHOOK: query: -- OR predicate not supported by compact indexes
+EXPLAIN SELECT * FROM src WHERE key < 10 OR key > 480 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
@@ -297,9 +307,11 @@ POSTHOOK: Lineage: default__src_src_inde
5 val_5
8 val_8
9 val_9
-PREHOOK: query: DROP INDEX src_index on src
+PREHOOK: query: -- columns are not covered by indexes
+DROP INDEX src_index on src
PREHOOK: type: DROPINDEX
-POSTHOOK: query: DROP INDEX src_index on src
+POSTHOOK: query: -- columns are not covered by indexes
+DROP INDEX src_index on src
POSTHOOK: type: DROPINDEX
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
@@ -432,9 +444,11 @@ POSTHOOK: Lineage: default__src_src_inde
POSTHOOK: Lineage: default__src_src_val_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_val_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_val_index__.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- required partitions have not been built yet
+CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- required partitions have not been built yet
+CREATE INDEX src_part_index ON TABLE srcpart(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__srcpart_src_part_index__
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_update.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_update.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_update.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_update.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,12 @@
-PREHOOK: query: CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
+PREHOOK: query: -- Test if index is actually being used.
+
+-- Create temp, and populate it with some values in src.
+CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
-POSTHOOK: query: CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
+POSTHOOK: query: -- Test if index is actually being used.
+
+-- Create temp, and populate it with some values in src.
+CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@temp
PREHOOK: query: INSERT OVERWRITE TABLE temp SELECT * FROM src WHERE key < 50
@@ -13,9 +19,11 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@temp
POSTHOOK: Lineage: temp.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp.val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- Build an index on temp.
+CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- Build an index on temp.
+CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__temp_temp_index__
POSTHOOK: Lineage: temp.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -33,9 +41,11 @@ POSTHOOK: Lineage: default__temp_temp_in
POSTHOOK: Lineage: default__temp_temp_index__.key SIMPLE [(temp)temp.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: temp.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp.val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE temp SELECT * FROM src
+PREHOOK: query: -- overwrite temp table so index is out of date
+EXPLAIN INSERT OVERWRITE TABLE temp SELECT * FROM src
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN INSERT OVERWRITE TABLE temp SELECT * FROM src
+POSTHOOK: query: -- overwrite temp table so index is out of date
+EXPLAIN INSERT OVERWRITE TABLE temp SELECT * FROM src
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__temp_temp_index__._bucketname SIMPLE [(temp)temp.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__temp_temp_index__._offsets EXPRESSION [(temp)temp.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
@@ -245,9 +255,11 @@ POSTHOOK: Lineage: default__temp_temp_in
POSTHOOK: Lineage: default__temp_temp_index__.key SIMPLE [(temp)temp.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: temp.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp.val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: EXPLAIN SELECT * FROM temp WHERE key = 86
+PREHOOK: query: -- query should return indexed values
+EXPLAIN SELECT * FROM temp WHERE key = 86
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT * FROM temp WHERE key = 86
+POSTHOOK: query: -- query should return indexed values
+EXPLAIN SELECT * FROM temp WHERE key = 86
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__temp_temp_index__._bucketname SIMPLE [(temp)temp.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__temp_temp_index__._bucketname SIMPLE [(temp)temp.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto.q.out Wed Apr 10 00:06:55 2013
@@ -1,18 +1,24 @@
-PREHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key
+PREHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+-- without indexing
+SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key
+POSTHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
+-- without indexing
+SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
0 val_0
0 val_0
0 val_0
-PREHOOK: query: EXPLAIN
+PREHOOK: query: -- create indices
+EXPLAIN
CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: EXPLAIN
+POSTHOOK: query: -- create indices
+EXPLAIN
CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
ABSTRACT SYNTAX TREE:
@@ -113,7 +119,8 @@ POSTHOOK: Lineage: default__src_src2_ind
POSTHOOK: Lineage: default__src_src2_index__._offset SIMPLE [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src2_index__.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
#### A masked pattern was here ####
-PREHOOK: query: EXPLAIN
+PREHOOK: query: -- manual indexing
+EXPLAIN
SELECT a.bucketname AS `_bucketname`, COLLECT_SET(a.offset) as `_offsets`
FROM (SELECT `_bucketname` AS bucketname, `_offset` AS offset, `_bitmaps` AS bitmaps FROM default__src_src1_index__
WHERE key = 0) a
@@ -124,7 +131,8 @@ FROM (SELECT `_bucketname` AS bucketname
a.bucketname = b.bucketname AND a.offset = b.offset WHERE NOT
EWAH_BITMAP_EMPTY(EWAH_BITMAP_AND(a.bitmaps, b.bitmaps)) GROUP BY a.bucketname
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN
+POSTHOOK: query: -- manual indexing
+EXPLAIN
SELECT a.bucketname AS `_bucketname`, COLLECT_SET(a.offset) as `_offsets`
FROM (SELECT `_bucketname` AS bucketname, `_offset` AS offset, `_bitmaps` AS bitmaps FROM default__src_src1_index__
WHERE key = 0) a
Modified: hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,8 @@
-PREHOOK: query: CREATE INDEX src_part_index ON TABLE srcpart(key) as 'BITMAP' WITH DEFERRED REBUILD
+PREHOOK: query: -- test automatic use of index on table with partitions
+CREATE INDEX src_part_index ON TABLE srcpart(key) as 'BITMAP' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX src_part_index ON TABLE srcpart(key) as 'BITMAP' WITH DEFERRED REBUILD
+POSTHOOK: query: -- test automatic use of index on table with partitions
+CREATE INDEX src_part_index ON TABLE srcpart(key) as 'BITMAP' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__srcpart_src_part_index__
PREHOOK: query: ALTER INDEX src_part_index ON srcpart REBUILD
Modified: hive/trunk/ql/src/test/results/clientpositive/index_bitmap_compression.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_bitmap_compression.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_bitmap_compression.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_bitmap_compression.q.out Wed Apr 10 00:06:55 2013
@@ -15,9 +15,11 @@ POSTHOOK: Lineage: default__src_src_inde
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offset SIMPLE [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
+PREHOOK: query: -- automatic indexing
+EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
+POSTHOOK: query: -- automatic indexing
+EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__src_src_index__._bitmaps EXPRESSION [(src)src.FieldSchema(name:ROW__OFFSET__INSIDE__BLOCK, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_compression.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_compression.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_compression.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_compression.q.out Wed Apr 10 00:06:55 2013
@@ -14,9 +14,11 @@ POSTHOOK: Output: default@default__src_s
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
+PREHOOK: query: -- automatic indexing
+EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
+POSTHOOK: query: -- automatic indexing
+EXPLAIN SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_stale.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_stale.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_stale.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_stale.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,10 @@
-PREHOOK: query: CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
+PREHOOK: query: -- test that stale indexes are not used
+
+CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
-POSTHOOK: query: CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
+POSTHOOK: query: -- test that stale indexes are not used
+
+CREATE TABLE temp(key STRING, val STRING) STORED AS TEXTFILE
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@temp
PREHOOK: query: INSERT OVERWRITE TABLE temp SELECT * FROM src WHERE key < 50
@@ -13,9 +17,11 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@temp
POSTHOOK: Lineage: temp.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp.val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- Build an index on temp.
+CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- Build an index on temp.
+CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__temp_temp_index__
POSTHOOK: Lineage: temp.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -33,11 +39,13 @@ POSTHOOK: Lineage: default__temp_temp_in
POSTHOOK: Lineage: default__temp_temp_index__.key SIMPLE [(temp)temp.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: temp.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp.val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: INSERT OVERWRITE TABLE temp SELECT * FROM src
+PREHOOK: query: -- overwrite temp table so index is out of date
+INSERT OVERWRITE TABLE temp SELECT * FROM src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@temp
-POSTHOOK: query: INSERT OVERWRITE TABLE temp SELECT * FROM src
+POSTHOOK: query: -- overwrite temp table so index is out of date
+INSERT OVERWRITE TABLE temp SELECT * FROM src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@temp
@@ -48,9 +56,11 @@ POSTHOOK: Lineage: temp.key SIMPLE [(src
POSTHOOK: Lineage: temp.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp.val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: temp.val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: EXPLAIN SELECT * FROM temp WHERE key = 86
+PREHOOK: query: -- should return correct results bypassing index
+EXPLAIN SELECT * FROM temp WHERE key = 86
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT * FROM temp WHERE key = 86
+POSTHOOK: query: -- should return correct results bypassing index
+EXPLAIN SELECT * FROM temp WHERE key = 86
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__temp_temp_index__._bucketname SIMPLE [(temp)temp.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__temp_temp_index__._offsets EXPRESSION [(temp)temp.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_stale_partitioned.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_stale_partitioned.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_stale_partitioned.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_stale_partitioned.q.out Wed Apr 10 00:06:55 2013
@@ -1,6 +1,12 @@
-PREHOOK: query: CREATE TABLE temp(key STRING, val STRING) PARTITIONED BY (foo string) STORED AS TEXTFILE
+PREHOOK: query: -- Test if index is actually being used.
+
+-- Create temp, and populate it with some values in src.
+CREATE TABLE temp(key STRING, val STRING) PARTITIONED BY (foo string) STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
-POSTHOOK: query: CREATE TABLE temp(key STRING, val STRING) PARTITIONED BY (foo string) STORED AS TEXTFILE
+POSTHOOK: query: -- Test if index is actually being used.
+
+-- Create temp, and populate it with some values in src.
+CREATE TABLE temp(key STRING, val STRING) PARTITIONED BY (foo string) STORED AS TEXTFILE
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@temp
PREHOOK: query: ALTER TABLE temp ADD PARTITION (foo = 'bar')
@@ -20,9 +26,11 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@temp@foo=bar
POSTHOOK: Lineage: temp PARTITION(foo=bar).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp PARTITION(foo=bar).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: query: -- Build an index on temp.
+CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
PREHOOK: type: CREATEINDEX
-POSTHOOK: query: CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: query: -- Build an index on temp.
+CREATE INDEX temp_index ON TABLE temp(key) as 'COMPACT' WITH DEFERRED REBUILD
POSTHOOK: type: CREATEINDEX
POSTHOOK: Output: default@default__temp_temp_index__
POSTHOOK: Lineage: temp PARTITION(foo=bar).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -42,11 +50,13 @@ POSTHOOK: Lineage: default__temp_temp_in
POSTHOOK: Lineage: default__temp_temp_index__ PARTITION(foo=bar).key SIMPLE [(temp)temp.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: temp PARTITION(foo=bar).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp PARTITION(foo=bar).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: INSERT OVERWRITE TABLE temp PARTITION (foo = 'bar') SELECT * FROM src
+PREHOOK: query: -- overwrite temp table so index is out of date
+INSERT OVERWRITE TABLE temp PARTITION (foo = 'bar') SELECT * FROM src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@temp@foo=bar
-POSTHOOK: query: INSERT OVERWRITE TABLE temp PARTITION (foo = 'bar') SELECT * FROM src
+POSTHOOK: query: -- overwrite temp table so index is out of date
+INSERT OVERWRITE TABLE temp PARTITION (foo = 'bar') SELECT * FROM src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@temp@foo=bar
@@ -57,12 +67,14 @@ POSTHOOK: Lineage: temp PARTITION(foo=ba
POSTHOOK: Lineage: temp PARTITION(foo=bar).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: temp PARTITION(foo=bar).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: temp PARTITION(foo=bar).val SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: SELECT * FROM default__temp_temp_index__ WHERE key = 86 AND foo='bar'
+PREHOOK: query: -- query should not return any values
+SELECT * FROM default__temp_temp_index__ WHERE key = 86 AND foo='bar'
PREHOOK: type: QUERY
PREHOOK: Input: default@default__temp_temp_index__
PREHOOK: Input: default@default__temp_temp_index__@foo=bar
#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM default__temp_temp_index__ WHERE key = 86 AND foo='bar'
+POSTHOOK: query: -- query should not return any values
+SELECT * FROM default__temp_temp_index__ WHERE key = 86 AND foo='bar'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__temp_temp_index__
POSTHOOK: Input: default@default__temp_temp_index__@foo=bar
Modified: hive/trunk/ql/src/test/results/clientpositive/infer_bucket_sort.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/infer_bucket_sort.q.out?rev=1466300&r1=1466299&r2=1466300&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/infer_bucket_sort.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/infer_bucket_sort.q.out Wed Apr 10 00:06:55 2013
@@ -1,14 +1,22 @@
-PREHOOK: query: CREATE TABLE test_table (key STRING, value STRING) PARTITIONED BY (part STRING)
+PREHOOK: query: -- This tests inferring how data is bucketed/sorted from the operators in the reducer
+-- and populating that information in partitions' metadata
+
+CREATE TABLE test_table (key STRING, value STRING) PARTITIONED BY (part STRING)
PREHOOK: type: CREATETABLE
-POSTHOOK: query: CREATE TABLE test_table (key STRING, value STRING) PARTITIONED BY (part STRING)
+POSTHOOK: query: -- This tests inferring how data is bucketed/sorted from the operators in the reducer
+-- and populating that information in partitions' metadata
+
+CREATE TABLE test_table (key STRING, value STRING) PARTITIONED BY (part STRING)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@test_table
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by, should be bucketed and sorted by group by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, count(*) FROM src GROUP BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by, should be bucketed and sorted by group by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, count(*) FROM src GROUP BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -55,12 +63,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by where a key isn't selected, should not be bucketed or sorted
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, count(*) FROM src GROUP BY key, value
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by where a key isn't selected, should not be bucketed or sorted
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, count(*) FROM src GROUP BY key, value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -111,12 +121,14 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test join, should be bucketed and sorted by join key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, a.value FROM src a JOIN src b ON a.key = b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test join, should be bucketed and sorted by join key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, a.value FROM src a JOIN src b ON a.key = b.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -171,12 +183,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test join with two keys, should be bucketed and sorted by join keys
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, a.value FROM src a JOIN src b ON a.key = b.key AND a.value = b.value
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test join with two keys, should be bucketed and sorted by join keys
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, a.value FROM src a JOIN src b ON a.key = b.key AND a.value = b.value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -235,12 +249,14 @@ Bucket Columns: [key, value]
Sort Columns: [Order(col:key, order:1), Order(col:value, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test join with two keys and only one selected, should not be bucketed or sorted
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, '1' FROM src a JOIN src b ON a.key = b.key AND a.value = b.value
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test join with two keys and only one selected, should not be bucketed or sorted
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, '1' FROM src a JOIN src b ON a.key = b.key AND a.value = b.value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -303,12 +319,14 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test join on three tables on same key, should be bucketed and sorted by join key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, c.value FROM src a JOIN src b ON (a.key = b.key) JOIN src c ON (b.key = c.key)
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test join on three tables on same key, should be bucketed and sorted by join key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, c.value FROM src a JOIN src b ON (a.key = b.key) JOIN src c ON (b.key = c.key)
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -375,12 +393,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test join on three tables on different keys, should be bucketed and sorted by latter key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, c.value FROM src a JOIN src b ON (a.key = b.key) JOIN src c ON (b.value = c.value)
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test join on three tables on different keys, should be bucketed and sorted by latter key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT a.key, c.value FROM src a JOIN src b ON (a.key = b.key) JOIN src c ON (b.value = c.value)
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -451,12 +471,14 @@ Bucket Columns: [value]
Sort Columns: [Order(col:value, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test distribute by, should only be bucketed by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src DISTRIBUTE BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test distribute by, should only be bucketed by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src DISTRIBUTE BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -531,12 +553,14 @@ Bucket Columns: [key]
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test sort by, should be sorted by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src SORT BY key ASC
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test sort by, should be sorted by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src SORT BY key ASC
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -615,12 +639,14 @@ Bucket Columns: []
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test sort by desc, should be sorted by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src SORT BY key DESC
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test sort by desc, should be sorted by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src SORT BY key DESC
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -703,12 +729,14 @@ Bucket Columns: []
Sort Columns: [Order(col:key, order:0)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test cluster by, should be bucketed and sorted by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src CLUSTER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test cluster by, should be bucketed and sorted by key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src CLUSTER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -795,12 +823,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test distribute by and sort by different keys, should be bucketed by one key sorted by the other
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src DISTRIBUTE BY key SORT BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test distribute by and sort by different keys, should be bucketed by one key sorted by the other
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM src DISTRIBUTE BY key SORT BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -891,12 +921,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:value, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test join in simple subquery, should be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value from (SELECT a.key, b.value FROM src a JOIN src b ON (a.key = b.key)) subq
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test join in simple subquery, should be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value from (SELECT a.key, b.value FROM src a JOIN src b ON (a.key = b.key)) subq
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -991,12 +1023,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test join in simple subquery renaming key column, should be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT k, value FROM (SELECT a.key as k, b.value FROM src a JOIN src b ON (a.key = b.key)) subq
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test join in simple subquery renaming key column, should be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT k, value FROM (SELECT a.key as k, b.value FROM src a JOIN src b ON (a.key = b.key)) subq
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -1095,12 +1129,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in simple subquery, should be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, cnt from (SELECT key, count(*) as cnt FROM src GROUP BY key) subq
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in simple subquery, should be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, cnt from (SELECT key, count(*) as cnt FROM src GROUP BY key) subq
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -1203,12 +1239,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in simple subquery renaming key column, should be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT k, cnt FROM (SELECT key as k, count(*) as cnt FROM src GROUP BY key) subq
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in simple subquery renaming key column, should be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT k, cnt FROM (SELECT key as k, count(*) as cnt FROM src GROUP BY key) subq
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -1315,12 +1353,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in subquery with where outside, should still be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT key, count(1) AS value FROM src group by key) a where key < 10
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in subquery with where outside, should still be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT key, count(1) AS value FROM src group by key) a where key < 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -1431,12 +1471,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in subquery with expression on value, should still be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value + 1 FROM (SELECT key, count(1) AS value FROM src group by key) a where key < 10
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in subquery with expression on value, should still be bucketed and sorted on key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value + 1 FROM (SELECT key, count(1) AS value FROM src group by key) a where key < 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -1551,12 +1593,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in subquery with lateral view outside, should still be bucketed and sorted
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT key FROM src group by key) a lateral view explode(array(1, 2)) value as value
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in subquery with lateral view outside, should still be bucketed and sorted
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT key FROM src group by key) a lateral view explode(array(1, 2)) value as value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -1675,12 +1719,16 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in subquery with another group by outside, should be bucketed and sorted by the
+-- key of the outer group by
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT count(1), value FROM (SELECT key, count(1) as value FROM src group by key) a group by value
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in subquery with another group by outside, should be bucketed and sorted by the
+-- key of the outer group by
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT count(1), value FROM (SELECT key, count(1) as value FROM src group by key) a group by value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -1803,12 +1851,16 @@ Bucket Columns: [value]
Sort Columns: [Order(col:value, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in subquery with select on outside reordering the columns, should be bucketed and
+-- sorted by the column the group by key ends up in
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT value, key FROM (SELECT key, count(1) as value FROM src group by key) a
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in subquery with select on outside reordering the columns, should be bucketed and
+-- sorted by the column the group by key ends up in
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT value, key FROM (SELECT key, count(1) as value FROM src group by key) a
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -1935,12 +1987,14 @@ Bucket Columns: [value]
Sort Columns: [Order(col:value, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in subquery followed by distribute by, should only be bucketed by the distribute key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT key, count(1) as value FROM src group by key) a distribute by key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in subquery followed by distribute by, should only be bucketed by the distribute key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT key, count(1) as value FROM src group by key) a distribute by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -2071,12 +2125,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in subquery followed by sort by, should only be sorted by the sort key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT key, count(1) as value FROM src group by key) a sort by key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in subquery followed by sort by, should only be sorted by the sort key
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT key, count(1) as value FROM src group by key) a sort by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -2211,12 +2267,14 @@ Bucket Columns: [key]
Sort Columns: [Order(col:key, order:1)]
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by in subquery followed by transform script, should not be bucketed or sorted
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT TRANSFORM (a.key, a.value) USING 'cat' AS (key, value) FROM (SELECT key, count(1) AS value FROM src GROUP BY KEY) a
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by in subquery followed by transform script, should not be bucketed or sorted
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT TRANSFORM (a.key, a.value) USING 'cat' AS (key, value) FROM (SELECT key, count(1) AS value FROM src GROUP BY KEY) a
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -2355,12 +2413,14 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+PREHOOK: query: -- Test group by on function, should be bucketed and sorted by key and value because the function is applied in the mapper
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT concat(key, "a") AS key, value, count(*) FROM src GROUP BY concat(key, "a"), value) a
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_table@part=1
-POSTHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
+POSTHOOK: query: -- Test group by on function, should be bucketed and sorted by key and value because the function is applied in the mapper
+INSERT OVERWRITE TABLE test_table PARTITION (part = '1')
SELECT key, value FROM (SELECT concat(key, "a") AS key, value, count(*) FROM src GROUP BY concat(key, "a"), value) a
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src