You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2015/12/02 21:40:28 UTC
hive git commit: HIVE-12444 - Global Limit optimization on ACID table
without base directory may throw exception ADDENDUM (Wei Zheng via Eugene
Koifman)
Repository: hive
Updated Branches:
refs/heads/master 0bab072d1 -> 1c69604c6
HIVE-12444 - Global Limit optimization on ACID table without base directory may throw exception ADDENDUM (Wei Zheng via Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1c69604c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1c69604c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1c69604c
Branch: refs/heads/master
Commit: 1c69604c684d03b7eca9ac5ff4365077b78ab8be
Parents: 0bab072
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Wed Dec 2 12:40:11 2015 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Wed Dec 2 12:40:11 2015 -0800
----------------------------------------------------------------------
.../queries/clientpositive/acid_globallimit.q | 18 ++++
.../clientpositive/acid_globallimit.q.out | 90 ++++++++++++++++++++
.../clientpositive/tez/acid_globallimit.q.out | 80 +++++++++++++++++
3 files changed, 188 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1c69604c/ql/src/test/queries/clientpositive/acid_globallimit.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/acid_globallimit.q b/ql/src/test/queries/clientpositive/acid_globallimit.q
new file mode 100644
index 0000000..82d8af0
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/acid_globallimit.q
@@ -0,0 +1,18 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.fetch.task.conversion=none;
+set hive.limit.optimize.enable=true;
+
+-- Global Limit optimization does not work with ACID table. Make sure to skip it for ACID table.
+CREATE TABLE acidtest1(c1 INT, c2 STRING)
+CLUSTERED BY (c1) INTO 3 BUCKETS
+STORED AS ORC
+TBLPROPERTIES ("transactional"="true");
+
+insert into table acidtest1 select cint, cstring1 from alltypesorc where cint is not null order by cint;
+
+explain
+select cast (c1 as string) from acidtest1 limit 10;
+select cast (c1 as string) from acidtest1 limit 10;
+
+drop table acidtest1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/1c69604c/ql/src/test/results/clientpositive/acid_globallimit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_globallimit.q.out b/ql/src/test/results/clientpositive/acid_globallimit.q.out
new file mode 100644
index 0000000..6a2a792
--- /dev/null
+++ b/ql/src/test/results/clientpositive/acid_globallimit.q.out
@@ -0,0 +1,90 @@
+PREHOOK: query: -- Global Limit optimization does not work with ACID table. Make sure to skip it for ACID table.
+CREATE TABLE acidtest1(c1 INT, c2 STRING)
+CLUSTERED BY (c1) INTO 3 BUCKETS
+STORED AS ORC
+TBLPROPERTIES ("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acidtest1
+POSTHOOK: query: -- Global Limit optimization does not work with ACID table. Make sure to skip it for ACID table.
+CREATE TABLE acidtest1(c1 INT, c2 STRING)
+CLUSTERED BY (c1) INTO 3 BUCKETS
+STORED AS ORC
+TBLPROPERTIES ("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acidtest1
+PREHOOK: query: insert into table acidtest1 select cint, cstring1 from alltypesorc where cint is not null order by cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@acidtest1
+POSTHOOK: query: insert into table acidtest1 select cint, cstring1 from alltypesorc where cint is not null order by cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@acidtest1
+POSTHOOK: Lineage: acidtest1.c1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: acidtest1.c2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+PREHOOK: query: explain
+select cast (c1 as string) from acidtest1 limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select cast (c1 as string) from acidtest1 limit 10
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: acidtest1
+ Statistics: Num rows: 9173 Data size: 101822 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToString(c1) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 9173 Data size: 101822 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select cast (c1 as string) from acidtest1 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acidtest1
+#### A masked pattern was here ####
+POSTHOOK: query: select cast (c1 as string) from acidtest1 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acidtest1
+#### A masked pattern was here ####
+185520768
+186950964
+186967185
+187206627
+187503456
+188474907
+190070046
+190435023
+190587882
+191372331
+PREHOOK: query: drop table acidtest1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@acidtest1
+PREHOOK: Output: default@acidtest1
+POSTHOOK: query: drop table acidtest1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@acidtest1
+POSTHOOK: Output: default@acidtest1
http://git-wip-us.apache.org/repos/asf/hive/blob/1c69604c/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out b/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out
new file mode 100644
index 0000000..fdbddb0
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/acid_globallimit.q.out
@@ -0,0 +1,80 @@
+PREHOOK: query: -- Global Limit optimization does not work with ACID table. Make sure to skip it for ACID table.
+CREATE TABLE acidtest1(c1 INT, c2 STRING)
+CLUSTERED BY (c1) INTO 3 BUCKETS
+STORED AS ORC
+TBLPROPERTIES ("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acidtest1
+POSTHOOK: query: -- Global Limit optimization does not work with ACID table. Make sure to skip it for ACID table.
+CREATE TABLE acidtest1(c1 INT, c2 STRING)
+CLUSTERED BY (c1) INTO 3 BUCKETS
+STORED AS ORC
+TBLPROPERTIES ("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acidtest1
+PREHOOK: query: insert into table acidtest1 select cint, cstring1 from alltypesorc where cint is not null order by cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@acidtest1
+POSTHOOK: query: insert into table acidtest1 select cint, cstring1 from alltypesorc where cint is not null order by cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@acidtest1
+POSTHOOK: Lineage: acidtest1.c1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: acidtest1.c2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+PREHOOK: query: explain
+select cast (c1 as string) from acidtest1 limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select cast (c1 as string) from acidtest1 limit 10
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Stage-0
+ Fetch Operator
+ limit:10
+ Stage-1
+ Map 1
+ File Output Operator [FS_3]
+ compressed:false
+ Statistics:Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"}
+ Limit [LIM_2]
+ Number of rows:10
+ Statistics:Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator [SEL_1]
+ outputColumnNames:["_col0"]
+ Statistics:Num rows: 9173 Data size: 100794 Basic stats: COMPLETE Column stats: NONE
+ TableScan [TS_0]
+ ACID table:true
+ alias:acidtest1
+ Statistics:Num rows: 9173 Data size: 100794 Basic stats: COMPLETE Column stats: NONE
+
+PREHOOK: query: select cast (c1 as string) from acidtest1 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acidtest1
+#### A masked pattern was here ####
+POSTHOOK: query: select cast (c1 as string) from acidtest1 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acidtest1
+#### A masked pattern was here ####
+-340178543
+-339581189
+-339244391
+356535438
+-338184935
+-338131778
+356416560
+844686816
+355274340
+-821957276
+PREHOOK: query: drop table acidtest1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@acidtest1
+PREHOOK: Output: default@acidtest1
+POSTHOOK: query: drop table acidtest1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@acidtest1
+POSTHOOK: Output: default@acidtest1