You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2016/08/22 22:33:30 UTC
hive git commit: HIVE-14199 Enable Bucket Pruning for ACID tables
(Saket Saurabh via Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master 39c858c77 -> b2338ff4e
HIVE-14199 Enable Bucket Pruning for ACID tables (Saket Saurabh via Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b2338ff4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b2338ff4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b2338ff4
Branch: refs/heads/master
Commit: b2338ff4ec6ce3b3bf3478b82d6e4b5d5ec8ca8e
Parents: 39c858c
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Mon Aug 22 15:33:16 2016 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Mon Aug 22 15:33:16 2016 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 3 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 10 ++
.../clientpositive/acid_bucket_pruning.q | 21 +++
.../tez/acid_bucket_pruning.q.out | 151 +++++++++++++++++++
4 files changed, 184 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b2338ff4/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 16c09da..662ae3e 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -391,7 +391,8 @@ minitez.query.files.shared=acid_globallimit.q,\
union_type_chk.q
-minitez.query.files=acid_vectorization_missing_cols.q,\
+minitez.query.files=acid_bucket_pruning.q,\
+ acid_vectorization_missing_cols.q,\
bucket_map_join_tez1.q,\
smb_cache.q,\
bucket_map_join_tez2.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/b2338ff4/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index c97c335..a542dc4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1683,6 +1683,16 @@ public final class Utilities {
}
return Integer.parseInt(m.group(2));
}
+ // Check to see if the bucketName matches the pattern "bucket_([0-9]+).*"
+ // This can happen in ACID cases when we have splits on delta files, where the filenames
+ // are of the form delta_x_y/bucket_a.
+ if (bucketName.startsWith(AcidUtils.BUCKET_PREFIX)) {
+ m = AcidUtils.BUCKET_DIGIT_PATTERN.matcher(bucketName);
+ if (m.find()) {
+ return Integer.parseInt(m.group());
+ }
+ // Note that legacy bucket digit pattern are being ignored here.
+ }
return -1;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b2338ff4/ql/src/test/queries/clientpositive/acid_bucket_pruning.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/acid_bucket_pruning.q b/ql/src/test/queries/clientpositive/acid_bucket_pruning.q
new file mode 100644
index 0000000..24f8de1
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/acid_bucket_pruning.q
@@ -0,0 +1,21 @@
+set hive.mapred.mode=nonstrict;
+set hive.optimize.ppd=true;
+set hive.optimize.index.filter=true;
+set hive.tez.bucket.pruning=true;
+set hive.explain.user=false;
+set hive.fetch.task.conversion=none;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+-- Bucket pruning only works for ACID when split-update (U=D+I) has been enabled for the table.
+-- For e.g., this can be done by setting 'transactional_properties' = 'default'.
+-- This also means that bucket pruning will not work for ACID tables with legacy behaviour.
+
+CREATE TABLE acidTblDefault(a INT) CLUSTERED BY(a) INTO 16 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default');
+INSERT INTO TABLE acidTblDefault SELECT cint FROM alltypesorc WHERE cint IS NOT NULL ORDER BY cint;
+INSERT INTO TABLE acidTblDefault VALUES (1);
+
+-- Exactly one of the buckets should be selected out of the 16 buckets
+-- by the following selection query.
+EXPLAIN EXTENDED
+SELECT * FROM acidTblDefault WHERE a = 1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b2338ff4/ql/src/test/results/clientpositive/tez/acid_bucket_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/acid_bucket_pruning.q.out b/ql/src/test/results/clientpositive/tez/acid_bucket_pruning.q.out
new file mode 100644
index 0000000..e71bc12
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/acid_bucket_pruning.q.out
@@ -0,0 +1,151 @@
+PREHOOK: query: -- Bucket pruning only works for ACID when split-update (U=D+I) has been enabled for the table.
+-- For e.g., this can be done by setting 'transactional_properties' = 'default'.
+-- This also means that bucket pruning will not work for ACID tables with legacy behaviour.
+
+CREATE TABLE acidTblDefault(a INT) CLUSTERED BY(a) INTO 16 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acidTblDefault
+POSTHOOK: query: -- Bucket pruning only works for ACID when split-update (U=D+I) has been enabled for the table.
+-- For e.g., this can be done by setting 'transactional_properties' = 'default'.
+-- This also means that bucket pruning will not work for ACID tables with legacy behaviour.
+
+CREATE TABLE acidTblDefault(a INT) CLUSTERED BY(a) INTO 16 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acidTblDefault
+PREHOOK: query: INSERT INTO TABLE acidTblDefault SELECT cint FROM alltypesorc WHERE cint IS NOT NULL ORDER BY cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@acidtbldefault
+POSTHOOK: query: INSERT INTO TABLE acidTblDefault SELECT cint FROM alltypesorc WHERE cint IS NOT NULL ORDER BY cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@acidtbldefault
+POSTHOOK: Lineage: acidtbldefault.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+PREHOOK: query: INSERT INTO TABLE acidTblDefault VALUES (1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@acidtbldefault
+POSTHOOK: query: INSERT INTO TABLE acidTblDefault VALUES (1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@acidtbldefault
+POSTHOOK: Lineage: acidtbldefault.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: -- Exactly one of the buckets should be selected out of the 16 buckets
+-- by the following selection query.
+EXPLAIN EXTENDED
+SELECT * FROM acidTblDefault WHERE a = 1
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Exactly one of the buckets should be selected out of the 16 buckets
+-- by the following selection query.
+EXPLAIN EXTENDED
+SELECT * FROM acidTblDefault WHERE a = 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: acidtbldefault
+ filterExpr: (a = 1) (type: boolean)
+ buckets included: [1,] of 16
+ Statistics: Num rows: 8983 Data size: 35932 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (a = 1) (type: boolean)
+ Statistics: Num rows: 4491 Data size: 17964 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 4491 Data size: 17964 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 4491 Data size: 17964 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types int
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: acidtbldefault
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ properties:
+ bucket_count 16
+ bucket_field_name a
+ columns a
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ name default.acidtbldefault
+ numFiles 17
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct acidtbldefault { i32 a}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ totalSize 35932
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ properties:
+ bucket_count 16
+ bucket_field_name a
+ columns a
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ name default.acidtbldefault
+ numFiles 17
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct acidtbldefault { i32 a}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ totalSize 35932
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbldefault
+ name: default.acidtbldefault
+ Truncated Path -> Alias:
+ /acidtbldefault [acidtbldefault]
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+