You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2022/01/13 08:29:57 UTC
[hive] branch master updated: HIVE-21074: Hive bucketed table query pruning does not work for IS NOT NULL condition (#2931) (Adam Szita, originally contributed by Thai Bui, reviewed by Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
szita pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 856f6b4 HIVE-21074: Hive bucketed table query pruning does not work for IS NOT NULL condition (#2931) (Adam Szita, originally contributed by Thai Bui, reviewed by Peter Vary)
856f6b4 is described below
commit 856f6b4260d7b39342b2f82cb668fc427664a054
Author: Adam Szita <40...@users.noreply.github.com>
AuthorDate: Thu Jan 13 09:29:45 2022 +0100
HIVE-21074: Hive bucketed table query pruning does not work for IS NOT NULL condition (#2931) (Adam Szita, originally contributed by Thai Bui, reviewed by Peter Vary)
---
.../ql/optimizer/FixedBucketPruningOptimizer.java | 17 +++---
.../test/queries/clientpositive/bucketpruning1.q | 5 ++
.../clientpositive/llap/bucketpruning1.q.out | 71 ++++++++++++++++++++++
.../llap/tez_fixed_bucket_pruning.q.out | 1 +
4 files changed, 85 insertions(+), 9 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java
index 4bf05f3..0eea698 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java
@@ -165,16 +165,15 @@ public class FixedBucketPruningOptimizer extends Transform {
} else if (expr.getOperator() == Operator.AND) {
boolean found = false;
for (ExpressionTree subExpr : expr.getChildren()) {
- if (subExpr.getOperator() != Operator.LEAF) {
- return;
- }
- // one of the branches is definitely a bucket-leaf
- PredicateLeaf l = leaves.get(subExpr.getLeaf());
- if (bucketLeaves.contains(l)) {
- if (!addLiteral(literals, l)) {
- return;
+ if (subExpr.getOperator() == Operator.LEAF) {
+ // one of the branches is definitely a bucket-leaf
+ PredicateLeaf l = leaves.get(subExpr.getLeaf());
+ if (bucketLeaves.contains(l)) {
+ if (!addLiteral(literals, l)) {
+ return;
+ }
+ found = true;
}
- found = true;
}
}
if (!found) {
diff --git a/ql/src/test/queries/clientpositive/bucketpruning1.q b/ql/src/test/queries/clientpositive/bucketpruning1.q
index 0f797f7..d867241 100644
--- a/ql/src/test/queries/clientpositive/bucketpruning1.q
+++ b/ql/src/test/queries/clientpositive/bucketpruning1.q
@@ -10,6 +10,11 @@ CREATE TABLE srcbucket_pruned(key int, value string) partitioned by (ds string)
-- cannot prune 2-key scenarios without a smarter optimizer
CREATE TABLE srcbucket_unpruned(key int, value string) partitioned by (ds string) CLUSTERED BY (key,value) INTO 16 BUCKETS STORED AS TEXTFILE;
+-- valid AND cases: when an AND condition is a bucket column, then pruning should work
+
+explain extended
+select * from srcbucket_pruned where key = 1 and value is not null;
+
-- good cases
explain extended
diff --git a/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out b/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out
index 82e4c75..7464761 100644
--- a/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out
@@ -15,6 +15,77 @@ POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@srcbucket_unpruned
PREHOOK: query: explain extended
+select * from srcbucket_pruned where key = 1 and value is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_pruned
+#### A masked pattern was here ####
+POSTHOOK: query: explain extended
+select * from srcbucket_pruned where key = 1 and value is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_pruned
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT CAST(1 AS INT) AS `key`, `value`, `ds`
+FROM `default`.`srcbucket_pruned`
+WHERE `key` = 1 AND `value` IS NOT NULL
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcbucket_pruned
+ filterExpr: ((key = 1) and value is not null) (type: boolean)
+ buckets included: [13,] of 16
+ Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((key = 1) and value is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 268 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 1 (type: int), value (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0,_col1,_col2
+ columns.types int:string:string
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Execution mode: vectorized, llap
+ LLAP IO: unknown
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain extended
select * from srcbucket_pruned where key = 1
PREHOOK: type: QUERY
PREHOOK: Input: default@srcbucket_pruned
diff --git a/ql/src/test/results/clientpositive/llap/tez_fixed_bucket_pruning.q.out b/ql/src/test/results/clientpositive/llap/tez_fixed_bucket_pruning.q.out
index ad08f6b..2f12630 100644
--- a/ql/src/test/results/clientpositive/llap/tez_fixed_bucket_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_fixed_bucket_pruning.q.out
@@ -1126,6 +1126,7 @@ STAGE PLANS:
TableScan
alias: s1
filterExpr: ((idp_data_date = DATE'2017-12-28') and finplan_detail_object_id is not null and l3_snapshot_number is not null) (type: boolean)
+ buckets included: [50,] of 64
Statistics: Num rows: 180340 Data size: 14427200 Basic stats: COMPLETE Column stats: COMPLETE
GatherStats: false
Filter Operator