You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2025/11/17 08:44:05 UTC
(hive) branch branch-4.2 updated (8094bd5cac1 -> eed6cd8ab45)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a change to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/hive.git
from 8094bd5cac1 HIVE-29318: PostgreSQL requires every NULL parameter to have an explicit SQL type (#6186)
new 1d9323a33e6 HIVE-29290: Wrong results when n-way join contains both anti and outer join (#6173)
new c6e577ca1fa HIVE-29304: HS2 fails to start if cluster is configured with a mix of LDAP and Kerberos (#6166)
new 8aae2dd8585 HIVE-29176: Wrong result when HiveAntiJoin is replacing an IS NULL filter on a nullable column (#6062)
new dec805a6a0a HIVE-29254: Display TxnId associated with the query in show processlist command (#6141)
new eed6cd8ab45 HIVE-29277: Invalid index should return null for get_json_object (#6144)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hive/ql/processor/TestShowProcessList.java | 144 ++++++++++
.../hive/jdbc/ZooKeeperHiveClientHelper.java | 2 +-
.../hadoop/hive/ql/exec/CommonJoinOperator.java | 8 +-
.../hive/ql/exec/CommonMergeJoinOperator.java | 3 +-
.../hive/ql/optimizer/calcite/HiveCalciteUtil.java | 44 +--
.../calcite/rules/HiveAntiSemiJoinRule.java | 175 +++++++++---
.../ql/processors/ShowProcessListProcessor.java | 2 +
.../hadoop/hive/ql/session/ProcessListInfo.java | 16 +-
.../org/apache/hadoop/hive/ql/udf/UDFJson.java | 2 +
.../calcite/rules/TestHiveAntiSemiJoinRule.java | 248 +++++++++++++++++
.../ql/optimizer/calcite/rules/TestRuleHelper.java | 113 ++++++--
.../org/apache/hadoop/hive/ql/udf/TestUDFJson.java | 3 +
ql/src/test/queries/clientpositive/antijoin3.q | 22 ++
ql/src/test/queries/clientpositive/antijoin4.q | 64 +++++
.../queries/clientpositive/udf_get_json_object.q | 2 +
.../results/clientpositive/llap/antijoin3.q.out | 250 ++++++++++++++++++
.../results/clientpositive/llap/antijoin4.q.out | 294 +++++++++++++++++++++
.../results/clientpositive/llap/llap_smb_ptf.q.out | 47 ++--
.../clientpositive/llap/udf_get_json_object.q.out | 6 +-
.../cli/operation/ShowProcessListOperation.java | 6 +-
.../apache/hive/service/server/HiveServer2.java | 5 +-
21 files changed, 1332 insertions(+), 124 deletions(-)
create mode 100644 itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/processor/TestShowProcessList.java
create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestHiveAntiSemiJoinRule.java
create mode 100644 ql/src/test/queries/clientpositive/antijoin3.q
create mode 100644 ql/src/test/queries/clientpositive/antijoin4.q
create mode 100644 ql/src/test/results/clientpositive/llap/antijoin3.q.out
create mode 100644 ql/src/test/results/clientpositive/llap/antijoin4.q.out
(hive) 01/05: HIVE-29290: Wrong results when n-way join contains both anti and outer join (#6173)
Posted by dk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 1d9323a33e6d7fe1f649ec83f683fda25b4ff9bc
Author: Krisztian Kasa <ka...@gmail.com>
AuthorDate: Wed Nov 12 06:43:36 2025 +0100
HIVE-29290: Wrong results when n-way join contains both anti and outer join (#6173)
(cherry picked from commit c054a47c7c22875c93ca0d29b7ac730cf7deba04)
---
.../hadoop/hive/ql/exec/CommonJoinOperator.java | 8 +-
.../hive/ql/exec/CommonMergeJoinOperator.java | 3 +-
ql/src/test/queries/clientpositive/antijoin3.q | 22 ++
.../results/clientpositive/llap/antijoin3.q.out | 250 +++++++++++++++++++++
4 files changed, 281 insertions(+), 2 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
index 58ae7f3ef42..bb0e274212e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
@@ -990,7 +990,13 @@ protected void checkAndGenObject() throws HiveException {
} else {
if (!alw.hasRows()) {
hasEmpty = true;
- alw.addRow(dummyObj[i]);
+ if (!isRightOfAntiJoin) {
+ alw.addRow(dummyObj[i]);
+ }
+ } else if (isRightOfAntiJoin && !needsPostEvaluation) {
+ // For anti join the right side should be empty. For needsPostEvaluation case we will
+ // wait till evaluation is done. For other cases we can directly return from here.
+ return;
} else if (!hasEmpty && alw.isSingleRow()) {
if (hasAnyFiltered(alias, alw.rowIter().first())) {
hasEmpty = true;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
index c677796de73..f9e7a40e10e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
@@ -428,7 +428,8 @@ private List<Byte> joinObject(int[] smallestPos, boolean clear) throws HiveExcep
private void putDummyOrEmpty(Byte i) {
// put a empty list or null
- if (noOuterJoin) {
+ boolean isRightOfAntiJoin = (i != 0 && condn[i-1].getType() == JoinDesc.ANTI_JOIN);
+ if (noOuterJoin || isRightOfAntiJoin) {
storage[i] = emptyList;
} else {
storage[i] = dummyObjVectors[i];
diff --git a/ql/src/test/queries/clientpositive/antijoin3.q b/ql/src/test/queries/clientpositive/antijoin3.q
new file mode 100644
index 00000000000..b2ce2b0fc80
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/antijoin3.q
@@ -0,0 +1,22 @@
+set hive.merge.nway.joins=true;
+
+create table taba(id string);
+create table tabb(id string);
+create table tabc(id string);
+
+INSERT INTO TABLE taba VALUES ('1'),('2');
+INSERT INTO TABLE tabc VALUES ('1'),('2'),('2');
+
+explain
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null;
+explain cbo
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null;
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null;
+
+INSERT INTO TABLE tabb VALUES ('1');
+
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null;
+
+INSERT INTO TABLE tabb VALUES ('2');
+
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null;
diff --git a/ql/src/test/results/clientpositive/llap/antijoin3.q.out b/ql/src/test/results/clientpositive/llap/antijoin3.q.out
new file mode 100644
index 00000000000..b12a4fe603f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/antijoin3.q.out
@@ -0,0 +1,250 @@
+PREHOOK: query: create table taba(id string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@taba
+POSTHOOK: query: create table taba(id string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@taba
+PREHOOK: query: create table tabb(id string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tabb
+POSTHOOK: query: create table tabb(id string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tabb
+PREHOOK: query: create table tabc(id string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tabc
+POSTHOOK: query: create table tabc(id string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tabc
+PREHOOK: query: INSERT INTO TABLE taba VALUES ('1'),('2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@taba
+POSTHOOK: query: INSERT INTO TABLE taba VALUES ('1'),('2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@taba
+POSTHOOK: Lineage: taba.id SCRIPT []
+PREHOOK: query: INSERT INTO TABLE tabc VALUES ('1'),('2'),('2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tabc
+POSTHOOK: query: INSERT INTO TABLE tabc VALUES ('1'),('2'),('2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tabc
+POSTHOOK: Lineage: tabc.id SCRIPT []
+PREHOOK: query: explain
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@taba
+PREHOOK: Input: default@tabb
+PREHOOK: Input: default@tabc
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@taba
+POSTHOOK: Input: default@tabb
+POSTHOOK: Input: default@tabc
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: id (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 170 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: id is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: c
+ filterExpr: id is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: id (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Anti Join 0 to 1
+ Left Outer Join 0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 561 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), null (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 6 Data size: 561 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 561 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain cbo
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@taba
+PREHOOK: Input: default@tabb
+PREHOOK: Input: default@tabc
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@taba
+POSTHOOK: Input: default@tabb
+POSTHOOK: Input: default@tabc
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(id=[$1], id1=[null:VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], id2=[$0])
+ HiveJoin(condition=[=($0, $1)], joinType=[right], algorithm=[none], cost=[not available])
+ HiveProject(id=[$0])
+ HiveFilter(condition=[IS NOT NULL($0)])
+ HiveTableScan(table=[[default, tabc]], table:alias=[c])
+ HiveAntiJoin(condition=[=($0, $1)], joinType=[anti])
+ HiveProject(id=[$0])
+ HiveTableScan(table=[[default, taba]], table:alias=[a])
+ HiveProject(id=[$0])
+ HiveFilter(condition=[IS NOT NULL($0)])
+ HiveTableScan(table=[[default, tabb]], table:alias=[b])
+
+PREHOOK: query: select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@taba
+PREHOOK: Input: default@tabb
+PREHOOK: Input: default@tabc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@taba
+POSTHOOK: Input: default@tabb
+POSTHOOK: Input: default@tabc
+#### A masked pattern was here ####
+1 NULL 1
+2 NULL 2
+2 NULL 2
+PREHOOK: query: INSERT INTO TABLE tabb VALUES ('1')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tabb
+POSTHOOK: query: INSERT INTO TABLE tabb VALUES ('1')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tabb
+POSTHOOK: Lineage: tabb.id SCRIPT []
+PREHOOK: query: select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@taba
+PREHOOK: Input: default@tabb
+PREHOOK: Input: default@tabc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@taba
+POSTHOOK: Input: default@tabb
+POSTHOOK: Input: default@tabc
+#### A masked pattern was here ####
+2 NULL 2
+2 NULL 2
+PREHOOK: query: INSERT INTO TABLE tabb VALUES ('2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tabb
+POSTHOOK: query: INSERT INTO TABLE tabb VALUES ('2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tabb
+POSTHOOK: Lineage: tabb.id SCRIPT []
+PREHOOK: query: select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@taba
+PREHOOK: Input: default@tabb
+PREHOOK: Input: default@tabc
+#### A masked pattern was here ####
+POSTHOOK: query: select * from taba A left outer join tabb B on (A.id = B.id) left outer join tabc C on (C.id = A.id) where B.id is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@taba
+POSTHOOK: Input: default@tabb
+POSTHOOK: Input: default@tabc
+#### A masked pattern was here ####
(hive) 05/05: HIVE-29277: Invalid index should return null for get_json_object (#6144)
Posted by dk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/hive.git
commit eed6cd8ab45a5d42f7cdc2e63d243b272b108c55
Author: Wechar Yu <yu...@gmail.com>
AuthorDate: Thu Nov 13 20:03:05 2025 +0800
HIVE-29277: Invalid index should return null for get_json_object (#6144)
(cherry picked from commit 866cc7d2a69d2d8b018917488ab243244707aae8)
---
ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java | 2 ++
ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java | 3 +++
ql/src/test/queries/clientpositive/udf_get_json_object.q | 2 ++
ql/src/test/results/clientpositive/llap/udf_get_json_object.q.out | 6 +++++-
4 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java
index 9e7e05361f9..fd4e598da92 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFJson.java
@@ -241,6 +241,8 @@ private Object extract(Object json, String path, boolean skipMapProc) {
if (indexList.size() > 0) {
json = extract_json_withindex(json, indexList);
+ } else if (path.indexOf('[') >= 0) {
+ return null; // index is invalid
}
return json;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java
index c2d1f42fb73..3b9849f9553 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java
@@ -105,6 +105,9 @@ public void testRootArray() throws HiveException {
runTest("[1,2,3]", "$0", null, udf);
runTest("[1,2,3]", "0", null, udf);
runTest("[1,2,3]", "$.", null, udf);
+ runTest("[1,2,3]", "$[1 ]", null, udf);
+ runTest("[1,2,3]", "$[ 1]", null, udf);
+ runTest("[1,2,3]", "$[invalid]", null, udf);
runTest("[1,2,3]", "$", "[1,2,3]", udf);
runTest("{\"a\":4}", "$", "{\"a\":4}", udf);
diff --git a/ql/src/test/queries/clientpositive/udf_get_json_object.q b/ql/src/test/queries/clientpositive/udf_get_json_object.q
index e5762c2e552..c156e496ac1 100644
--- a/ql/src/test/queries/clientpositive/udf_get_json_object.q
+++ b/ql/src/test/queries/clientpositive/udf_get_json_object.q
@@ -64,4 +64,6 @@ get_json_object('[{"k1":[{"k11":[1,2,3]}]}]', '$[0].k1[0].k11[1]');
SELECT
get_json_object('[1,2,3]', '[2]'),
get_json_object('[1,2,3]', '$0'),
+get_json_object('[1,2,3]', '$[0 ]'),
+get_json_object('[1,2,3]', '$[ 0]'),
get_json_object('[1,2,3]', '$[3]');
\ No newline at end of file
diff --git a/ql/src/test/results/clientpositive/llap/udf_get_json_object.q.out b/ql/src/test/results/clientpositive/llap/udf_get_json_object.q.out
index d2ae71c96b1..05e50c97b1c 100644
--- a/ql/src/test/results/clientpositive/llap/udf_get_json_object.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_get_json_object.q.out
@@ -227,6 +227,8 @@ POSTHOOK: Input: _dummy_database@_dummy_table
PREHOOK: query: SELECT
get_json_object('[1,2,3]', '[2]'),
get_json_object('[1,2,3]', '$0'),
+get_json_object('[1,2,3]', '$[0 ]'),
+get_json_object('[1,2,3]', '$[ 0]'),
get_json_object('[1,2,3]', '$[3]')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
@@ -234,8 +236,10 @@ PREHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: query: SELECT
get_json_object('[1,2,3]', '[2]'),
get_json_object('[1,2,3]', '$0'),
+get_json_object('[1,2,3]', '$[0 ]'),
+get_json_object('[1,2,3]', '$[ 0]'),
get_json_object('[1,2,3]', '$[3]')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
#### A masked pattern was here ####
-NULL NULL NULL
+NULL NULL NULL NULL NULL
(hive) 03/05: HIVE-29176: Wrong result when HiveAntiJoin is replacing an IS NULL filter on a nullable column (#6062)
Posted by dk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 8aae2dd85855f48f05d0ad85639892f8828f1ae6
Author: Thomas Rebele <th...@users.noreply.github.com>
AuthorDate: Thu Nov 13 06:14:30 2025 +0100
HIVE-29176: Wrong result when HiveAntiJoin is replacing an IS NULL filter on a nullable column (#6062)
(cherry picked from commit 330b01822c575662353376e715a05f814950b4db)
---
.../hive/ql/optimizer/calcite/HiveCalciteUtil.java | 44 +--
.../calcite/rules/HiveAntiSemiJoinRule.java | 175 +++++++++---
.../calcite/rules/TestHiveAntiSemiJoinRule.java | 248 +++++++++++++++++
.../ql/optimizer/calcite/rules/TestRuleHelper.java | 113 ++++++--
ql/src/test/queries/clientpositive/antijoin4.q | 64 +++++
.../results/clientpositive/llap/antijoin3.q.out | 2 +-
.../results/clientpositive/llap/antijoin4.q.out | 294 +++++++++++++++++++++
.../results/clientpositive/llap/llap_smb_ptf.q.out | 47 ++--
8 files changed, 871 insertions(+), 116 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index a7305326c00..5cbfb5dbe0c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -1233,43 +1233,17 @@ public FixNullabilityShuttle(RexBuilder rexBuilder,
}
/**
- * Checks if any of the expression given as list expressions are from right side of the join.
- * This is used during anti join conversion.
- *
- * @param joinRel Join node whose right side has to be searched.
- * @param expressions The list of expression to search.
- * @return true if any of the expressions is from right side of join.
+ * Given a join, creates a bitset of the joined columns originating from the right-hand side.
+ * @param joinRel a join that concatenates all columns from its inputs (so no semi-join)
+ * @return a bitset
*/
- public static boolean hasAnyExpressionFromRightSide(RelNode joinRel, List<RexNode> expressions) {
- List<RelDataTypeField> joinFields = joinRel.getRowType().getFieldList();
- int nTotalFields = joinFields.size();
- List<RelDataTypeField> leftFields = (joinRel.getInputs().get(0)).getRowType().getFieldList();
- int nFieldsLeft = leftFields.size();
- ImmutableBitSet rightBitmap = ImmutableBitSet.range(nFieldsLeft, nTotalFields);
-
- for (RexNode node : expressions) {
- ImmutableBitSet inputBits = RelOptUtil.InputFinder.bits(node);
- if (rightBitmap.contains(inputBits)) {
- return true;
- }
- }
- return false;
- }
-
- public static boolean hasAllExpressionsFromRightSide(RelNode joinRel, List<RexNode> expressions) {
- List<RelDataTypeField> joinFields = joinRel.getRowType().getFieldList();
- int nTotalFields = joinFields.size();
- List<RelDataTypeField> leftFields = (joinRel.getInputs().get(0)).getRowType().getFieldList();
- int nFieldsLeft = leftFields.size();
- ImmutableBitSet rightBitmap = ImmutableBitSet.range(nFieldsLeft, nTotalFields);
-
- for (RexNode node : expressions) {
- ImmutableBitSet inputBits = RelOptUtil.InputFinder.bits(node);
- if (!rightBitmap.contains(inputBits)) {
- return false;
- }
+ public static ImmutableBitSet getRightSideBitset(RelNode joinRel) {
+ if(joinRel.getInputs().size() != 2) {
+ throw new IllegalArgumentException("The relation must have exactly two children:\n" + RelOptUtil.toString(joinRel));
}
- return true;
+ int nTotalFields = joinRel.getRowType().getFieldCount();
+ int nFieldsLeft = (joinRel.getInputs().get(0)).getRowType().getFieldCount();
+ return ImmutableBitSet.range(nFieldsLeft, nTotalFields);
}
/**
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAntiSemiJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAntiSemiJoinRule.java
index 3c4c2ab693c..02f2f3d6825 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAntiSemiJoinRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAntiSemiJoinRule.java
@@ -17,29 +17,40 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+import com.google.common.collect.ImmutableList;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RexImplicationChecker;
import org.apache.calcite.plan.Strong;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexExecutor;
+import org.apache.calcite.rex.RexExecutorImpl;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Util;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAntiJoin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.Optional;
/**
* Planner rule that converts a join plus filter to anti join.
@@ -86,14 +97,17 @@ protected void perform(RelOptRuleCall call, Project project, Filter filter, Join
assert (filter != null);
- List<RexNode> filterList = getResidualFilterNodes(filter, join);
- if (filterList == null) {
+ ImmutableBitSet rhsFields = HiveCalciteUtil.getRightSideBitset(join);
+ Optional<List<RexNode>> optFilterList = getResidualFilterNodes(filter, join, rhsFields);
+ if (optFilterList.isEmpty()) {
return;
}
+ List<RexNode> filterList = optFilterList.get();
// If any projection is there from right side, then we can not convert to anti join.
- boolean hasProjection = HiveCalciteUtil.hasAnyExpressionFromRightSide(join, project.getProjects());
- if (hasProjection) {
+ ImmutableBitSet projectedFields = RelOptUtil.InputFinder.bits(project.getProjects(), null);
+ boolean projectionUsesRHS = projectedFields.intersects(rhsFields);
+ if (projectionUsesRHS) {
return;
}
@@ -124,13 +138,14 @@ protected void perform(RelOptRuleCall call, Project project, Filter filter, Join
/**
* Extracts the non-null filter conditions from given filter node.
*
- * @param filter The filter condition to be checked.
- * @param join Join node whose right side has to be searched.
+ * @param filter The filter condition to be checked.
+ * @param join Join node whose right side has to be searched.
+ * @param rhsFields
* @return null : Anti join condition is not matched for filter.
- * Empty list : No residual filter conditions present.
- * Valid list containing the filter to be applied after join.
+ * Empty list : No residual filter conditions present.
+ * Valid list containing the filter to be applied after join.
*/
- private List<RexNode> getResidualFilterNodes(Filter filter, Join join) {
+ private Optional<List<RexNode>> getResidualFilterNodes(Filter filter, Join join, ImmutableBitSet rhsFields) {
// 1. If null filter is not present from right side then we can not convert to anti join.
// 2. If any non-null filter is present from right side, we can not convert it to anti join.
// 3. Keep other filters which needs to be executed after join.
@@ -140,43 +155,123 @@ private List<RexNode> getResidualFilterNodes(Filter filter, Join join) {
List<RexNode> aboveFilters = RelOptUtil.conjunctions(filter.getCondition());
boolean hasNullFilterOnRightSide = false;
List<RexNode> filterList = new ArrayList<>();
+ final ImmutableBitSet notNullColumnsFromRightSide = getNotNullColumnsFromRightSide(join);
+
for (RexNode filterNode : aboveFilters) {
- if (filterNode.getKind() == SqlKind.IS_NULL) {
- // Null filter from right side table can be removed and its a pre-condition for anti join conversion.
- if (HiveCalciteUtil.hasAllExpressionsFromRightSide(join, Collections.singletonList(filterNode))
- && isStrong(((RexCall) filterNode).getOperands().get(0))) {
- hasNullFilterOnRightSide = true;
- } else {
- filterList.add(filterNode);
- }
- } else {
- if (HiveCalciteUtil.hasAnyExpressionFromRightSide(join, Collections.singletonList(filterNode))) {
- // If some non null condition is present from right side, we can not convert the join to anti join as
- // anti join does not project the fields from right side.
- return null;
- } else {
- filterList.add(filterNode);
- }
+ final ImmutableBitSet usedFields = RelOptUtil.InputFinder.bits(filterNode);
+ boolean usesFieldFromRHS = usedFields.intersects(rhsFields);
+
+ if(!usesFieldFromRHS) {
+ // Only LHS fields or constants, so the filterNode is part of the residual filter
+ filterList.add(filterNode);
+ continue;
+ }
+
+ // In the following we check for filter nodes that let us deduce that
+ // "an (originally) not-null column of RHS IS NULL because the LHS row will not be matched"
+
+ if(filterNode.getKind() != SqlKind.IS_NULL) {
+ return Optional.empty();
+ }
+
+ boolean usesRHSFieldsOnly = rhsFields.contains(usedFields);
+ if (!usesRHSFieldsOnly) {
+ // If there is a mix between LHS and RHS fields, don't convert to anti-join
+ return Optional.empty();
+ }
+
+ // Null filter from right side table can be removed and it is a pre-condition for anti join conversion.
+ RexNode arg = ((RexCall) filterNode).getOperands().get(0);
+ if (isStrong(arg, notNullColumnsFromRightSide)) {
+ hasNullFilterOnRightSide = true;
+ } else if(!isStrong(arg, rhsFields)) {
+ // if all RHS fields are null and the IS NULL is still not fulfilled, bail out
+ return Optional.empty();
}
}
if (!hasNullFilterOnRightSide) {
- return null;
+ return Optional.empty();
}
- return filterList;
+ return Optional.of(filterList);
}
- private boolean isStrong(RexNode rexNode) {
- AtomicBoolean hasCast = new AtomicBoolean(false);
- rexNode.accept(new RexVisitorImpl<Void>(true) {
- @Override
- public Void visitCall(RexCall call) {
- if (call.getKind() == SqlKind.CAST) {
- hasCast.set(true);
- }
- return super.visitCall(call);
+ private ImmutableBitSet getNotNullColumnsFromRightSide(RelNode joinRel) {
+ // we need to shift the indices of the second child to the right
+ int shift = (joinRel.getInput(0)).getRowType().getFieldCount();
+ ImmutableBitSet rhsNotnullColumns = deduceNotNullColumns(joinRel.getInput(1));
+ return rhsNotnullColumns.shift(shift);
+ }
+
+ /**
+ * Deduce which columns of the <code>relNode</code> are definitively NOT NULL.
+ */
+ private ImmutableBitSet deduceNotNullColumns(RelNode relNode) {
+ // adapted from org.apache.calcite.plan.RelOptUtil.containsNullableFields
+ RelOptCluster cluster = relNode.getCluster();
+ final RexBuilder rexBuilder = cluster.getRexBuilder();
+ final RelMetadataQuery mq = cluster.getMetadataQuery();
+ ImmutableBitSet.Builder result = ImmutableBitSet.builder();
+ ImmutableBitSet.Builder candidatesBuilder = ImmutableBitSet.builder();
+ List<RelDataTypeField> fieldList = relNode.getRowType().getFieldList();
+ for (int i=0; i<fieldList.size(); i++) {
+ if (fieldList.get(i).getType().isNullable()) {
+ candidatesBuilder.set(i);
+ }
+ else {
+ result.set(i);
+ }
+ }
+ ImmutableBitSet candidates = candidatesBuilder.build();
+ if (candidates.isEmpty()) {
+ // All columns are declared NOT NULL, no need to change
+ return result.build();
+ }
+ final RexExecutor executor = cluster.getPlanner().getExecutor();
+ if (!(executor instanceof RexExecutorImpl)) {
+ // Cannot proceed without an executor.
+ return result.build();
+ }
+
+ final RexImplicationChecker checker =
+ new RexImplicationChecker(rexBuilder, executor, relNode.getRowType());
+ final RelOptPredicateList predicates = mq.getPulledUpPredicates(relNode);
+
+ ImmutableList<RexNode> preds = predicates.pulledUpPredicates;
+ final List<RexNode> antecedent = new ArrayList<>(preds);
+ final RexNode first = RexUtil.composeConjunction(rexBuilder, antecedent);
+ for (int c : candidates) {
+ RelDataTypeField field = fieldList.get(c);
+ final RexNode second = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL,
+ rexBuilder.makeInputRef(field.getType(), field.getIndex()));
+ // Suppose we have EMP(empno INT NOT NULL, mgr INT),
+ // and predicates [empno > 0, mgr > 0].
+ // We make first: "empno > 0 AND mgr > 0"
+ // and second: "mgr IS NOT NULL"
+ // and ask whether first implies second.
+ // It does, so we have no nullable columns.
+ if(checker.implies(first, second)) {
+ result.set(c);
}
- });
- return !hasCast.get() && Strong.isStrong(rexNode);
+ }
+ return result.build();
+ }
+
+ private boolean isStrong(RexNode rexNode, ImmutableBitSet rightSideBitset) {
+ try {
+ rexNode.accept(new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitCall(RexCall call) {
+ if (call.getKind() == SqlKind.CAST) {
+ throw Util.FoundOne.NULL;
+ }
+ return super.visitCall(call);
+ }
+ });
+ } catch (Util.FoundOne e) {
+ // Hive's CAST might introduce NULL for NOT NULL fields
+ return false;
+ }
+ return Strong.isNull(rexNode, rightSideBitset);
}
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestHiveAntiSemiJoinRule.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestHiveAntiSemiJoinRule.java
new file mode 100644
index 00000000000..97e808eaa40
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestHiveAntiSemiJoinRule.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.runtime.Hook;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.Collections;
+
+import static org.apache.hadoop.hive.ql.optimizer.calcite.rules.TestRuleHelper.*;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestHiveAntiSemiJoinRule {
+
+ PlanFixture fixture() {
+ RelOptPlanner planner = buildPlanner(Collections.singletonList(HiveAntiSemiJoinRule.INSTANCE));
+ // executor is needed to determine nullability of RHS columns
+ planner.setExecutor(new HiveRexExecutorImpl());
+ return new PlanFixture(planner)
+ .registerTable("t1", T1Record.class)
+ .registerTable("t2", T2Record.class)
+ .registerTable("t3", T3Record.class);
+ }
+
+ @Test
+ public void testFilterOnNullableColumn() {
+ PlanFixture fixture = fixture();
+ RelBuilder b = fixture.createRelBuilder();
+
+ // @formatter:off
+ RelNode plan = b
+ .scan("t1")
+ .scan("t2")
+ .join(JoinRelType.LEFT, b.equals(
+ b.field(2, 0, "t1nullable"),
+ b.field(2, 1, "t2id")))
+ .filter(b.isNull(b.field("t2nullable")))
+ .project(b.field("t1id"))
+ .build();
+
+ String expectedPlan = "HiveProject(t1id=[$0])\n"
+ + " HiveFilter(condition=[IS NULL($5)])\n"
+ + " HiveJoin(condition=[=($2, $3)], joinType=[left], algorithm=[none], cost=[not available])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " LogicalTableScan(table=[[t2]])\n";
+ // @formatter:on
+
+ assertPlans(fixture.getPlanner(), plan, expectedPlan, expectedPlan);
+ }
+
+ @Test
+ public void testFilterOnFormerlyNullableColumn() {
+ PlanFixture fixture = fixture();
+ RelBuilder b = fixture.createRelBuilder();
+
+ // @formatter:off
+ RelNode plan = b
+ .scan("t1")
+ .scan("t2")
+ .filter(b.isNotNull(b.field("t2nullable")))
+ .join(JoinRelType.LEFT, b.equals(
+ b.field(2, 0, "t1nullable"),
+ b.field(2, 1, "t2nullable")))
+ // the IS NOT NULL on the RHS ensures that the values
+ // we get from t2nullable are actually NOT NULL
+ .filter(b.isNull(b.field("t2nullable")))
+ .project(b.field("t1id"))
+ .build();
+
+ String prePlan = "HiveProject(t1id=[$0])\n"
+ + " HiveFilter(condition=[IS NULL($5)])\n"
+ + " HiveJoin(condition=[=($2, $5)], joinType=[left], algorithm=[none], cost=[not available])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " HiveFilter(condition=[IS NOT NULL($2)])\n"
+ + " LogicalTableScan(table=[[t2]])\n";
+
+ String postPlan = "HiveProject(t1id=[$0])\n"
+ + " HiveAntiJoin(condition=[=($2, $5)], joinType=[anti])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " HiveFilter(condition=[IS NOT NULL($2)])\n"
+ + " LogicalTableScan(table=[[t2]])\n";
+ // @formatter:on
+
+ assertPlans(fixture.getPlanner(), plan, prePlan, postPlan);
+ }
+
+
+ @Test
+ public void testFilterIsNullFromBothSides() {
+ PlanFixture fixture = fixture();
+
+ RelNode plan;
+ try (Hook.Closeable ignore = Hook.REL_BUILDER_SIMPLIFY.addThread(Hook.propertyJ(false))) {
+ RelBuilder b = fixture.createRelBuilder();
+ // @formatter:off
+ plan = b.scan("t1")
+ .scan("t2")
+ .join(JoinRelType.LEFT, b.equals(b.field(2, 0, "t1nullable"), b.field(2, 1, "t2id")))
+ .filter(b.isNull(b.call(SqlStdOperatorTable.PLUS, b.field("t2nullable"), b.field("t1nullable"))))
+ .project(b.field("t1id")).build();
+ // @formatter:on
+ }
+
+ // @formatter:off
+ String expectedPlan = "HiveProject(t1id=[$0])\n"
+ + " HiveFilter(condition=[IS NULL(+($5, $2))])\n"
+ + " HiveJoin(condition=[=($2, $3)], joinType=[left], algorithm=[none], cost=[not available])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " LogicalTableScan(table=[[t2]])\n";
+ // @formatter:on
+
+ assertPlans(fixture.getPlanner(), plan, expectedPlan, expectedPlan);
+ }
+
+ @Test
+ public void testFilterOnNotNullColumn() {
+ PlanFixture fixture = fixture();
+ RelBuilder b = fixture.createRelBuilder();
+
+ // @formatter:off
+ RelNode plan = b
+ .scan("t1")
+ .scan("t2")
+ .join(JoinRelType.LEFT, b.equals(
+ b.field(2, 0, "t1nullable"),
+ b.field(2, 1, "t2id")))
+ .filter(b.isNull(b.field("t2notnull")))
+ .project(b.field("t1id"))
+ .build();
+
+ String prePlan = "HiveProject(t1id=[$0])\n"
+ + " HiveFilter(condition=[IS NULL($4)])\n"
+ + " HiveJoin(condition=[=($2, $3)], joinType=[left], algorithm=[none], cost=[not available])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " LogicalTableScan(table=[[t2]])\n";
+
+ String postPlan = "HiveProject(t1id=[$0])\n"
+ + " HiveAntiJoin(condition=[=($2, $3)], joinType=[anti])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " LogicalTableScan(table=[[t2]])\n";
+ // @formatter:on
+
+ assertPlans(fixture.getPlanner(), plan, prePlan, postPlan);
+ }
+
+ /** Check RHS without any nullable columns */
+ @Test
+ public void testFilterOnNotNullColumn2() {
+ PlanFixture fixture = fixture();
+ RelBuilder b = fixture.createRelBuilder();
+
+ // @formatter:off
+ RelNode plan = b
+ .scan("t1")
+ .scan("t3")
+ .join(JoinRelType.LEFT, b.equals(
+ b.field(2, 0, "t1nullable"),
+ b.field(2, 1, "t3id")))
+ .filter(b.isNull(b.field("t3notnull")))
+ .project(b.field("t1id"))
+ .build();
+
+ String prePlan = "HiveProject(t1id=[$0])\n"
+ + " HiveFilter(condition=[IS NULL($4)])\n"
+ + " HiveJoin(condition=[=($2, $3)], joinType=[left], algorithm=[none], cost=[not available])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " LogicalTableScan(table=[[t3]])\n";
+
+ String postPlan = "HiveProject(t1id=[$0])\n"
+ + " HiveAntiJoin(condition=[=($2, $3)], joinType=[anti])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " LogicalTableScan(table=[[t3]])\n";
+ // @formatter:on
+
+ assertPlans(fixture.getPlanner(), plan, prePlan, postPlan);
+ }
+
+ @Test
+ public void testFilterOnNullAndNotNullColumn() {
+ PlanFixture fixture = fixture();
+ RelBuilder b = fixture.createRelBuilder();
+
+ // @formatter:off
+ RelNode plan = b
+ .scan("t1")
+ .scan("t2")
+ .join(JoinRelType.LEFT, b.equals(
+ b.field(2, 0, "t1nullable"),
+ b.field(2, 1, "t2id")))
+ .filter(b.and(b.isNull(b.field("t2notnull")), b.isNull((b.field("t2nullable")))))
+ .project(b.field("t1id"))
+ .build();
+
+ String prePlan = "HiveProject(t1id=[$0])\n"
+ + " HiveFilter(condition=[AND(IS NULL($4), IS NULL($5))])\n"
+ + " HiveJoin(condition=[=($2, $3)], joinType=[left], algorithm=[none], cost=[not available])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " LogicalTableScan(table=[[t2]])\n";
+
+ String postPlan = "HiveProject(t1id=[$0])\n"
+ + " HiveAntiJoin(condition=[=($2, $3)], joinType=[anti])\n"
+ + " LogicalTableScan(table=[[t1]])\n"
+ + " LogicalTableScan(table=[[t2]])\n";
+ // @formatter:on
+
+ assertPlans(fixture.getPlanner(), plan, prePlan, postPlan);
+ }
+
+ static class T1Record {
+ public int t1id;
+ public int t1notnull;
+ public Integer t1nullable;
+ }
+
+ static class T2Record {
+ public int t2id;
+ public int t2notnull;
+ public Integer t2nullable;
+ }
+
+ static class T3Record {
+ public int t3id;
+ public int t3notnull;
+ }
+}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestRuleHelper.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestRuleHelper.java
index 8c49f58c424..937e2c0499d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestRuleHelper.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestRuleHelper.java
@@ -19,12 +19,7 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
-import org.apache.calcite.plan.AbstractRelOptPlanner;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptSchema;
-import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.*;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.rel.RelNode;
@@ -37,19 +32,13 @@
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
-import org.junit.Test;
-import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
-import java.util.Collection;
-import java.util.Collections;
+import java.util.*;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.*;
public class TestRuleHelper {
@@ -73,7 +62,99 @@ public static AbstractRelOptPlanner buildPlanner(Collection<RelOptRule> rules) {
return new HepPlanner(programBuilder.build());
}
- public static RelBuilder buildRelBuilder(AbstractRelOptPlanner planner,
+ public static class HiveTableMock {
+ final Class<?> recordClass;
+ final RelOptHiveTable tableMock;
+ final Table hiveTableMock;
+ final List<String> name;
+
+ public HiveTableMock(List<String> name, Class<?> recordClass, PlanFixture mockBuilder) {
+ this.name = List.copyOf(name);
+ this.recordClass = recordClass;
+ tableMock = mock(RelOptHiveTable.class);
+ hiveTableMock = mock(Table.class);
+
+ RelDataType rowTypeMock = JAVA_TYPE_FACTORY.createStructType(recordClass);
+ doReturn(rowTypeMock).when(tableMock).getRowType();
+
+ LogicalTableScan tableScan = LogicalTableScan.create(mockBuilder.optCluster, tableMock, Collections.emptyList());
+ doReturn(tableScan).when(tableMock).toRel(ArgumentMatchers.any());
+
+ doReturn(this.name).when(tableMock).getQualifiedName();
+
+ lenient().doReturn(hiveTableMock).when(tableMock).getHiveTableMD();
+ }
+ }
+
+ /**
+ * A fixture for creating plans with <code>HiveRelNode</code>s.
+ */
+ public static class PlanFixture {
+ final RelOptCluster optCluster;
+
+ final Map<List<String>, HiveTableMock> tables = new HashMap<>();
+
+ Class<?> defaultRecordClass;
+
+ public PlanFixture(RelOptPlanner planner) {
+ RexBuilder rexBuilder = new RexBuilder(JAVA_TYPE_FACTORY);
+ optCluster = RelOptCluster.create(planner, rexBuilder);
+ }
+
+ /**
+ * Register a table in the schema, using the attributes of the class as columns.
+ */
+ public PlanFixture registerTable(String name, Class<?> recordClass) {
+ return registerTable(List.of(name), recordClass);
+ }
+
+ /**
+ * Similar to {@link #registerTable(String, Class)}, but with a qualified name.
+ * <p>
+ * See {@link RelOptTable#getQualifiedName()}.
+ */
+ public PlanFixture registerTable(List<String> name, Class<?> recordClass) {
+ name = List.copyOf(name);
+ tables.put(name, new HiveTableMock(name, recordClass, this));
+ return this;
+ }
+
+ /**
+ * Allows to use any table names when scanning.
+ * <p>
+ * The scanned table will provide the attributes of the class as columns.
+ */
+ public PlanFixture setDefaultRecordClass(Class<?> recordClass) {
+ this.defaultRecordClass = recordClass;
+ return this;
+ }
+
+ public RelOptPlanner getPlanner() {
+ return optCluster.getPlanner();
+ }
+
+ public RelBuilder createRelBuilder() {
+ final RelOptSchema schemaMock;
+ schemaMock = mock(RelOptSchema.class);
+ // create a copy that we can modify in our method
+ Map<List<String>, HiveTableMock> tableMap = new HashMap<>(tables);
+
+ when(schemaMock.getTableForMember(any())).thenAnswer(i -> {
+ List<String> tableName = i.getArgument(0);
+ HiveTableMock hiveTableMock = tableMap.get(tableName);
+ if(hiveTableMock == null) {
+ Objects.requireNonNull(defaultRecordClass, "Table " + tableName + " was not registered with the mock, and no default table provided");
+ hiveTableMock = new HiveTableMock(tableName, defaultRecordClass, this);
+ tableMap.put(tableName, hiveTableMock);
+ }
+ return hiveTableMock.tableMock;
+ } );
+
+ return HiveRelFactories.HIVE_BUILDER.create(optCluster, schemaMock);
+ }
+ }
+
+ public static RelBuilder buildRelBuilder(RelOptPlanner planner,
RelOptSchema schemaMock, RelOptHiveTable tableMock, Table hiveTableMock, Class<?> clazz) {
RexBuilder rexBuilder = new RexBuilder(JAVA_TYPE_FACTORY);
@@ -101,7 +182,7 @@ static RexNode and(RelBuilder relBuilder, RexNode... args) {
return relBuilder.call(SqlStdOperatorTable.AND, args);
}
- static void assertPlans(AbstractRelOptPlanner planner, RelNode plan, String expectedPrePlan, String expectedPostPlan) {
+ static void assertPlans(RelOptPlanner planner, RelNode plan, String expectedPrePlan, String expectedPostPlan) {
planner.setRoot(plan);
RelNode optimizedRelNode = planner.findBestExp();
assertEquals("Original plans do not match", expectedPrePlan, RelOptUtil.toString(plan));
diff --git a/ql/src/test/queries/clientpositive/antijoin4.q b/ql/src/test/queries/clientpositive/antijoin4.q
new file mode 100644
index 00000000000..81cb5ce9954
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/antijoin4.q
@@ -0,0 +1,64 @@
+SET hive.vectorized.execution.enabled=false;
+set hive.mapred.mode=nonstrict;
+SET hive.auto.convert.join=false;
+SET hive.auto.convert.anti.join=true;
+-- SORT_QUERY_RESULTS
+
+create table antijoin3_t1 (t1id int not null, t1notnull string not null, t1nullable string);
+create table antijoin3_t2 (t2id int not null, t2notnull string not null, t2nullable string);
+create table antijoin3_t3 (t3id int not null);
+
+insert into antijoin3_t1 values
+(0, "val_0", null),
+(1, "val_1", null),
+(2, "val_2", "val_2"),
+(3, "val_3", "val_3"),
+(4, "val_4", "val_4");
+
+insert into antijoin3_t2 values
+(0, "val_0", null),
+(1, "val_1", null),
+(4, "val_4", "val_4"),
+(5, "val_5", "val_5");
+
+insert into antijoin3_t3 values (0), (4), (6);
+
+-- do not introduce anti-join if filtering a nullable column with IS NULL
+explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2nullable is null;
+select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2nullable is null;
+
+-- but introduce anti-join if filtering a NOT NULL column with IS NULL
+explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null;
+select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null;
+
+-- play it safe and do not introduce antijoin for filters combining LHS and RHS columns
+explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where (coalesce(t1notnull,t2notnull)) is null;
+select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where (coalesce(t1notnull,t2notnull)) is null;
+
+-- selecting constants do not prevent an anti-join (HIVE-29164)
+explain cbo select t1id, t1notnull, t1nullable, "foo" from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null;
+select t1id, t1notnull, t1nullable, "foo" from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null;
+
+-- check whether nullability is propagated correctly
+explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+where t2notnull is null;
+
+select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+where t2notnull is null;
+
+explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+-- t3id is from the RHS of the left join, so it becomes nullable, so no antijoin
+where t3id is null;
+
+select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+-- t3id is from the RHS of the left join, so it becomes nullable, so no antijoin
+where t3id is null;
+
diff --git a/ql/src/test/results/clientpositive/llap/antijoin3.q.out b/ql/src/test/results/clientpositive/llap/antijoin3.q.out
index b12a4fe603f..1451f519e47 100644
--- a/ql/src/test/results/clientpositive/llap/antijoin3.q.out
+++ b/ql/src/test/results/clientpositive/llap/antijoin3.q.out
@@ -177,7 +177,7 @@ POSTHOOK: Input: default@tabb
POSTHOOK: Input: default@tabc
#### A masked pattern was here ####
CBO PLAN:
-HiveProject(id=[$1], id1=[null:VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], id2=[$0])
+HiveProject(a.id=[$1], b.id=[null:VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], c.id=[$0])
HiveJoin(condition=[=($0, $1)], joinType=[right], algorithm=[none], cost=[not available])
HiveProject(id=[$0])
HiveFilter(condition=[IS NOT NULL($0)])
diff --git a/ql/src/test/results/clientpositive/llap/antijoin4.q.out b/ql/src/test/results/clientpositive/llap/antijoin4.q.out
new file mode 100644
index 00000000000..62cbf85908d
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/antijoin4.q.out
@@ -0,0 +1,294 @@
+PREHOOK: query: create table antijoin3_t1 (t1id int not null, t1notnull string not null, t1nullable string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@antijoin3_t1
+POSTHOOK: query: create table antijoin3_t1 (t1id int not null, t1notnull string not null, t1nullable string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@antijoin3_t1
+PREHOOK: query: create table antijoin3_t2 (t2id int not null, t2notnull string not null, t2nullable string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@antijoin3_t2
+POSTHOOK: query: create table antijoin3_t2 (t2id int not null, t2notnull string not null, t2nullable string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@antijoin3_t2
+PREHOOK: query: create table antijoin3_t3 (t3id int not null)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@antijoin3_t3
+POSTHOOK: query: create table antijoin3_t3 (t3id int not null)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@antijoin3_t3
+PREHOOK: query: insert into antijoin3_t1 values
+(0, "val_0", null),
+(1, "val_1", null),
+(2, "val_2", "val_2"),
+(3, "val_3", "val_3"),
+(4, "val_4", "val_4")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@antijoin3_t1
+POSTHOOK: query: insert into antijoin3_t1 values
+(0, "val_0", null),
+(1, "val_1", null),
+(2, "val_2", "val_2"),
+(3, "val_3", "val_3"),
+(4, "val_4", "val_4")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@antijoin3_t1
+POSTHOOK: Lineage: antijoin3_t1.t1id SCRIPT []
+POSTHOOK: Lineage: antijoin3_t1.t1notnull SCRIPT []
+POSTHOOK: Lineage: antijoin3_t1.t1nullable SCRIPT []
+PREHOOK: query: insert into antijoin3_t2 values
+(0, "val_0", null),
+(1, "val_1", null),
+(4, "val_4", "val_4"),
+(5, "val_5", "val_5")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@antijoin3_t2
+POSTHOOK: query: insert into antijoin3_t2 values
+(0, "val_0", null),
+(1, "val_1", null),
+(4, "val_4", "val_4"),
+(5, "val_5", "val_5")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@antijoin3_t2
+POSTHOOK: Lineage: antijoin3_t2.t2id SCRIPT []
+POSTHOOK: Lineage: antijoin3_t2.t2notnull SCRIPT []
+POSTHOOK: Lineage: antijoin3_t2.t2nullable SCRIPT []
+PREHOOK: query: insert into antijoin3_t3 values (0), (4), (6)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@antijoin3_t3
+POSTHOOK: query: insert into antijoin3_t3 values (0), (4), (6)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@antijoin3_t3
+POSTHOOK: Lineage: antijoin3_t3.t3id SCRIPT []
+PREHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2nullable is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2nullable is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveFilter(condition=[IS NULL($4)])
+ HiveJoin(condition=[=($0, $3)], joinType=[left], algorithm=[none], cost=[not available])
+ HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveTableScan(table=[[default, antijoin3_t1]], table:alias=[t1])
+ HiveProject(t2id=[$0], t2nullable=[$2])
+ HiveTableScan(table=[[default, antijoin3_t2]], table:alias=[t2])
+
+PREHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2nullable is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+POSTHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2nullable is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+0 val_0 NULL
+1 val_1 NULL
+2 val_2 val_2
+3 val_3 val_3
+PREHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveAntiJoin(condition=[=($0, $3)], joinType=[anti])
+ HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveTableScan(table=[[default, antijoin3_t1]], table:alias=[t1])
+ HiveProject(t2id=[$0], t2notnull=[$1])
+ HiveTableScan(table=[[default, antijoin3_t2]], table:alias=[t2])
+
+PREHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+POSTHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+2 val_2 val_2
+3 val_3 val_3
+PREHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where (coalesce(t1notnull,t2notnull)) is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where (coalesce(t1notnull,t2notnull)) is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveValues(tuples=[[]])
+
+PREHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where (coalesce(t1notnull,t2notnull)) is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+POSTHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where (coalesce(t1notnull,t2notnull)) is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+PREHOOK: query: explain cbo select t1id, t1notnull, t1nullable, "foo" from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo select t1id, t1notnull, t1nullable, "foo" from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2], _c3=[_UTF-16LE'foo':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"])
+ HiveAntiJoin(condition=[=($0, $3)], joinType=[anti])
+ HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveTableScan(table=[[default, antijoin3_t1]], table:alias=[t1])
+ HiveProject(t2id=[$0], t2notnull=[$1])
+ HiveTableScan(table=[[default, antijoin3_t2]], table:alias=[t2])
+
+PREHOOK: query: select t1id, t1notnull, t1nullable, "foo" from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+POSTHOOK: query: select t1id, t1notnull, t1nullable, "foo" from antijoin3_t1 t1 left join antijoin3_t2 t2 on t1id=t2id where t2notnull is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+#### A masked pattern was here ####
+2 val_2 val_2 foo
+3 val_3 val_3 foo
+PREHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+where t2notnull is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+PREHOOK: Input: default@antijoin3_t3
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+where t2notnull is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+POSTHOOK: Input: default@antijoin3_t3
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveAntiJoin(condition=[=($0, $3)], joinType=[anti])
+ HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveTableScan(table=[[default, antijoin3_t1]], table:alias=[t1])
+ HiveProject(t2id=[$0], t2notnull=[$1], t3id=[$2])
+ HiveJoin(condition=[=($0, $2)], joinType=[left], algorithm=[none], cost=[not available])
+ HiveProject(t2id=[$0], t2notnull=[$1])
+ HiveTableScan(table=[[default, antijoin3_t2]], table:alias=[antijoin3_t2])
+ HiveProject(t3id=[$0])
+ HiveTableScan(table=[[default, antijoin3_t3]], table:alias=[antijoin3_t3])
+
+PREHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+where t2notnull is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+PREHOOK: Input: default@antijoin3_t3
+#### A masked pattern was here ####
+POSTHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+where t2notnull is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+POSTHOOK: Input: default@antijoin3_t3
+#### A masked pattern was here ####
+2 val_2 val_2
+3 val_3 val_3
+PREHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+
+where t3id is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+PREHOOK: Input: default@antijoin3_t3
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+
+where t3id is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+POSTHOOK: Input: default@antijoin3_t3
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveFilter(condition=[IS NULL($4)])
+ HiveJoin(condition=[=($0, $3)], joinType=[left], algorithm=[none], cost=[not available])
+ HiveProject(t1id=[$0], t1notnull=[$1], t1nullable=[$2])
+ HiveTableScan(table=[[default, antijoin3_t1]], table:alias=[t1])
+ HiveJoin(condition=[=($0, $1)], joinType=[left], algorithm=[none], cost=[not available])
+ HiveProject(t2id=[$0])
+ HiveTableScan(table=[[default, antijoin3_t2]], table:alias=[antijoin3_t2])
+ HiveProject(t3id=[$0])
+ HiveTableScan(table=[[default, antijoin3_t3]], table:alias=[antijoin3_t3])
+
+PREHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+
+where t3id is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@antijoin3_t1
+PREHOOK: Input: default@antijoin3_t2
+PREHOOK: Input: default@antijoin3_t3
+#### A masked pattern was here ####
+POSTHOOK: query: select t1id, t1notnull, t1nullable from antijoin3_t1 t1
+left join (select * from antijoin3_t2 left join antijoin3_t3 on t2id=t3id) sq
+on t1id=t2id
+
+where t3id is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@antijoin3_t1
+POSTHOOK: Input: default@antijoin3_t2
+POSTHOOK: Input: default@antijoin3_t3
+#### A masked pattern was here ####
+1 val_1 NULL
+2 val_2 val_2
+3 val_3 val_3
diff --git a/ql/src/test/results/clientpositive/llap/llap_smb_ptf.q.out b/ql/src/test/results/clientpositive/llap/llap_smb_ptf.q.out
index f9f128b394c..ba1d7fdde05 100644
--- a/ql/src/test/results/clientpositive/llap/llap_smb_ptf.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_smb_ptf.q.out
@@ -586,12 +586,18 @@ STAGE PLANS:
expressions: _col0 (type: int)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Group By Operator
+ keys: _col0 (type: int)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
@@ -663,8 +669,8 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
Statistics: Num rows: 1 Data size: 304 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col7 (type: int), _col0 (type: smallint), _col1 (type: string), _col2 (type: smallint), _col3 (type: string), _col5 (type: smallint), _col6 (type: string), _col8 (type: smallint), _col9 (type: smallint), _col10 (type: string), _col11 (type: string), _col12 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ expressions: _col7 (type: int), _col0 (type: smallint), _col1 (type: string), _col2 (type: smallint), _col3 (type: string), _col5 (type: smallint), _col6 (type: string), _col8 (type: smallint), _col9 (type: smallint), _col11 (type: string), _col12 (type: string), _col10 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
Statistics: Num rows: 1 Data size: 304 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
@@ -672,36 +678,29 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1 Data size: 304 Basic stats: PARTIAL Column stats: NONE
- value expressions: _col1 (type: smallint), _col2 (type: string), _col3 (type: smallint), _col4 (type: string), _col6 (type: smallint), _col7 (type: string), _col8 (type: smallint), _col9 (type: smallint), _col10 (type: string), _col11 (type: string), _col12 (type: string)
+ value expressions: _col1 (type: smallint), _col2 (type: string), _col3 (type: smallint), _col4 (type: string), _col5 (type: smallint), _col6 (type: string), _col7 (type: smallint), _col8 (type: smallint), _col9 (type: string), _col10 (type: string), _col11 (type: string)
Reducer 5
Execution mode: llap
Reduce Operator Tree:
Merge Join Operator
condition map:
- Left Outer Join 0 to 1
+ Anti Join 0 to 1
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col14
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
Statistics: Num rows: 1 Data size: 334 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col0 (type: int), _col1 (type: smallint), _col2 (type: string), _col3 (type: smallint), _col4 (type: string), _col6 (type: smallint), _col7 (type: string), _col8 (type: smallint), _col9 (type: smallint), _col11 (type: string), _col12 (type: string), _col10 (type: string), _col14 (type: int)
+ expressions: _col0 (type: int), _col1 (type: smallint), _col2 (type: string), _col3 (type: smallint), _col4 (type: string), _col5 (type: smallint), _col6 (type: string), _col7 (type: smallint), _col8 (type: smallint), _col9 (type: string), _col10 (type: string), _col11 (type: string), '201611160940' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
Statistics: Num rows: 1 Data size: 334 Basic stats: PARTIAL Column stats: NONE
- Filter Operator
- predicate: _col12 is null (type: boolean)
+ File Output Operator
+ compressed: false
Statistics: Num rows: 1 Data size: 334 Basic stats: PARTIAL Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: smallint), _col2 (type: string), _col3 (type: smallint), _col4 (type: string), _col5 (type: smallint), _col6 (type: string), _col7 (type: smallint), _col8 (type: smallint), _col9 (type: string), _col10 (type: string), _col11 (type: string), '201611160940' (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
- Statistics: Num rows: 1 Data size: 334 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 334 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Reducer 7
Execution mode: vectorized, llap
Reduce Operator Tree:
(hive) 02/05: HIVE-29304: HS2 fails to start if cluster is configured with a mix of LDAP and Kerberos (#6166)
Posted by dk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/hive.git
commit c6e577ca1fa9306a00ed087df67c46fb2c4500fe
Author: dengzh <de...@gmail.com>
AuthorDate: Thu Nov 13 09:14:07 2025 +0800
HIVE-29304: HS2 fails to start if cluster is configured with a mix of LDAP and Kerberos (#6166)
(cherry picked from commit 8d7c9c42f76bbd0a258640b038ac305b9e71169d)
---
jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java | 2 +-
service/src/java/org/apache/hive/service/server/HiveServer2.java | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
index 6ab722b7371..00ef4785a70 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
@@ -208,7 +208,7 @@ static void configureConnParams(JdbcConnectionParams connParams) throws ZooKeepe
try (CuratorFramework zooKeeperClient = getZkClient(connParams)) {
final List<String> serverHosts = getServerHosts(connParams, zooKeeperClient);
if (serverHosts.isEmpty()) {
- throw new ZooKeeperHiveClientException("No more HiveServer2 URIs from ZooKeeper to attempt");
+ throw new ZooKeeperHiveClientException("Empty HiveServer2 URIs found from ZooKeeper to attempt");
}
// Pick a server node randomly
final String serverNode = serverHosts.get(ThreadLocalRandom.current().nextInt(serverHosts.size()));
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index c145283c120..ced00eeb100 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -99,6 +99,7 @@
import org.apache.hadoop.hive.ql.txn.compactor.Worker;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.common.IPStackUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.hive.common.util.HiveVersionInfo;
@@ -606,7 +607,7 @@ public static boolean isAllTransportMode(HiveConf hiveConf) {
private ACLProvider getACLProvider(HiveConf hiveConf) {
final boolean isSecure =
- AuthType.isKerberosAuthMode(hiveConf) &&
+ UserGroupInformation.isSecurityEnabled() &&
HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ZOOKEEPER_USE_KERBEROS);
return new ACLProvider() {
@@ -683,7 +684,7 @@ private void addConfsToPublish(HiveConf hiveConf, Map<String, String> confsToPub
*/
private static void setUpZooKeeperAuth(HiveConf hiveConf) {
try {
- if (AuthType.isKerberosAuthMode(hiveConf) &&
+ if (UserGroupInformation.isSecurityEnabled() &&
StringUtils.isNotEmpty(HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM)) &&
HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ZOOKEEPER_USE_KERBEROS)) {
// Install the JAAS Configuration for the runtime
(hive) 04/05: HIVE-29254: Display TxnId associated with the query in show processlist command (#6141)
Posted by dk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/hive.git
commit dec805a6a0a0d299ce471dfdc38028ead7e84aa8
Author: tarak271 <ta...@cloudera.com>
AuthorDate: Thu Nov 13 13:26:01 2025 +0530
HIVE-29254: Display TxnId associated with the query in show processlist command (#6141)
* HIVE-29254-Display TxnId associated with the query in show processlist command
* HIVE-29254-Display TxnId associated with the query in show processlist command
* HIVE-29254-Display TxnId associated with the query in show processlist command
* HIVE-29254-Display TxnId associated with the query in show processlist command
* HIVE-29254-Display TxnId associated with the query in show processlist command
(cherry picked from commit f6e29e9853c5fe33a672e19bb45962a5a8d34366)
---
.../hive/ql/processor/TestShowProcessList.java | 144 +++++++++++++++++++++
.../ql/processors/ShowProcessListProcessor.java | 2 +
.../hadoop/hive/ql/session/ProcessListInfo.java | 16 ++-
.../cli/operation/ShowProcessListOperation.java | 6 +-
4 files changed, 165 insertions(+), 3 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/processor/TestShowProcessList.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/processor/TestShowProcessList.java
new file mode 100644
index 00000000000..7c57db9b249
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/processor/TestShowProcessList.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.processor;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URI;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Collections;
+
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+public class TestShowProcessList {
+ protected static final Logger LOG = LoggerFactory.getLogger(TestShowProcessList.class);
+
+ private static MiniHS2 miniHS2 = null;
+ private static HiveConf conf;
+ private static String user;
+ private static ThreadPoolExecutor executor;
+
+ static HiveConf defaultConf() throws Exception {
+ String confDir = "../../data/conf/llap/";
+ HiveConf.setHiveSiteLocation(new URI("file://" + new File(confDir).toURI().getPath() + "/hive-site.xml").toURL());
+ System.out.println("Setting hive-site: " + HiveConf.getHiveSiteLocation());
+ HiveConf defaultConf = new HiveConf();
+ defaultConf.addResource(new URI("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml").toURL());
+ return defaultConf;
+ }
+
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ conf = defaultConf();
+ user = System.getProperty("user.name");
+ conf.setVar(HiveConf.ConfVars.USERS_IN_ADMIN_ROLE, user);
+ conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
+ "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
+ String suffix = TestShowProcessList.class.getCanonicalName() + "-" + System.currentTimeMillis();
+ String dir = new File(System.getProperty("java.io.tmpdir") + File.separator + suffix).getPath()
+ .replaceAll("\\\\", "/") + "/warehouse";
+ conf.set(MetastoreConf.ConfVars.WAREHOUSE.name(), dir);
+ TestTxnDbUtil.setConfValues(conf);
+ TestTxnDbUtil.prepDb(conf);
+ MiniHS2.cleanupLocalDir();
+ Class.forName(MiniHS2.getJdbcDriverName());
+ miniHS2 = new MiniHS2(conf, MiniHS2.MiniClusterType.LLAP);
+ Map<String, String> confOverlay = new HashMap<>();
+ miniHS2.start(confOverlay);
+
+ executor = new ThreadPoolExecutor(5, 10, 50, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
+ }
+
+ @AfterClass
+ public static void afterTest() throws Exception {
+ if (miniHS2 != null && miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ TestTxnDbUtil.cleanDb(conf);
+ }
+
+ @Test
+ public void testQueries() throws Exception {
+ int connections = 10;
+ //Initiate 10 parallel connections, each with a query that begins a transaction.
+ for (int i = 0; i < connections; i++) {
+ executor.submit(() -> {
+ try (Connection con = DriverManager.getConnection(miniHS2.getJdbcURL(), user, "bar");
+ Statement stmt = con.createStatement()) {
+ stmt.execute("drop database if exists DB_" + Thread.currentThread().threadId() + " cascade");
+ } catch (Exception exception) {
+ LOG.error(exception.getMessage());
+ LOG.error(Arrays.toString(exception.getStackTrace()));
+ }
+ });
+ }
+ Set<Integer> txnIds = new HashSet<>();
+ try (Connection testCon = DriverManager.getConnection(miniHS2.getJdbcURL(), user, "bar");
+ Statement s = testCon.createStatement()) {
+ while (executor.getActiveCount() > 0) {
+ // retrieve txnIds from show processlist output
+ txnIds.addAll(getTxnIdsFromShowProcesslist(s));
+ }
+ }
+ System.out.println(txnIds);
+ // max txnId should be equal to the number of connections
+ int maxTxnId = Collections.max(txnIds);
+ Assert.assertEquals(maxTxnId, connections);
+ }
+
+ private static Set<Integer> getTxnIdsFromShowProcesslist(Statement s) {
+ Set<Integer> txnIds = new HashSet<>();
+ try (ResultSet rs = s.executeQuery("show processlist")) {
+ while (rs.next()) {
+ int txnId = Integer.parseInt(rs.getString("Txn ID"));
+ // TxnId can be 0 because the query has not yet opened txn when show processlist is run.
+ if (txnId > 0) {
+ txnIds.add(txnId);
+ }
+ }
+ } catch (Exception exception) {
+ LOG.error("Exception when checking hive state", exception);
+ LOG.error(Arrays.toString(exception.getStackTrace()));
+
+ }
+ return txnIds;
+ }
+
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/ShowProcessListProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/ShowProcessListProcessor.java
index 90182aaaab6..985cd74e6ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/ShowProcessListProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/ShowProcessListProcessor.java
@@ -59,6 +59,7 @@ private Schema getSchema() {
sch.addToFieldSchemas(new FieldSchema("Session Idle Time (s)", STRING_TYPE_NAME, ""));
sch.addToFieldSchemas(new FieldSchema("Query ID", STRING_TYPE_NAME, ""));
sch.addToFieldSchemas(new FieldSchema("State", STRING_TYPE_NAME, ""));
+ sch.addToFieldSchemas(new FieldSchema("Txn ID", STRING_TYPE_NAME, ""));
sch.addToFieldSchemas(new FieldSchema("Opened Timestamp (s)", STRING_TYPE_NAME, ""));
sch.addToFieldSchemas(new FieldSchema("Elapsed Time (s)", STRING_TYPE_NAME, ""));
sch.addToFieldSchemas(new FieldSchema("Runtime (s)", STRING_TYPE_NAME, ""));
@@ -91,6 +92,7 @@ public CommandProcessorResponse run(String command) throws CommandProcessorExcep
query.getSessionIdleTime(),
query.getQueryId(),
query.getState(),
+ query.getTxnId(),
query.getBeginTime(),
query.getElapsedTime(),
query.getRuntime()
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/ProcessListInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/session/ProcessListInfo.java
index 19e31157f8b..04b59141b6d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/ProcessListInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/ProcessListInfo.java
@@ -32,10 +32,11 @@ public class ProcessListInfo {
private final String runtime; // tracks only running portion of the query.
private final long elapsedTime;
private final String state;
+ private final long txnId;
private ProcessListInfo(String userName, String ipAddr, String sessionId, long sessionActiveTime,
long sessionIdleTime, String queryId, String executionEngine, String beginTime,
- String runtime, long elapsedTime, String state) {
+ String runtime, long elapsedTime, String state, long txnId) {
this.userName = userName;
this.ipAddr = ipAddr;
this.sessionId = sessionId;
@@ -47,6 +48,7 @@ private ProcessListInfo(String userName, String ipAddr, String sessionId, long s
this.runtime = runtime;
this.elapsedTime = elapsedTime;
this.state = state;
+ this.txnId = txnId;
}
public String getSessionId() {
@@ -93,6 +95,10 @@ public String getState() {
return state;
}
+ public Long getTxnId() {
+ return txnId;
+ }
+
public static class Builder {
private String userName;
private String ipAddr;
@@ -105,6 +111,7 @@ public static class Builder {
private String runtime;
private long elapsedTime;
private String state;
+ private long txnId;
public Builder setSessionId(String sessionId) {
this.sessionId = sessionId;
@@ -161,10 +168,15 @@ public Builder setState(String state) {
return this;
}
+ public Builder setTxnId(long txnId) {
+ this.txnId = txnId;
+ return this;
+ }
+
public ProcessListInfo build() {
ProcessListInfo processListInfo = new ProcessListInfo(userName, ipAddr, sessionId, sessionActiveTime,
sessionIdleTime, queryId, executionEngine, beginTime, runtime,
- elapsedTime, state);
+ elapsedTime, state, txnId);
return processListInfo;
}
}
diff --git a/service/src/java/org/apache/hive/service/cli/operation/ShowProcessListOperation.java b/service/src/java/org/apache/hive/service/cli/operation/ShowProcessListOperation.java
index 12143b97085..e667f3f6b91 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/ShowProcessListOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/ShowProcessListOperation.java
@@ -71,7 +71,10 @@ private List<ProcessListInfo> getLiveQueryInfos(HiveSession parentSession) {
LocalDateTime beginTime = LocalDateTime.ofInstant(
Instant.ofEpochMilli(query.getBeginTime()), ZoneId.systemDefault()
);
-
+ long txnId = 0;
+ if (op.queryState != null && op.queryState.getTxnManager() != null) {
+ txnId = op.queryState.getTxnManager().getCurrentTxnId();
+ }
return new ProcessListInfo.Builder()
.setUserName(session.getUserName())
.setIpAddr(session.getIpAddress())
@@ -84,6 +87,7 @@ private List<ProcessListInfo> getLiveQueryInfos(HiveSession parentSession) {
.setRuntime(query.getRuntime() == null ? "Not finished" : String.valueOf(query.getRuntime() / 1000))
.setElapsedTime(query.getElapsedTime() / 1000)
.setState(query.getState())
+ .setTxnId(txnId)
.build();
})
.collect(Collectors.toList());