You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2022/05/02 06:35:41 UTC
[hive] branch master updated: HIVE-26193: Fix Iceberg partitioned tables null bucket handling (Peter Vary reviewed by Marton Bod and Adam Szita) (#3261)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 94b383c82a HIVE-26193: Fix Iceberg partitioned tables null bucket handling (Peter Vary reviewed by Marton Bod and Adam Szita) (#3261)
94b383c82a is described below
commit 94b383c82a7c5d2e619c3bab84dfca88b4591b4e
Author: pvary <pv...@cloudera.com>
AuthorDate: Mon May 2 08:35:29 2022 +0200
HIVE-26193: Fix Iceberg partitioned tables null bucket handling (Peter Vary reviewed by Marton Bod and Adam Szita) (#3261)
---
.../iceberg/mr/hive/GenericUDFIcebergBucket.java | 2 +-
.../queries/positive/dynamic_partition_writes.q | 4 +-
.../positive/dynamic_partition_writes.q.out | 43 ++++++++++++++--------
.../hive/ql/exec/vector/VectorizedRowBatchCtx.java | 4 +-
4 files changed, 32 insertions(+), 21 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/GenericUDFIcebergBucket.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/GenericUDFIcebergBucket.java
index cab4bb11ab..52b0a1edbf 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/GenericUDFIcebergBucket.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/GenericUDFIcebergBucket.java
@@ -185,7 +185,7 @@ public class GenericUDFIcebergBucket extends GenericUDF {
public Object evaluate(DeferredObject[] arguments) throws HiveException {
DeferredObject argument = arguments[0];
- if (argument == null) {
+ if (argument == null || argument.get() == null) {
return null;
} else {
evaluator.apply(argument);
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_writes.q b/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_writes.q
index 93094529f6..622e870350 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_writes.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/dynamic_partition_writes.q
@@ -10,9 +10,9 @@ drop table if exists tbl_target_mixed;
create external table tbl_src (a int, b string, c bigint) stored by iceberg stored as orc;
-insert into tbl_src values (1, 'EUR', 10), (2, 'EUR', 10), (3, 'USD', 11), (4, 'EUR', 12), (5, 'HUF', 30), (6, 'USD', 10), (7, 'USD', 100), (8, 'PLN', 20), (9, 'PLN', 11), (10, 'CZK', 5);
+insert into tbl_src values (1, 'EUR', 10), (2, 'EUR', 10), (3, 'USD', 11), (4, 'EUR', 12), (5, 'HUF', 30), (6, 'USD', 10), (7, 'USD', 100), (8, 'PLN', 20), (9, 'PLN', 11), (10, 'CZK', 5), (12, NULL, NULL);
--need at least 2 files to ensure ClusteredWriter encounters out-of-order records
-insert into tbl_src values (10, 'EUR', 12), (20, 'EUR', 11), (30, 'USD', 100), (40, 'EUR', 10), (50, 'HUF', 30), (60, 'USD', 12), (70, 'USD', 20), (80, 'PLN', 100), (90, 'PLN', 18), (100, 'CZK', 12);
+insert into tbl_src values (10, 'EUR', 12), (20, 'EUR', 11), (30, 'USD', 100), (40, 'EUR', 10), (50, 'HUF', 30), (60, 'USD', 12), (70, 'USD', 20), (80, 'PLN', 100), (90, 'PLN', 18), (100, 'CZK', 12), (110, NULL, NULL);
create external table tbl_target_identity (a int) partitioned by (ccy string) stored by iceberg stored as orc;
explain insert overwrite table tbl_target_identity select a, b from tbl_src;
diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out
index 36fb39c091..53be8c39ed 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out
@@ -22,19 +22,19 @@ POSTHOOK: query: create external table tbl_src (a int, b string, c bigint) store
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tbl_src
-PREHOOK: query: insert into tbl_src values (1, 'EUR', 10), (2, 'EUR', 10), (3, 'USD', 11), (4, 'EUR', 12), (5, 'HUF', 30), (6, 'USD', 10), (7, 'USD', 100), (8, 'PLN', 20), (9, 'PLN', 11), (10, 'CZK', 5)
+PREHOOK: query: insert into tbl_src values (1, 'EUR', 10), (2, 'EUR', 10), (3, 'USD', 11), (4, 'EUR', 12), (5, 'HUF', 30), (6, 'USD', 10), (7, 'USD', 100), (8, 'PLN', 20), (9, 'PLN', 11), (10, 'CZK', 5), (12, NULL, NULL)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@tbl_src
-POSTHOOK: query: insert into tbl_src values (1, 'EUR', 10), (2, 'EUR', 10), (3, 'USD', 11), (4, 'EUR', 12), (5, 'HUF', 30), (6, 'USD', 10), (7, 'USD', 100), (8, 'PLN', 20), (9, 'PLN', 11), (10, 'CZK', 5)
+POSTHOOK: query: insert into tbl_src values (1, 'EUR', 10), (2, 'EUR', 10), (3, 'USD', 11), (4, 'EUR', 12), (5, 'HUF', 30), (6, 'USD', 10), (7, 'USD', 100), (8, 'PLN', 20), (9, 'PLN', 11), (10, 'CZK', 5), (12, NULL, NULL)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_src
-PREHOOK: query: insert into tbl_src values (10, 'EUR', 12), (20, 'EUR', 11), (30, 'USD', 100), (40, 'EUR', 10), (50, 'HUF', 30), (60, 'USD', 12), (70, 'USD', 20), (80, 'PLN', 100), (90, 'PLN', 18), (100, 'CZK', 12)
+PREHOOK: query: insert into tbl_src values (10, 'EUR', 12), (20, 'EUR', 11), (30, 'USD', 100), (40, 'EUR', 10), (50, 'HUF', 30), (60, 'USD', 12), (70, 'USD', 20), (80, 'PLN', 100), (90, 'PLN', 18), (100, 'CZK', 12), (110, NULL, NULL)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@tbl_src
-POSTHOOK: query: insert into tbl_src values (10, 'EUR', 12), (20, 'EUR', 11), (30, 'USD', 100), (40, 'EUR', 10), (50, 'HUF', 30), (60, 'USD', 12), (70, 'USD', 20), (80, 'PLN', 100), (90, 'PLN', 18), (100, 'CZK', 12)
+POSTHOOK: query: insert into tbl_src values (10, 'EUR', 12), (20, 'EUR', 11), (30, 'USD', 100), (40, 'EUR', 10), (50, 'HUF', 30), (60, 'USD', 12), (70, 'USD', 20), (80, 'PLN', 100), (90, 'PLN', 18), (100, 'CZK', 12), (110, NULL, NULL)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_src
@@ -76,9 +76,9 @@ Stage-3
<-Map 1 [SIMPLE_EDGE] vectorized
PARTITION_ONLY_SHUFFLE [RS_13]
PartitionCols:_col1
- Select Operator [SEL_12] (rows=20 width=91)
+ Select Operator [SEL_12] (rows=22 width=87)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=20 width=91)
+ TableScan [TS_0] (rows=22 width=87)
default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"]
Reducer 3 vectorized
File Output Operator [FS_21]
@@ -90,7 +90,7 @@ Stage-3
PARTITION_ONLY_SHUFFLE [RS_16]
Group By Operator [GBY_15] (rows=1 width=400)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"]
- Select Operator [SEL_14] (rows=20 width=91)
+ Select Operator [SEL_14] (rows=22 width=87)
Output:["a","ccy"]
Please refer to the previous Select Operator [SEL_12]
@@ -121,6 +121,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
9 PLN
10 CZK
10 EUR
+12 NULL
20 EUR
30 USD
40 EUR
@@ -130,6 +131,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
80 PLN
90 PLN
100 CZK
+110 NULL
PREHOOK: query: create external table tbl_target_bucket (a int, ccy string) partitioned by spec (bucket (2, ccy)) stored by iceberg stored as orc
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
@@ -168,9 +170,9 @@ Stage-3
<-Map 1 [SIMPLE_EDGE] vectorized
PARTITION_ONLY_SHUFFLE [RS_13]
PartitionCols:iceberg_bucket(_col1, 2)
- Select Operator [SEL_12] (rows=20 width=91)
+ Select Operator [SEL_12] (rows=22 width=87)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=20 width=91)
+ TableScan [TS_0] (rows=22 width=87)
default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"]
Reducer 3 vectorized
File Output Operator [FS_21]
@@ -182,7 +184,7 @@ Stage-3
PARTITION_ONLY_SHUFFLE [RS_16]
Group By Operator [GBY_15] (rows=1 width=400)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"]
- Select Operator [SEL_14] (rows=20 width=91)
+ Select Operator [SEL_14] (rows=22 width=87)
Output:["a","ccy"]
Please refer to the previous Select Operator [SEL_12]
@@ -213,6 +215,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
9 PLN
10 CZK
10 EUR
+12 NULL
20 EUR
30 USD
40 EUR
@@ -222,6 +225,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
80 PLN
90 PLN
100 CZK
+110 NULL
PREHOOK: query: create external table tbl_target_mixed (a int, ccy string, c bigint) partitioned by spec (ccy, bucket (3, c)) stored by iceberg stored as orc
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
@@ -260,9 +264,9 @@ Stage-3
<-Map 1 [SIMPLE_EDGE] vectorized
PARTITION_ONLY_SHUFFLE [RS_13]
PartitionCols:_col1, iceberg_bucket(_col2, 3)
- Select Operator [SEL_12] (rows=20 width=99)
+ Select Operator [SEL_12] (rows=22 width=94)
Output:["_col0","_col1","_col2"]
- TableScan [TS_0] (rows=20 width=99)
+ TableScan [TS_0] (rows=22 width=94)
default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"]
Reducer 3 vectorized
File Output Operator [FS_21]
@@ -274,7 +278,7 @@ Stage-3
PARTITION_ONLY_SHUFFLE [RS_16]
Group By Operator [GBY_15] (rows=1 width=568)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["min(a)","max(a)","count(1)","count(a)","compute_bit_vector_hll(a)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)","min(c)","max(c)","count(c)","compute_bit_vector_hll(c)"]
- Select Operator [SEL_14] (rows=20 width=99)
+ Select Operator [SEL_14] (rows=22 width=94)
Output:["a","ccy","c"]
Please refer to the previous Select Operator [SEL_12]
@@ -305,6 +309,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
9 PLN 11
10 CZK 5
10 EUR 12
+12 NULL NULL
20 EUR 11
30 USD 100
40 EUR 10
@@ -314,6 +319,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
80 PLN 100
90 PLN 18
100 CZK 12
+110 NULL NULL
PREHOOK: query: select * from default.tbl_target_mixed.partitions
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_target_mixed
@@ -324,14 +330,15 @@ POSTHOOK: Input: default@tbl_target_mixed
POSTHOOK: Output: hdfs://### HDFS PATH ###
{"ccy":"EUR","c_bucket":0} 1 1
{"ccy":"EUR","c_bucket":1} 2 1
-{"ccy":"HUF","c_bucket":1} 2 1
{"ccy":"EUR","c_bucket":2} 3 1
{"ccy":"USD","c_bucket":1} 3 1
{"ccy":"CZK","c_bucket":1} 1 1
{"ccy":"USD","c_bucket":0} 2 1
{"ccy":"USD","c_bucket":2} 1 1
{"ccy":"CZK","c_bucket":2} 1 1
+{"ccy":"HUF","c_bucket":1} 2 1
{"ccy":"PLN","c_bucket":2} 1 1
+{"ccy":null,"c_bucket":null} 2 1
{"ccy":"PLN","c_bucket":0} 2 1
{"ccy":"PLN","c_bucket":1} 1 1
PREHOOK: query: select * from default.tbl_target_mixed.files
@@ -342,6 +349,7 @@ POSTHOOK: query: select * from default.tbl_target_mixed.files
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl_target_mixed
POSTHOOK: Output: hdfs://### HDFS PATH ###
+0 hdfs://### HDFS PATH ### ORC 0 {"ccy":null,"c_bucket":null} 2 417 {1:7,2:5,3:5} {1:2,2:2,3:2} {1:0,2:2,3:2} {} {1: } {1:n } NULL [3] NULL 0
0 hdfs://### HDFS PATH ### ORC 0 {"ccy":"CZK","c_bucket":1} 1 449 {1:6,2:12,3:6} {1:1,2:1,3:1} {1:0,2:0,3:0} {} {1:d ,2:CZK,3: } {1:d ,2:CZK,3: } NULL [3] NULL 0
0 hdfs://### HDFS PATH ### ORC 0 {"ccy":"CZK","c_bucket":2} 1 432 {1:6,2:12,3:6} {1:1,2:1,3:1} {1:0,2:0,3:0} {} {1:
,2:CZK,3: } {1:
@@ -395,7 +403,7 @@ Stage-3
Output:["_col0","_col1","_col2"]
Filter Operator [FIL_14] (rows=4 width=99)
predicate:(b = 'EUR')
- TableScan [TS_0] (rows=20 width=99)
+ TableScan [TS_0] (rows=22 width=94)
default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"]
Reducer 3 vectorized
File Output Operator [FS_24]
@@ -453,7 +461,7 @@ Stage-3
Output:["_col0","_col1","_col2"]
Filter Operator [FIL_12] (rows=1 width=99)
predicate:((c = 100L) and (b = 'USD'))
- TableScan [TS_0] (rows=20 width=99)
+ TableScan [TS_0] (rows=22 width=94)
default@tbl_src,tbl_src,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"]
PARTITION_ONLY_SHUFFLE [RS_17]
Group By Operator [GBY_16] (rows=1 width=568)
@@ -494,6 +502,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
10 CZK 5
10 EUR 12
10 EUR 12
+12 NULL NULL
20 EUR 11
20 EUR 11
30 USD 100
@@ -506,6 +515,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
80 PLN 100
90 PLN 18
100 CZK 12
+110 NULL NULL
PREHOOK: query: select * from default.tbl_target_mixed.files
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_target_mixed
@@ -521,6 +531,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
0 hdfs://### HDFS PATH ### ORC 0 {"ccy":"EUR","c_bucket":2} 3 448 {1:8,2:17,3:5} {1:3,2:3,3:3} {1:0,2:0,3:0} {} {1: ,2:EUR,3:
} {1:( ,2:EUR,3:
} NULL [3] NULL 0
+0 hdfs://### HDFS PATH ### ORC 0 {"ccy":null,"c_bucket":null} 2 417 {1:7,2:5,3:5} {1:2,2:2,3:2} {1:0,2:2,3:2} {} {1: } {1:n } NULL [3] NULL 0
0 hdfs://### HDFS PATH ### ORC 0 {"ccy":"CZK","c_bucket":1} 1 449 {1:6,2:12,3:6} {1:1,2:1,3:1} {1:0,2:0,3:0} {} {1:d ,2:CZK,3: } {1:d ,2:CZK,3: } NULL [3] NULL 0
0 hdfs://### HDFS PATH ### ORC 0 {"ccy":"CZK","c_bucket":2} 1 432 {1:6,2:12,3:6} {1:1,2:1,3:1} {1:0,2:0,3:0} {} {1:
,2:CZK,3: } {1:
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
index cdb5b170d1..0abbd59b9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
@@ -608,12 +608,12 @@ public class VectorizedRowBatchCtx {
case CHAR:
case VARCHAR: {
BytesColumnVector bcv = (BytesColumnVector) col;
- String sVal = value.toString();
- if (sVal == null) {
+ if (value == null) {
bcv.noNulls = false;
bcv.isNull[0] = true;
bcv.isRepeating = true;
} else {
+ String sVal = value.toString();
bcv.fill(sVal.getBytes());
}
}