You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2025/07/21 20:40:33 UTC
(impala) branch master updated (8a691a350 -> 365ce0b12)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 8a691a350 IMPALA-12648: [DOCS] Documentation for Kill Query statements
new ee69ed1d0 IMPALA-13625: Allow reading Parquet int32/int64 as decimal without logical types
new 365ce0b12 IMPALA-11512: Add tests for BINARY type support in Iceberg
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/parquet/parquet-data-converter.h | 21 ++
be/src/exec/parquet/parquet-metadata-utils.cc | 23 +-
.../queries/QueryTest/iceberg-binary-type.test | 292 +++++++++++++++++++++
.../queries/QueryTest/parquet-type-widening.test | 82 +++++-
tests/query_test/test_iceberg.py | 3 +
5 files changed, 418 insertions(+), 3 deletions(-)
create mode 100644 testdata/workloads/functional-query/queries/QueryTest/iceberg-binary-type.test
(impala) 01/02: IMPALA-13625: Allow reading Parquet int32/int64 as decimal without logical types
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ee69ed1d0386f41689269a522e2aed490e52987d
Author: Daniel Vanko <dv...@cloudera.com>
AuthorDate: Wed Jun 25 11:42:51 2025 +0200
IMPALA-13625: Allow reading Parquet int32/int64 as decimal without logical types
This patch allows reading columns with integer logical type as decimals.
This can occur when we're trying to read files that were written as INT but
the column was altered to a suitable DECIMAL. In this case the precision
is based on physical type and equals 9 and 18, for int32 and int64
respectively.
Test:
* add new e2e tests
Change-Id: I56006eb3cca28c81ec8467d77b35005fbf669680
Reviewed-on: http://gerrit.cloudera.org:8080/22922
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exec/parquet/parquet-data-converter.h | 21 ++++++
be/src/exec/parquet/parquet-metadata-utils.cc | 23 +++++-
.../queries/QueryTest/parquet-type-widening.test | 82 +++++++++++++++++++++-
3 files changed, 123 insertions(+), 3 deletions(-)
diff --git a/be/src/exec/parquet/parquet-data-converter.h b/be/src/exec/parquet/parquet-data-converter.h
index eb042362e..93c403bef 100644
--- a/be/src/exec/parquet/parquet-data-converter.h
+++ b/be/src/exec/parquet/parquet-data-converter.h
@@ -70,6 +70,21 @@ class ParquetDataConverter {
}
int32_t GetPrecision() const {
+ // If logical type is INTEGER, the precision is determined by the physical type.
+ if (parquet_element_->__isset.logicalType
+ && UNLIKELY(parquet_element_->logicalType.__isset.INTEGER)) {
+ switch (parquet_element_->type) {
+ case parquet::Type::INT32:
+ return ColumnType::MAX_DECIMAL4_PRECISION + 1;
+ case parquet::Type::INT64:
+ return ColumnType::MAX_DECIMAL8_PRECISION + 1;
+ default:
+ DCHECK(false) << "Unexpected physical type for INTEGER logical type: "
+ << to_string(parquet_element_->type);
+ break;
+ }
+ }
+
if (parquet_element_->__isset.logicalType
&& parquet_element_->logicalType.__isset.DECIMAL) {
return parquet_element_->logicalType.DECIMAL.precision;
@@ -77,6 +92,7 @@ class ParquetDataConverter {
return parquet_element_->precision;
}
+
/// Returns true if we need to do a conversion from the Parquet type to the slot type.
bool CheckIfNeedsConversion() {
if (!MATERIALIZED) return false;
@@ -87,6 +103,11 @@ class ParquetDataConverter {
return true;
}
if (col_type_->type == TYPE_DECIMAL) {
+ // If the logical type is INTEGER for a Decimal slot, conversion is needed.
+ if (parquet_element_->__isset.logicalType
+ && UNLIKELY(parquet_element_->logicalType.__isset.INTEGER)) {
+ return true;
+ }
if (col_type_->precision != GetPrecision()) {
// Decimal values can be stored by Decimal4Value (4 bytes), Decimal8Value, and
// Decimal16Value. We only need to do a conversion for different precision if
diff --git a/be/src/exec/parquet/parquet-metadata-utils.cc b/be/src/exec/parquet/parquet-metadata-utils.cc
index 2cb1c3bad..f2749f6ac 100644
--- a/be/src/exec/parquet/parquet-metadata-utils.cc
+++ b/be/src/exec/parquet/parquet-metadata-utils.cc
@@ -89,6 +89,11 @@ bool IsSupportedType(PrimitiveType impala_type,
return encodings->second.find(parquet_type) != encodings->second.end();
}
+/// Returns true if Parquet's logical type is INTEGER.
+bool IsIntLogicalType(const parquet::SchemaElement& element) {
+ return element.__isset.logicalType && element.logicalType.__isset.INTEGER;
+}
+
/// Returns true if encoding 'e' is supported by Impala, false otherwise.
static bool IsEncodingSupported(parquet::Encoding::type e) {
switch (e) {
@@ -208,7 +213,21 @@ int32_t GetScale(const parquet::SchemaElement& schema_element) {
}
// Precision is required, this should be called after checking IsPrecisionSet()
+// unless logical type is INTEGER, in which case the precision is based on physical type.
int32_t GetPrecision(const parquet::SchemaElement& schema_element) {
+ if (UNLIKELY(IsIntLogicalType(schema_element))) {
+ switch (schema_element.type) {
+ case parquet::Type::INT32:
+ return ColumnType::MAX_DECIMAL4_PRECISION + 1;
+ case parquet::Type::INT64:
+ return ColumnType::MAX_DECIMAL8_PRECISION + 1;
+ default:
+ DCHECK(false) << "Unexpected physical type for INTEGER logical type: "
+ << to_string(schema_element.type);
+ break;
+ }
+ }
+
DCHECK(IsPrecisionSet(schema_element));
if (schema_element.__isset.logicalType && schema_element.logicalType.__isset.DECIMAL) {
return schema_element.logicalType.DECIMAL.precision;
@@ -370,7 +389,7 @@ Status ParquetMetadataUtils::ValidateColumn(const char* filename,
// We require that the precision be a positive value, and not larger than the
// precision in table schema.
- if (!IsPrecisionSet(schema_element)) {
+ if (!IsPrecisionSet(schema_element) && !IsIntLogicalType(schema_element)) {
ErrorMsg msg(TErrorCode::PARQUET_MISSING_PRECISION, filename, schema_element.name);
return Status(msg);
} else {
@@ -388,7 +407,7 @@ Status ParquetMetadataUtils::ValidateColumn(const char* filename,
}
}
- if (!is_converted_type_decimal) {
+ if (!is_converted_type_decimal && !IsIntLogicalType(schema_element)) {
// TODO: is this validation useful? It is not required at all to read the data and
// might only serve to reject otherwise perfectly readable files.
ErrorMsg msg(TErrorCode::PARQUET_BAD_CONVERTED_TYPE, filename,
diff --git a/testdata/workloads/functional-query/queries/QueryTest/parquet-type-widening.test b/testdata/workloads/functional-query/queries/QueryTest/parquet-type-widening.test
index f0f11c788..74c951e23 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/parquet-type-widening.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/parquet-type-widening.test
@@ -6,4 +6,84 @@ select * from primitive_type_widening;
10,20,30,40,50,60,70,80,90,1230.4560546875
---- TYPES
SMALLINT,INT,BIGINT,DOUBLE,INT,BIGINT,DOUBLE,INT,DOUBLE,DOUBLE
-====
\ No newline at end of file
+====
+---- QUERY
+# IMPALA-13625: Allow reading Parquet int32/int64 as decimal without logical types
+create table read_int_as_decimal (c1 int, c2 bigint) stored as parquet;
+insert into read_int_as_decimal
+values
+ (1, cast(10000000000 as bigint)),
+ (MIN_INT(), MIN_BIGINT()),
+ (MAX_INT(), MAX_BIGINT());
+select * from read_int_as_decimal order by c1;
+---- RESULTS
+-2147483648,-9223372036854775808
+1,10000000000
+2147483647,9223372036854775807
+---- TYPES
+INT, BIGINT
+====
+---- QUERY
+alter table read_int_as_decimal change c1 c1 decimal(10,0);
+select c1 from read_int_as_decimal order by c1;
+---- RESULTS
+-2147483648
+1
+2147483647
+---- TYPES
+DECIMAL
+====
+---- QUERY
+alter table read_int_as_decimal change c1 c1 decimal(8,0);
+select c1 from read_int_as_decimal order by c1;
+---- CATCH
+column 'c1' has a precision that does not match the table metadata precision. File metadata precision: 10, table metadata precision: 8.
+====
+---- QUERY
+alter table read_int_as_decimal change c1 c1 decimal(20,4);
+select c1 from read_int_as_decimal order by c1;
+---- RESULTS
+-2147483648.0000
+1.0000
+2147483647.0000
+---- TYPES
+DECIMAL
+====
+---- QUERY
+alter table read_int_as_decimal change c1 c1 decimal(12,4);
+select c1 from read_int_as_decimal order by c1;
+---- RESULTS
+1.0000
+NULL
+NULL
+====
+---- QUERY
+alter table read_int_as_decimal change c2 c2 decimal(19,0);
+select c2 from read_int_as_decimal order by c2;
+---- RESULTS
+-9223372036854775808
+10000000000
+9223372036854775807
+====
+---- QUERY
+alter table read_int_as_decimal change c2 c2 decimal(2,0);
+select c2 from read_int_as_decimal order by c2;
+---- CATCH
+column 'c2' has a precision that does not match the table metadata precision. File metadata precision: 19, table metadata precision: 2.
+====
+---- QUERY
+alter table read_int_as_decimal change c2 c2 decimal(25,5);
+select c2 from read_int_as_decimal order by c2;
+---- RESULTS
+-9223372036854775808.00000
+10000000000.00000
+9223372036854775807.00000
+====
+---- QUERY
+alter table read_int_as_decimal change c2 c2 decimal(20,5);
+select c2 from read_int_as_decimal order by c2;
+---- RESULTS
+10000000000.00000
+NULL
+NULL
+====
(impala) 02/02: IMPALA-11512: Add tests for BINARY type support in Iceberg
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 365ce0b12f778cae7f46edd5e9e99db736b465b1
Author: Daniel Vanko <dv...@cloudera.com>
AuthorDate: Mon Jul 14 14:41:55 2025 +0200
IMPALA-11512: Add tests for BINARY type support in Iceberg
This patch adds tests for the binary type in Iceberg tables.
Change-Id: I9221050a4bee57b8fbb85280478304e5b28efd21
Reviewed-on: http://gerrit.cloudera.org:8080/23167
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../queries/QueryTest/iceberg-binary-type.test | 292 +++++++++++++++++++++
tests/query_test/test_iceberg.py | 3 +
2 files changed, 295 insertions(+)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-binary-type.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-binary-type.test
new file mode 100644
index 000000000..e3348a497
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-binary-type.test
@@ -0,0 +1,292 @@
+====
+---- QUERY
+CREATE TABLE iceberg_binary_tbl (
+ id INT,
+ string_col STRING,
+ binary_col BINARY
+) STORED AS ICEBERG TBLPROPERTIES('format-version'='2')
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+INSERT INTO iceberg_binary_tbl VALUES
+ (1, 'ascii', CAST('binary1' AS BINARY)),
+ (2, 'ascii', CAST('binary2' AS BINARY)),
+ (3, 'null', NULL),
+ (4, 'empty', CAST('' AS BINARY)),
+ (5, 'valid utf8', CAST('árvíztűrőtükörfúró' AS BINARY)),
+ (6, 'valid utf8', CAST('你好hello' AS BINARY)),
+ (7, 'invalid utf8', CAST(UNHEX('00FF00FF') AS BINARY)),
+ (8, 'invalid utf8', CAST(UNHEX('FF4433221100') AS BINARY))
+---- RUNTIME_PROFILE
+NumModifiedRows: 8
+====
+---- QUERY
+SELECT id, string_col, CAST(binary_col AS STRING) FROM iceberg_binary_tbl
+WHERE string_col != "invalid utf8"
+---- TYPES
+INT, STRING, STRING
+---- RESULTS: RAW_STRING
+1,'ascii','binary1'
+2,'ascii','binary2'
+3,'null','NULL'
+4,'empty',''
+5,'valid utf8','árvíztűrőtükörfúró'
+6,'valid utf8','你好hello'
+====
+---- QUERY
+SELECT id, HEX(CAST(binary_col AS STRING)) FROM iceberg_binary_tbl
+WHERE string_col = "invalid utf8"
+---- TYPES
+INT, STRING
+---- RESULTS
+7,'00FF00FF'
+8,'FF4433221100'
+====
+---- QUERY
+SELECT id, HEX(CAST(binary_col AS STRING)) FROM iceberg_binary_tbl
+WHERE binary_col = CAST(UNHEX("FF4433221100") AS BINARY)
+---- TYPES
+INT, STRING
+---- RESULTS
+8,'FF4433221100'
+====
+---- QUERY
+SET utf8_mode=0;
+SELECT string_col, LENGTH(binary_col) FROM iceberg_binary_tbl
+---- TYPES
+STRING, INT
+---- RESULTS
+'ascii',7
+'ascii',7
+'null',NULL
+'empty',0
+'valid utf8',26
+'valid utf8',11
+'invalid utf8',4
+'invalid utf8',6
+====
+---- QUERY
+SET utf8_mode=1;
+SELECT string_col, LENGTH(binary_col) FROM iceberg_binary_tbl
+---- TYPES
+STRING, INT
+---- RESULTS
+'ascii',7
+'ascii',7
+'null',NULL
+'empty',0
+'valid utf8',26
+'valid utf8',11
+'invalid utf8',4
+'invalid utf8',6
+====
+---- QUERY
+# Test UPDATE on iceberg_binary_tbl
+UPDATE iceberg_binary_tbl
+SET binary_col = CAST('updated' AS BINARY) WHERE id = 2
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+# Verify UPDATE results
+SELECT id, string_col, CAST(binary_col AS STRING) FROM iceberg_binary_tbl
+WHERE binary_col = CAST('updated' AS BINARY)
+---- TYPES
+INT, STRING, STRING
+---- RESULTS
+2,'ascii','updated'
+====
+---- QUERY
+# Test DELETE on iceberg_binary_tbl with binary column condition
+DELETE FROM iceberg_binary_tbl WHERE binary_col = CAST('updated' AS BINARY)
+---- RUNTIME_PROFILE
+NumModifiedRows: 0
+NumDeletedRows: 1
+====
+---- QUERY
+# Verify DELETE results
+SELECT id, string_col, CAST(binary_col AS STRING) FROM iceberg_binary_tbl
+WHERE string_col != "invalid utf8"
+---- TYPES
+INT, STRING, STRING
+---- RESULTS: RAW_STRING
+1,'ascii','binary1'
+3,'null','NULL'
+4,'empty',''
+5,'valid utf8','árvíztűrőtükörfúró'
+6,'valid utf8','你好hello'
+====
+---- QUERY
+CREATE TABLE iceberg_binary_tbl_big (
+ id INT,
+ int_col INT,
+ binary_col BINARY,
+ binary_col_with_nulls BINARY,
+ timestamp_col TIMESTAMP
+)
+PARTITIONED BY SPEC (MONTH(timestamp_col))
+STORED AS ICEBERG TBLPROPERTIES('format-version'='2')
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+INSERT INTO TABLE iceberg_binary_tbl_big
+SELECT id, int_col, CAST(string_col AS BINARY),
+ CAST(CASE WHEN id % 2 = 0 THEN date_string_col ELSE NULL END AS BINARY),
+ timestamp_col
+ FROM functional.alltypes
+---- RUNTIME_PROFILE
+NumModifiedRows: 310
+NumModifiedRows: 300
+NumModifiedRows: 280
+====
+---- QUERY
+SELECT binary_col_with_nulls FROM iceberg_binary_tbl_big
+WHERE binary_col_with_nulls = CAST("01/02/09" AS BINARY)
+---- TYPES
+BINARY
+---- RESULTS
+'01/02/09'
+'01/02/09'
+'01/02/09'
+'01/02/09'
+'01/02/09'
+====
+---- QUERY
+SELECT binary_col_with_nulls FROM iceberg_binary_tbl_big
+WHERE binary_col_with_nulls > CAST("12/31/09" AS BINARY)
+---- TYPES
+BINARY
+---- RESULTS
+'12/31/10'
+'12/31/10'
+'12/31/10'
+'12/31/10'
+'12/31/10'
+====
+---- QUERY
+SELECT DISTINCT binary_col_with_nulls FROM iceberg_binary_tbl_big
+WHERE binary_col_with_nulls < CAST("01/02/09" AS BINARY)
+---- TYPES
+BINARY
+---- RESULTS
+'01/01/09'
+'01/01/10'
+====
+---- QUERY
+SET DISABLE_OUTERMOST_TOPN=0;
+SELECT binary_col_with_nulls FROM iceberg_binary_tbl_big
+WHERE binary_col = CAST("4" AS BINARY)
+ORDER BY binary_col_with_nulls LIMIT 3
+---- TYPES
+BINARY
+---- RESULTS
+'01/01/09'
+'01/01/10'
+'01/02/09'
+====
+---- QUERY
+SET DISABLE_OUTERMOST_TOPN=1;
+SELECT binary_col_with_nulls FROM iceberg_binary_tbl_big
+WHERE binary_col = CAST("4" AS BINARY)
+ORDER BY binary_col_with_nulls LIMIT 3
+---- TYPES
+BINARY
+---- RESULTS
+'01/01/09'
+'01/01/10'
+'01/02/09'
+====
+---- QUERY
+SELECT COUNT(binary_col_with_nulls), max(binary_col), min(binary_col)
+FROM iceberg_binary_tbl_big
+---- TYPES
+BIGINT, BINARY, BINARY
+---- RESULTS
+3650,'9','0'
+====
+---- QUERY
+# NDV is not yet added for BINARY, casting is needed (IMPALA-11351)
+SELECT COUNT(DISTINCT binary_col_with_nulls), NDV(CAST(binary_col_with_nulls AS STRING))
+FROM iceberg_binary_tbl_big
+---- TYPES
+BIGINT, BIGINT
+---- RESULTS
+730,736
+====
+---- QUERY
+# Test multiple count distinct
+SELECT COUNT(DISTINCT binary_col), COUNT(DISTINCT binary_col_with_nulls)
+FROM iceberg_binary_tbl_big
+WHERE id < 20
+---- TYPES
+BIGINT, BIGINT
+---- RESULTS
+10,2
+====
+---- QUERY
+SELECT bb1.id, bb2.id, bb1.binary_col_with_nulls, bb2.binary_col_with_nulls
+FROM iceberg_binary_tbl_big bb1 LEFT JOIN iceberg_binary_tbl_big bb2
+ ON bb1.binary_col_with_nulls = bb2.binary_col_with_nulls
+WHERE bb1.id < 3 AND bb2.id < 3
+---- TYPES
+INT, INT, BINARY, BINARY
+---- RESULTS
+0,2,'01/01/09','01/01/09'
+0,0,'01/01/09','01/01/09'
+2,2,'01/01/09','01/01/09'
+2,0,'01/01/09','01/01/09'
+====
+---- QUERY
+# Test UPDATE on iceberg_binary_tbl_big
+UPDATE iceberg_binary_tbl_big
+SET binary_col = CAST('updated' AS BINARY)
+WHERE binary_col_with_nulls < CAST("01/02/09" AS BINARY)
+---- RUNTIME_PROFILE
+NumModifiedRows: 5
+NumDeletedRows: 5
+====
+---- QUERY
+# Verify UPDATE results
+SELECT id, CAST(binary_col AS STRING), CAST(binary_col_with_nulls AS STRING)
+FROM iceberg_binary_tbl_big
+WHERE binary_col = CAST('updated' AS BINARY)
+---- TYPES
+INT, STRING, STRING
+---- RESULTS
+0,'updated','01/01/09'
+2,'updated','01/01/09'
+4,'updated','01/01/09'
+6,'updated','01/01/09'
+8,'updated','01/01/09'
+3650,'updated','01/01/10'
+3652,'updated','01/01/10'
+3654,'updated','01/01/10'
+3656,'updated','01/01/10'
+3658,'updated','01/01/10'
+====
+---- QUERY
+# Test DELETE on iceberg_binary_tbl_big with binary column conditions
+DELETE FROM iceberg_binary_tbl_big
+ WHERE binary_col = CAST('updated' AS BINARY)
+ AND binary_col_with_nulls = CAST("01/01/10" AS BINARY)
+---- RUNTIME_PROFILE
+NumModifiedRows: 0
+NumDeletedRows: 5
+====
+---- QUERY
+# Verify DELETE results
+SELECT id, CAST(binary_col AS STRING), CAST(binary_col_with_nulls AS STRING)
+ FROM iceberg_binary_tbl_big
+ WHERE binary_col = CAST('updated' AS BINARY)
+---- TYPES
+INT, STRING, STRING
+---- RESULTS
+0,'updated','01/01/09'
+2,'updated','01/01/09'
+4,'updated','01/01/09'
+6,'updated','01/01/09'
+8,'updated','01/01/09'
+====
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index 8fc06738e..f3f5bcf63 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -75,6 +75,9 @@ class TestIcebergTable(IcebergTestSuite):
def test_alter_iceberg_tables_default(self, vector, unique_database):
self.run_test_case('QueryTest/iceberg-alter-default', vector, use_db=unique_database)
+ def test_iceberg_binary_type(self, vector, unique_database):
+ self.run_test_case('QueryTest/iceberg-binary-type', vector, use_db=unique_database)
+
def test_external_iceberg_tables(self, vector, unique_database):
self.run_test_case('QueryTest/iceberg-external', vector, unique_database)