You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2026/05/15 12:53:55 UTC
(impala) 01/02: IMPALA-14589: [Part 3] Add support for Iceberg V3 write-default values
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9a2e4072c615259027814c2c26cd29c1e3e0bc7f
Author: Arnab Karmakar <ar...@gmail.com>
AuthorDate: Sun Apr 26 03:19:49 2026 -0700
IMPALA-14589: [Part 3] Add support for Iceberg V3 write-default values
This patch adds write-default support for Iceberg V3 tables.
**write-default**: Applied when WRITING new rows that don't specify a
value for a column during INSERT operations. This ensures consistent
defaults across all engines writing to the table, regardless of whether
they're inserting via Spark, Flink, or Impala.
How write-default Works:
- When INSERT omits a column, InsertStmt checks for write-default in the
column's Iceberg schema metadata
- If present, the write-default value is parsed and used as the column's
value in the INSERT
- If no write-default is defined, NULL is used (existing behavior)
- Write-default values are also used for partition columns when they are
not explicitly specified in column permutation
Type Support:
**Supported Types** (with write-default):
- Numeric: BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL
- Text: STRING, VARCHAR, CHAR
- Temporal: DATE
**Unsupported Types** (rejected with AnalysisException):
- TIMESTAMP
- BINARY/FIXED
Testing:
- Positive tests: Extended iceberg-v3-default-values.test with
write-default coverage:
* Write-default for all supported primitive types (BOOLEAN through DATE)
* INSERT omitting columns with write-default values
* INSERT with explicit values overrides write-default
* INSERT with explicit NULL overrides write-default
- Negative tests:
* Removed negative test that blocked write-default (no longer needed)
* Test TIMESTAMP and BINARY write-default rejection
Change-Id: I3bbccaad07f8c2f1fef59950de850f43d4044c08
Reviewed-on: http://gerrit.cloudera.org:8080/24232
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Zoltan Borok-Nagy <bo...@cloudera.com>
---
.../org/apache/impala/analysis/InsertStmt.java | 62 +++++++--
.../iceberg_v3_all_types/metadata/v4.metadata.json | 30 +++--
.../test_default_part/metadata/v6.metadata.json | 3 +-
.../QueryTest/iceberg-v3-default-values.test | 142 +++++++++++++++++++++
.../queries/QueryTest/iceberg-v3-negative.test | 28 +++-
5 files changed, 235 insertions(+), 30 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
index 0233c7033..fee4d51ba 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
@@ -999,25 +999,17 @@ public class InsertStmt extends DmlStatementBase {
} else {
// Unmentioned non-clustering columns get NULL literals with the appropriate
// target type because Parquet cannot handle NULL_TYPE (IMPALA-617).
- // TODO: Add write-default support for Iceberg tables
- if (isIcebergTarget()) {
- IcebergColumn iceCol = (IcebergColumn) tblColumn;
- if (iceCol.getWriteDefault() != null) {
- throw new AnalysisException("Write-default values are not yet " +
- "supported. Column '" + iceCol.getName() + "' has a write-default " +
- "value defined.");
- }
- }
- NullLiteral nullExpr = NullLiteral.create(tblColumn.getType());
- resultExprs_.add(nullExpr);
+ // For Iceberg tables, use write-default if available.
+ Expr defaultExpr = getDefaultExpr(tblColumn);
+ resultExprs_.add(defaultExpr);
// In the case of INSERT INTO iceberg_tbl (col_a, col_b, ...), if the
// partition columns are not in the columnPermutation_, we should fill it
- // with NullLiteral to partitionKeyExprs_ (IMPALA-11408).
+ // with default value or NullLiteral to partitionKeyExprs_ (IMPALA-11408).
if (isIcebergTarget() && !CollectionUtils.isEmpty(columnPermutation_)
&& icebergPartSpec != null) {
IcebergColumn targetColumn = (IcebergColumn) tblColumn;
if (IcebergUtil.isPartitionColumn(targetColumn, icebergPartSpec)) {
- partitionKeyExprs_.add(nullExpr);
+ partitionKeyExprs_.add(defaultExpr);
partitionColPos_.add(targetColumn.getPosition());
}
}
@@ -1283,4 +1275,48 @@ public class InsertStmt extends DmlStatementBase {
if (queryStmt_ != null) hasChange |= queryStmt_.resolveTableMask(analyzer);
return hasChange;
}
+
+ /**
+ * Returns the default expression for an unmentioned column in an INSERT statement.
+ * For Iceberg tables, attempts to use the write-default value from metadata.
+ * Falls back to NULL if no write-default is available or parsing fails.
+ */
+ private Expr getDefaultExpr(Column tblColumn) throws AnalysisException {
+ Expr defaultExpr = null;
+ if (isIcebergTarget() && tblColumn instanceof IcebergColumn) {
+ IcebergColumn iceCol = (IcebergColumn) tblColumn;
+ String writeDefault = iceCol.getWriteDefault();
+
+ if (writeDefault != null && !writeDefault.isEmpty()) {
+ Type colType = tblColumn.getType();
+ // Reject TIMESTAMP - LiteralExpr doesn't support it
+ if (colType.isTimestamp()) {
+ throw new AnalysisException(
+ "Iceberg write-default values for TIMESTAMP type are not supported. " +
+ "Please specify value for column '" + tblColumn.getName() +
+ "' explicitly.");
+ }
+ // Reject BINARY
+ if (colType.isBinary()) {
+ throw new AnalysisException(
+ "Iceberg write-default values for BINARY/FIXED types are not supported. " +
+ "Please specify value for column '" + tblColumn.getName() +
+ "' explicitly.");
+ }
+
+ try {
+ defaultExpr = LiteralExpr.createFromUnescapedStr(writeDefault, colType);
+ } catch (AnalysisException e) {
+ throw new AnalysisException("Failed to parse write-default value '" +
+ writeDefault + "' for column '" + tblColumn.getName() + "': " +
+ e.getMessage() + ". Please specify value explicitly.");
+ }
+ }
+ }
+
+ if (defaultExpr == null) {
+ defaultExpr = NullLiteral.create(tblColumn.getType());
+ }
+ return defaultExpr;
+ }
}
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_all_types/metadata/v4.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_all_types/metadata/v4.metadata.json
index 0a0faabff..28ea001e4 100644
--- a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_all_types/metadata/v4.metadata.json
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_all_types/metadata/v4.metadata.json
@@ -34,70 +34,80 @@
"name": "col_boolean",
"required": false,
"type": "boolean",
- "initial-default": false
+ "initial-default": false,
+ "write-default": true
},
{
"id": 3,
"name": "col_int",
"required": false,
"type": "int",
- "initial-default": -1
+ "initial-default": -1,
+ "write-default": -999
},
{
"id": 4,
"name": "col_bigint",
"required": false,
"type": "long",
- "initial-default": -1
+ "initial-default": -1,
+ "write-default": -999999999
},
{
"id": 5,
"name": "col_float",
"required": false,
"type": "float",
- "initial-default": 1.5
+ "initial-default": 1.5,
+ "write-default": 9.99
},
{
"id": 6,
"name": "col_double",
"required": false,
"type": "double",
- "initial-default": 2.5
+ "initial-default": 2.5,
+ "write-default": 8.88
},
{
"id": 7,
"name": "col_decimal",
"required": false,
"type": "decimal(10, 2)",
- "initial-default": "123.45"
+ "initial-default": "123.45",
+ "write-default": "999.99"
},
{
"id": 8,
"name": "col_string",
"required": false,
"type": "string",
- "initial-default": "initial_default"
+ "initial-default": "initial_default",
+ "write-default": "write_default"
},
{
"id": 9,
"name": "col_date",
"required": false,
"type": "date",
- "initial-default": "2024-01-01"
+ "initial-default": "2024-01-01",
+ "write-default": "2024-12-31"
},
{
"id": 10,
"name": "col_timestamp",
"required": false,
"type": "timestamp",
- "initial-default": "1969-12-31T23:59:59.123456"
+ "initial-default": "1969-12-31T23:59:59.123456",
+ "write-default": "2024-12-31T23:59:59.999999"
},
{
"id": 11,
"name": "col_binary",
"required": false,
"type": "binary",
- "initial-default": "0102030405060708"
+ "initial-default": "0102030405060708",
+ "write-default": "a1b2c3d4e5f60718"
},
{
"id": 12,
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v6.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v6.metadata.json
index 572d1fdfd..769863cd9 100644
--- a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v6.metadata.json
+++ b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v6.metadata.json
@@ -34,7 +34,8 @@
"name": "j",
"required": false,
"type": "int",
- "initial-default": -1
+ "initial-default": -1,
+ "write-default": -999
}
]
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-default-values.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-default-values.test
index ba58f3299..ff1fb7c2f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-default-values.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-default-values.test
@@ -294,3 +294,145 @@ FROM iceberg_v3_all_types ORDER BY id;
---- TYPES
INT,BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,STRING,DATE,TIMESTAMP,TIMESTAMP,TIMESTAMP
====
+---- QUERY
+# Test write-default: INSERT omitting column j should use write-default value -999
+# Column j has initial-default=-1 and write-default=-999 in metadata
+INSERT INTO iceberg_v3_default_value (i) VALUES (100);
+SELECT i, j FROM iceberg_v3_default_value WHERE i = 100;
+---- RESULTS
+100,-999
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Test write-default with explicit value
+# Explicitly providing a value should override write-default
+INSERT INTO iceberg_v3_default_value VALUES (101, 50);
+SELECT i, j FROM iceberg_v3_default_value WHERE i = 101;
+---- RESULTS
+101,50
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Test write-default with explicit NULL
+# Explicitly specifying NULL should write NULL, not the write-default
+INSERT INTO iceberg_v3_default_value VALUES (102, NULL);
+SELECT i, j FROM iceberg_v3_default_value WHERE i = 102;
+---- RESULTS
+102,NULL
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Verify all inserted rows with defaults
+SELECT i, j FROM iceberg_v3_default_value WHERE i >= 100 ORDER BY i;
+---- RESULTS
+100,-999
+101,50
+102,NULL
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Test write-default on test_default_part: INSERT omitting partitioned column j should use write-default value -999
+# Column j has initial-default=-1 and write-default=-999 in metadata
+# Since j is a partition column, the write-default also determines the partition
+INSERT INTO test_default_part (i) VALUES (100);
+SELECT i, j FROM test_default_part WHERE i = 100;
+---- RESULTS
+100,-999
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Test write-default on test_default_part with explicit partition value
+# Explicitly providing partition value should override write-default
+INSERT INTO test_default_part VALUES (101, 50);
+SELECT i, j FROM test_default_part WHERE i = 101;
+---- RESULTS
+101,50
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Test write-default on test_default_part with explicit NULL partition
+# Explicitly specifying NULL for partition column should write NULL, not the write-default
+INSERT INTO test_default_part VALUES (102, NULL);
+SELECT i, j FROM test_default_part WHERE i = 102;
+---- RESULTS
+102,NULL
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Verify all inserted rows with write-default on partitioned table
+SELECT i, j FROM test_default_part WHERE i >= 100 ORDER BY i;
+---- RESULTS
+100,-999
+101,50
+102,NULL
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Supported write-default types: BOOLEAN, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, DATE
+# NOT supported: TIMESTAMP, BINARY
+#
+# write-default values:
+# - col_boolean: true, col_int: -999, col_bigint: -999999999
+# - col_float: 9.99, col_double: 8.88, col_decimal: 999.99
+# - col_string: 'write_default', col_date: '2024-12-31'
+INSERT INTO iceberg_v3_all_types (id, col_timestamp, col_binary)
+VALUES (100, CAST('2024-01-01 00:00:00' AS TIMESTAMP), CAST(unhex('CAFEBABE') AS BINARY));
+SELECT id, col_boolean, col_int, col_bigint, col_float, col_double, col_decimal, col_string, col_date
+FROM iceberg_v3_all_types WHERE id = 100;
+---- RESULTS
+100,true,-999,-999999999,9.989999771118164,8.88,999.99,'write_default',2024-12-31
+---- TYPES
+INT,BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,STRING,DATE
+====
+---- QUERY
+# Test write-default with explicit values - explicit values should override write-default
+# Column list omits col_timestamp_epoch / col_timestamp_iso (initial-default only); those use NULL.
+INSERT INTO iceberg_v3_all_types (
+ id, col_boolean, col_int, col_bigint, col_float, col_double, col_decimal, col_string, col_date,
+ col_timestamp, col_binary)
+VALUES
+ (101, true, 123, 456789, 1.23, 4.56, 78.90, 'custom',
+ CAST('2025-06-15' AS DATE), CAST('2025-06-15 12:34:56.123456' AS TIMESTAMP), CAST(unhex('ABCD1234') AS BINARY));
+SELECT id, col_boolean, col_int, col_bigint, col_float, col_double, col_decimal, col_string, col_date
+FROM iceberg_v3_all_types WHERE id = 101;
+---- RESULTS
+101,true,123,456789,1.2300000190734863,4.56,78.90,'custom',2025-06-15
+---- TYPES
+INT,BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,STRING,DATE
+====
+---- QUERY
+# Test write-default with explicit NULL - NULL should override write-default
+INSERT INTO iceberg_v3_all_types (
+ id, col_boolean, col_int, col_bigint, col_float, col_double, col_decimal, col_string, col_date,
+ col_timestamp, col_binary)
+VALUES
+ (102, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+ CAST('2024-01-01 00:00:00' AS TIMESTAMP), CAST(unhex('DEADBEEF') AS BINARY));
+SELECT id, col_boolean, col_int, col_bigint, col_float, col_double, col_decimal, col_string, col_date
+FROM iceberg_v3_all_types WHERE id = 102;
+---- RESULTS
+102,NULL,NULL,NULL,NULL,NULL,NULL,'NULL',NULL
+---- TYPES
+INT,BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,STRING,DATE
+====
+---- QUERY
+# Verify all rows with write-default insertions
+SELECT id, col_boolean, col_int, col_bigint, col_float, col_double, col_decimal, col_string, col_date
+FROM iceberg_v3_all_types
+WHERE id >= 100
+ORDER BY id;
+---- RESULTS
+100,true,-999,-999999999,9.989999771118164,8.88,999.99,'write_default',2024-12-31
+101,true,123,456789,1.2300000190734863,4.56,78.90,'custom',2025-06-15
+102,NULL,NULL,NULL,NULL,NULL,NULL,'NULL',NULL
+---- TYPES
+INT,BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,STRING,DATE
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
index 7a052acc5..cc49d9b4f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
@@ -36,12 +36,6 @@ SELECT j.f1 FROM test_complex_default;
Default values are not supported for nested fields
====
---- QUERY
-# Negative test: INSERT omitting a column that has a write-default should fail until supported
-INSERT INTO iceberg_v3_default_value (i) VALUES (100);
----- CATCH
-Write-default values are not yet supported
-====
----- QUERY
# Default values on binary/fixed types are rejected when materialized.
# Table iceberg_v3_all_types: col_binary has a binary default in metadata.
# Selecting col_binary should fail during planning when the column is materialized.
@@ -67,3 +61,25 @@ SELECT * FROM test_complex_default;
---- CATCH
Default values are not supported for TIMESTAMPTZ types
====
+---- QUERY
+# Test TIMESTAMP write-default rejection
+# Table iceberg_v3_all_types has write-default for col_timestamp in metadata.
+# INSERT omitting col_timestamp should fail during analysis with clear error message.
+INSERT INTO iceberg_v3_all_types (id, col_boolean, col_int, col_bigint, col_float,
+ col_double, col_decimal, col_string, col_date, col_binary)
+VALUES (200, true, 1, 2, 3.14, 2.71, 100.50, 'test', CAST('2024-01-01' AS DATE),
+ CAST(unhex('ABCD') AS BINARY));
+---- CATCH
+Iceberg write-default values for TIMESTAMP type are not supported
+====
+---- QUERY
+# Test BINARY write-default rejection
+# Table iceberg_v3_all_types has write-default for col_binary in metadata.
+# INSERT omitting col_binary should fail during analysis with clear error message.
+INSERT INTO iceberg_v3_all_types (id, col_boolean, col_int, col_bigint, col_float,
+ col_double, col_decimal, col_string, col_date, col_timestamp)
+VALUES (201, false, 10, 20, 1.5, 2.5, 50.25, 'binary_test', CAST('2024-06-15' AS DATE),
+ CAST('2024-06-15 10:30:00' AS TIMESTAMP));
+---- CATCH
+Iceberg write-default values for BINARY/FIXED types are not supported
+====