You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/14 09:49:18 UTC
(datafusion) branch main updated: fix: preserve target extension metadata in INSERT (#24971)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 17c6d74deb fix: preserve target extension metadata in INSERT (#24971)
17c6d74deb is described below
commit 17c6d74deb11824cabf61010e8ce26a1c369b471
Author: Artem Osipov <59...@users.noreply.github.com>
AuthorDate: Mon Sep 14 09:12:47 2026 +0000
fix: preserve target extension metadata in INSERT (#24971)
## Which issue does this PR close?
- Closes #24970.
## Rationale for this change
`INSERT` currently aligns source expressions with target columns by
storage `DataType` only. If both fields use the same storage type but
the target is an Arrow extension type, the cast is skipped and the DML
projection loses the target extension metadata. A table sink can then
receive a schema that does not describe the target logical type.
## What changes are included in this PR?
After storage-type coercion, compare the resulting field metadata with
the target table field. When they differ, use the existing field-aware
`Cast` representation to enforce the target metadata. Existing casts are
replaced rather than nested.
The regression uses an unannotated `FixedSizeBinary(16)` source and an
`arrow.uuid` target so the behavior is independent of any particular
downstream consumer. Existing parameterized UUID INSERT snapshots are
updated to show the target field-aware cast.
## What is the testing strategy for this PR?
- Added `plan_insert_preserves_target_extension_metadata`.
- Ran all 575 `datafusion-sql` integration tests.
- Ran `cargo clippy -p datafusion-sql --all-targets --all-features -- -D
warnings`.
- Ran `cargo fmt --all -- --check`.
## Are there any user-facing changes?
Yes. SQL `INSERT` plans now preserve target Arrow field metadata,
including extension metadata. There are no public API changes.
---
datafusion/sql/src/statement.rs | 21 +++++++++++-
datafusion/sql/tests/cases/params.rs | 8 ++---
datafusion/sql/tests/common/mod.rs | 28 ++++++++++++++++
datafusion/sql/tests/sql_integration.rs | 58 +++++++++++++++++++++++++++++++++
4 files changed, 110 insertions(+), 5 deletions(-)
diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
index dde9420a68..c61f09f6f5 100644
--- a/datafusion/sql/src/statement.rs
+++ b/datafusion/sql/src/statement.rs
@@ -48,7 +48,7 @@ use datafusion_expr::logical_plan::DdlStatement;
use datafusion_expr::logical_plan::builder::project;
use datafusion_expr::utils::expr_to_columns;
use datafusion_expr::{
- Analyze, CreateCatalog, CreateCatalogSchema,
+ Analyze, Cast, CreateCatalog, CreateCatalogSchema,
CreateExternalTable as PlanCreateExternalTable, CreateFunction, CreateFunctionBody,
CreateIndex as PlanCreateIndex, CreateMemoryTable, CreateView, Deallocate,
DescribeTable, DmlStatement, DropCatalogSchema, DropFunction, DropTable, DropView,
@@ -2955,6 +2955,25 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
})
.cast_to(target_field.data_type(), &DFSchema::empty())?,
};
+ let (_, expr_field) = expr.to_field(source.schema())?;
+ // A storage-type cast alone does not apply extension metadata from the
+ // table schema when the source and target storage types are identical.
+ let expr = if target_field.extension_type_name().is_none()
+ || expr_field.metadata() == target_field.metadata()
+ {
+ expr
+ } else {
+ match expr {
+ Expr::Cast(cast) => Expr::Cast(Cast::new_from_field(
+ cast.expr,
+ Arc::clone(target_field),
+ )),
+ expr => Expr::Cast(Cast::new_from_field(
+ Box::new(expr),
+ Arc::clone(target_field),
+ )),
+ }
+ };
Ok(expr.alias(target_field.name()))
})
.collect::<Result<Vec<Expr>>>()?;
diff --git a/datafusion/sql/tests/cases/params.rs b/datafusion/sql/tests/cases/params.rs
index 68c560ead6..8b025d3945 100644
--- a/datafusion/sql/tests/cases/params.rs
+++ b/datafusion/sql/tests/cases/params.rs
@@ -837,11 +837,11 @@ fn test_insert_infer_with_metadata() {
@r#"
** Initial Plan:
Dml: op=[Insert Into] table=[person_with_uuid_extension]
- Projection: column1 AS id, column2 AS first_name, column3 AS last_name
+ Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
Values: ($1, $2, $3)
** Final Plan:
Dml: op=[Insert Into] table=[person_with_uuid_extension]
- Projection: column1 AS id, column2 AS first_name, column3 AS last_name
+ Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
Values: (FixedSizeBinary(16, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16") FieldMetadata { inner: {"ARROW:extension:name": "arrow.uuid"} } AS $1, Utf8("Alan") AS $2, Utf8("Turing") AS $3)
"#
);
@@ -859,11 +859,11 @@ fn test_insert_infer_with_metadata() {
** Initial Plan:
Prepare: "my_plan" [FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>, Utf8, Utf8]
Dml: op=[Insert Into] table=[person_with_uuid_extension]
- Projection: column1 AS id, column2 AS first_name, column3 AS last_name
+ Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
Values: ($1, $2, $3)
** Final Plan:
Dml: op=[Insert Into] table=[person_with_uuid_extension]
- Projection: column1 AS id, column2 AS first_name, column3 AS last_name
+ Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
Values: (FixedSizeBinary(16, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16") FieldMetadata { inner: {"ARROW:extension:name": "arrow.uuid"} } AS $1, Utf8("Alan") AS $2, Utf8("Turing") AS $3)
"#
);
diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rs
index e7c819bbf6..66447afd73 100644
--- a/datafusion/sql/tests/common/mod.rs
+++ b/datafusion/sql/tests/common/mod.rs
@@ -174,6 +174,20 @@ impl ContextProvider for MockContextProvider {
Field::new("first_name", DataType::Utf8, false),
Field::new("last_name", DataType::Utf8, false),
])),
+ "person_with_binary_id" => Ok(Schema::new(vec![
+ Field::new("id", DataType::FixedSizeBinary(16), false),
+ Field::new("first_name", DataType::Utf8, false),
+ Field::new("last_name", DataType::Utf8, false),
+ ])),
+ "string_with_extension" => Ok(Schema::new(vec![
+ Field::new("value", DataType::Utf8, false).with_metadata(
+ [(
+ "ARROW:extension:name".to_string(),
+ "example.string".to_string(),
+ )]
+ .into(),
+ ),
+ ])),
"orders" => Ok(Schema::new(vec![
Field::new("order_id", DataType::UInt32, false),
Field::new("o_orderkey", DataType::UInt32, false),
@@ -214,6 +228,20 @@ impl ContextProvider for MockContextProvider {
false,
),
])),
+ "array_with_field_metadata" => Ok(Schema::new(vec![
+ Field::new(
+ "left",
+ DataType::List(Arc::new(
+ Field::new_list_field(DataType::Int64, true).with_metadata(
+ [("PARQUET:field_id".to_string(), "2".to_string())].into(),
+ ),
+ )),
+ false,
+ )
+ .with_metadata(
+ [("PARQUET:field_id".to_string(), "1".to_string())].into(),
+ ),
+ ])),
"lineitem" => Ok(Schema::new(vec![
Field::new("l_orderkey", DataType::UInt32, false),
Field::new("l_item_id", DataType::UInt32, false),
diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs
index 2bd06ee4ba..6e1b91ec10 100644
--- a/datafusion/sql/tests/sql_integration.rs
+++ b/datafusion/sql/tests/sql_integration.rs
@@ -719,6 +719,64 @@ fn plan_insert_no_target_columns() {
);
}
+#[test]
+fn plan_insert_preserves_target_extension_metadata() {
+ let sql = "INSERT INTO person_with_uuid_extension \
+ SELECT id, first_name, last_name FROM person_with_binary_id";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+ Dml: op=[Insert Into] table=[person_with_uuid_extension]
+ Projection: CAST(person_with_binary_id.id AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, person_with_binary_id.first_name AS first_name, person_with_binary_id.last_name AS last_name
+ Projection: person_with_binary_id.id, person_with_binary_id.first_name, person_with_binary_id.last_name
+ TableScan: person_with_binary_id
+ "#
+ );
+}
+
+#[test]
+fn plan_insert_skips_cast_for_matching_extension_metadata() {
+ let sql = "INSERT INTO person_with_uuid_extension \
+ SELECT id, first_name, last_name FROM person_with_uuid_extension";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+ Dml: op=[Insert Into] table=[person_with_uuid_extension]
+ Projection: person_with_uuid_extension.id AS id, person_with_uuid_extension.first_name AS first_name, person_with_uuid_extension.last_name AS last_name
+ Projection: person_with_uuid_extension.id, person_with_uuid_extension.first_name, person_with_uuid_extension.last_name
+ TableScan: person_with_uuid_extension
+ "#
+ );
+}
+
+#[test]
+fn plan_insert_preserves_target_extension_metadata_on_type_cast() {
+ let sql = "INSERT INTO string_with_extension SELECT id FROM test_decimal";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+ Dml: op=[Insert Into] table=[string_with_extension]
+ Projection: CAST(test_decimal.id AS Utf8<{"ARROW:extension:name": "example.string"}>) AS value
+ Projection: test_decimal.id
+ TableScan: test_decimal
+ "#
+ );
+}
+
+#[test]
+fn plan_insert_does_not_promise_ordinary_target_field_metadata() {
+ let sql = "INSERT INTO array_with_field_metadata SELECT left FROM array";
+ let plan = logical_plan(sql).unwrap();
+ let LogicalPlan::Dml(dml) = &plan else {
+ panic!("expected DML plan");
+ };
+
+ assert!(dml.input.schema().field(0).metadata().is_empty());
+}
+
#[rstest]
#[case::duplicate_columns(
"INSERT INTO test_decimal (id, price, price) VALUES (1, 2, 3), (4, 5, 6)",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org