You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/14 20:30:12 UTC
(datafusion) branch gh-readonly-queue/main/pr-25944-678751ff7e7fdd4706e4c9f8d4eb6bea284ef6f2 created (now 7d4893cac3)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-25944-678751ff7e7fdd4706e4c9f8d4eb6bea284ef6f2
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 7d4893cac3 fix(proto): preserve EmptyRelation schema across logical plan round trip (#25944)
This branch includes the following new commits:
new 7d4893cac3 fix(proto): preserve EmptyRelation schema across logical plan round trip (#25944)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix(proto): preserve EmptyRelation schema across logical plan round trip (#25944)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25944-678751ff7e7fdd4706e4c9f8d4eb6bea284ef6f2
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 7d4893cac33e6e81eb16bf0f58981fc8633c5085
Author: Luca Bigon <lu...@bauplanlabs.com>
AuthorDate: Wed Oct 14 20:29:48 2026 +0000
fix(proto): preserve EmptyRelation schema across logical plan round trip (#25944)
## Which issue does this PR close?
- Closes #25937.
## Rationale for this change
The optimizer reduces queries like `SELECT * FROM t LIMIT 0` or `...
WHERE false` to an `EmptyRelation` that keeps the schema of its input,
but `EmptyRelationNode` only stored `produce_one_row`. After a protobuf
round trip the plan came back without columns, and plans where another
node refers to those columns, like `SELECT max(id), count(*) FROM t
WHERE false`, failed to decode.
## What changes are included in this PR?
- Add a `schema` field to `EmptyRelationNode`, encoded and decoded like
`DropViewNode`.
- Fall back to the previous empty schema for older payloads without it.
- Regenerate the prost and pbjson models.
- Add `roundtrip_empty_relation_schema` and update the expected JSON in
`plan_to_json`.
- Document the new generated field in the 56.0.0 upgrade guide.
## Are these changes tested?
Yes:
- `cargo test -p datafusion-proto --features json,avro --test
proto_integration`
- `cargo fmt --all -- --check`
- `cargo clippy -p datafusion-proto -p datafusion-proto-models
--all-targets --all-features -- -D warnings`
- `./dev/rust_lint.sh`
`json_to_plan` still decodes `{"emptyRelation":{}}`, which covers
payloads from older versions.
## Are there any user-facing changes?
Logical plans reduced to an `EmptyRelation` keep their schema across
protobuf round trips. The wire format remains backward compatible; the
generated `EmptyRelationNode` struct gains a `schema` field, documented
in the upgrade guide.
---
datafusion/proto-models/proto/datafusion.proto | 1 +
datafusion/proto-models/src/generated/pbjson.rs | 17 +++++++++++++
datafusion/proto-models/src/generated/prost.rs | 4 +++-
datafusion/proto/src/logical_plan/mod.rs | 13 ++++++++--
.../proto/tests/cases/roundtrip_logical_plan.rs | 28 ++++++++++++++++++++++
datafusion/proto/tests/cases/serialize.rs | 2 +-
6 files changed, 61 insertions(+), 4 deletions(-)
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index ba5dda6e8a..d437145618 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -181,6 +181,7 @@ message HashRepartition {
message EmptyRelationNode {
bool produce_one_row = 1;
+ datafusion_common.DfSchema schema = 2;
}
message CreateExternalTableNode {
diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs
index 05e066e1e2..f578c8a812 100644
--- a/datafusion/proto-models/src/generated/pbjson.rs
+++ b/datafusion/proto-models/src/generated/pbjson.rs
@@ -6695,10 +6695,16 @@ impl serde::Serialize for EmptyRelationNode {
if self.produce_one_row {
len += 1;
}
+ if self.schema.is_some() {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.EmptyRelationNode", len)?;
if self.produce_one_row {
struct_ser.serialize_field("produceOneRow", &self.produce_one_row)?;
}
+ if let Some(v) = self.schema.as_ref() {
+ struct_ser.serialize_field("schema", v)?;
+ }
struct_ser.end()
}
}
@@ -6711,11 +6717,13 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
const FIELDS: &[&str] = &[
"produce_one_row",
"produceOneRow",
+ "schema",
];
#[allow(clippy::enum_variant_names)]
enum GeneratedField {
ProduceOneRow,
+ Schema,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -6738,6 +6746,7 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
{
match value {
"produceOneRow" | "produce_one_row" => Ok(GeneratedField::ProduceOneRow),
+ "schema" => Ok(GeneratedField::Schema),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -6758,6 +6767,7 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
V: serde::de::MapAccess<'de>,
{
let mut produce_one_row__ = None;
+ let mut schema__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::ProduceOneRow => {
@@ -6766,10 +6776,17 @@ impl<'de> serde::Deserialize<'de> for EmptyRelationNode {
}
produce_one_row__ = Some(map_.next_value()?);
}
+ GeneratedField::Schema => {
+ if schema__.is_some() {
+ return Err(serde::de::Error::duplicate_field("schema"));
+ }
+ schema__ = map_.next_value()?;
+ }
}
}
Ok(EmptyRelationNode {
produce_one_row: produce_one_row__.unwrap_or_default(),
+ schema: schema__,
})
}
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index 4bfa7c6fe8..3522207cdf 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -264,10 +264,12 @@ pub struct HashRepartition {
#[prost(uint64, tag = "2")]
pub partition_count: u64,
}
-#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+#[derive(Clone, PartialEq, ::prost::Message)]
pub struct EmptyRelationNode {
#[prost(bool, tag = "1")]
pub produce_one_row: bool,
+ #[prost(message, optional, tag = "2")]
+ pub schema: ::core::option::Option<super::datafusion_common::DfSchema>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CreateExternalTableNode {
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index 9ebc21c876..5302b477f7 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -805,7 +805,14 @@ impl AsLogicalPlan for LogicalPlanNode {
.build()
}
LogicalPlanType::EmptyRelation(empty_relation) => {
- LogicalPlanBuilder::empty(empty_relation.produce_one_row).build()
+ if let Some(schema) = &empty_relation.schema {
+ Ok(LogicalPlan::EmptyRelation(EmptyRelation {
+ produce_one_row: empty_relation.produce_one_row,
+ schema: Arc::new(schema.try_into()?),
+ }))
+ } else {
+ LogicalPlanBuilder::empty(empty_relation.produce_one_row).build()
+ }
}
LogicalPlanType::CreateExternalTable(create_extern_table) => {
let pb_schema = (create_extern_table.schema.clone()).ok_or_else(|| {
@@ -2012,11 +2019,13 @@ impl AsLogicalPlan for LogicalPlanNode {
})
}
LogicalPlan::EmptyRelation(EmptyRelation {
- produce_one_row, ..
+ produce_one_row,
+ schema,
}) => Ok(LogicalPlanNode {
logical_plan_type: Some(LogicalPlanType::EmptyRelation(
protobuf::EmptyRelationNode {
produce_one_row: *produce_one_row,
+ schema: Some(schema.try_into()?),
},
)),
}),
diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
index 4a3e666932..d27aaf27a0 100644
--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
@@ -3853,6 +3853,34 @@ async fn roundtrip_empty_table_scan_with_projection() -> Result<()> {
Ok(())
}
+// Regression test for https://github.com/apache/datafusion/issues/25937:
+// the optimizer reduces `LIMIT 0` and `WHERE false` to an `EmptyRelation`, which
+// must keep its schema across a to_proto -> from_proto round trip: on its own,
+// and under a node that still references its columns.
+#[tokio::test]
+async fn roundtrip_empty_relation_schema() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("name", DataType::Utf8, true),
+ ]));
+ let table = Arc::new(EmptyTable::new(Arc::clone(&schema)));
+
+ let ctx = SessionContext::new();
+ ctx.register_table("empty", table)?;
+
+ for query in [
+ "SELECT * FROM empty LIMIT 0",
+ "SELECT max(id), count(*) FROM empty WHERE false",
+ ] {
+ let plan = ctx.sql(query).await?.into_optimized_plan()?;
+ let bytes = logical_plan_to_bytes(&plan)?;
+ let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
+
+ assert_eq!(plan, restored, "{query}");
+ }
+ Ok(())
+}
+
// Regression test for https://github.com/apache/datafusion/issues/22065:
// the decoder must preserve `null_aware = true` (NOT IN semantics), and the
// `NOT IN` value key count of a multi-column `(a, b) NOT IN` (which also holds
diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rs
index c71ca27b3e..72d96c0900 100644
--- a/datafusion/proto/tests/cases/serialize.rs
+++ b/datafusion/proto/tests/cases/serialize.rs
@@ -109,7 +109,7 @@ fn plan_to_json() {
schema: Arc::new(DFSchema::empty()),
});
let actual = logical_plan_to_json(&plan).unwrap();
- let expected = r#"{"emptyRelation":{}}"#.to_string();
+ let expected = r#"{"emptyRelation":{"schema":{}}}"#.to_string();
assert_eq!(actual, expected);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org