You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/08/13 17:28:24 UTC
(datafusion) 01/01: perf(session): cut `datafusion-session` compile time ~1.7x (#24326)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24326-519687ca22ffae61f90719a3be040a90f57da469
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 8c099dc15058cb95665e4a2312fd852d3c7042e4
Author: Daniël Heres <da...@gmail.com>
AuthorDate: Thu Aug 13 17:28:00 2026 +0000
perf(session): cut `datafusion-session` compile time ~1.7x (#24326)
## Which issue does this PR close?
Adresses: https://github.com/apache/datafusion/issues/13814
## Rationale for this change
`datafusion-session` is 1,619 lines of source and spends **18.8s** in
the
frontend during a cold `cargo build -p datafusion` (`cargo build
--timings`) —
11.6ms per line, the worst ratio in the workspace, against
0.09–0.55ms/line for
crates like `datafusion-datasource` or `datafusion-physical-plan`.
`-Zself-profile` puts ~72% of the crate's compile time in
`evaluate_obligation`,
and 98% of that in `Send`/`Sync`.
## What changes are included in this PR?
`TableProvider::{delete_from, update, merge_into}` are one-line
`not_impl_err!`
stubs that nevertheless build a coroutine capturing `Vec<Expr>` / `Expr`
—
proving that coroutine `Send` walks the whole `Expr`/`LogicalPlan` type
graph.
They are now written as the desugaring of `async fn` returning
`ready(..)`, so
no coroutine is created and there is nothing expensive to prove. The
signatures
are exactly what `#[async_trait]` generates — verified against
`-Zunpretty=expanded` output of this crate — so implementors are
unaffected.
I originally converted seven bodies, but measuring each one's marginal
contribution (reverting one at a time; `evaluate_obligation` self time,
all-converted baseline 659ms) showed only the ones taking `Expr` matter:
| method left as `async fn` | trait solving | marginal cost |
|---|---|---|
| `delete_from` | 1.29s | +631ms |
| `update` | 1.30s | +641ms |
| `merge_into` | 1.29s | +631ms |
| `truncate` | 697ms | +38ms |
| `insert_into` | 665ms | ~0 |
## Are these changes tested?
Interleaved A/B of `cargo rustc -p datafusion-session --lib`
```
base: 1.582s 1.576s 1.590s
fix: 0.933s 0.938s 0.932s
```
`evaluate_obligation` drops from 1.28s to 0.646s.
Those are standalone-build numbers. In the feature-unified build this
crate's
frontend is 18.8s rather than ~1.6s (each obligation is several times
dearer
there), so the absolute saving on a real build should be larger — I have
not
measured that directly, since isolating one crate's unit in a full build
is hard
to do without confounding it with machine drift.
## Are there any user-facing changes?
No public signature changes
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-authored-by: Claude Opus 5 <no...@anthropic.com>
---
Cargo.lock | 1 +
datafusion/session/Cargo.toml | 1 +
datafusion/session/src/table.rs | 56 ++++++++++++++++++++++++++++++-----------
3 files changed, 43 insertions(+), 15 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index b5d5674565..4cb6117da3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2609,6 +2609,7 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-plan",
+ "futures",
"parking_lot",
]
diff --git a/datafusion/session/Cargo.toml b/datafusion/session/Cargo.toml
index 2bbbdd20df..e6c277a8b8 100644
--- a/datafusion/session/Cargo.toml
+++ b/datafusion/session/Cargo.toml
@@ -37,6 +37,7 @@ datafusion-common = { workspace = true }
datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-physical-plan = { workspace = true }
+futures = { workspace = true }
parking_lot = { workspace = true }
# Note: add additional linter rules in lib.rs.
diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rs
index 69e7e731b3..9dbda40b9b 100644
--- a/datafusion/session/src/table.rs
+++ b/datafusion/session/src/table.rs
@@ -18,6 +18,7 @@
use std::any::Any;
use std::borrow::Cow;
use std::fmt::Debug;
+use std::future::ready;
use std::sync::Arc;
use crate::session::Session;
@@ -27,6 +28,7 @@ use datafusion_common::{Constraints, Statistics, not_impl_err};
use datafusion_common::{DFSchemaRef, Result, internal_err};
use datafusion_expr::Expr;
use datafusion_expr::statistics::StatisticsRequest;
+use futures::future::BoxFuture;
use datafusion_expr::dml::{InsertOp, MergeIntoClause};
use datafusion_expr::{
@@ -351,25 +353,41 @@ pub trait TableProvider: Any + Debug + Sync + Send {
///
/// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).
/// Empty `filters` deletes all rows.
- async fn delete_from(
- &self,
- _state: &dyn Session,
+ fn delete_from<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ _state: &'life1 dyn Session,
_filters: Vec<Expr>,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- not_impl_err!("DELETE not supported for {} table", self.table_type())
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ Box::pin(ready(not_impl_err!(
+ "DELETE not supported for {} table",
+ self.table_type()
+ )))
}
/// Update rows matching the filter predicates.
///
/// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).
/// Empty `filters` updates all rows.
- async fn update(
- &self,
- _state: &dyn Session,
+ fn update<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ _state: &'life1 dyn Session,
_assignments: Vec<(String, Expr)>,
_filters: Vec<Expr>,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- not_impl_err!("UPDATE not supported for {} table", self.table_type())
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ Box::pin(ready(not_impl_err!(
+ "UPDATE not supported for {} table",
+ self.table_type()
+ )))
}
/// Remove all rows from the table.
@@ -391,15 +409,23 @@ pub trait TableProvider: Any + Debug + Sync + Send {
/// The `clauses` describe the WHEN MATCHED / WHEN NOT MATCHED actions.
///
/// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).
- async fn merge_into(
- &self,
- _state: &dyn Session,
+ fn merge_into<'life0, 'life1, 'async_trait>(
+ &'life0 self,
+ _state: &'life1 dyn Session,
_source: Arc<dyn ExecutionPlan>,
_merge_schema: DFSchemaRef,
_on: Expr,
_clauses: Vec<MergeIntoClause>,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- not_impl_err!("MERGE INTO not supported for {} table", self.table_type())
+ ) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
+ where
+ 'life0: 'async_trait,
+ 'life1: 'async_trait,
+ Self: 'async_trait,
+ {
+ Box::pin(ready(not_impl_err!(
+ "MERGE INTO not supported for {} table",
+ self.table_type()
+ )))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org