You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/08/14 06:50:43 UTC
(datafusion) branch main updated: Reduce datafusion-spark dependency footprint (#24351)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 84a8e6db91 Reduce datafusion-spark dependency footprint (#24351)
84a8e6db91 is described below
commit 84a8e6db91d19c203536faa0bbf1466bd5ea692f
Author: dentiny <de...@gmail.com>
AuthorDate: Fri Aug 14 06:42:16 2026 +0000
Reduce datafusion-spark dependency footprint (#24351)
## Which issue does this PR close?
N/A
## Rationale for this change
`datafusion-spark` pulls unnecessary crates into its production
dependency graph, increasing compilation time and artifact size for
downstream users.
## What changes are included in this PR?
- Replace compatibility re-exports with the narrower crates that define
`TableFunction` and `FunctionRegistry`.
- Remove the unused `crypto_expressions` activation; Spark provides its
own SHA-1, SHA-2, and CRC32 implementations.
- Fix the `quote` import so builds without the optional `core` feature
continue to compile.
These changes reduce the production dependency graph from 279 to 257
packages. Cargo Machete does not report these dependencies because they
are referenced in source through re-exports or activated through Cargo
features.
## Are these changes tested?
Yes
## Are there any user-facing changes?
No. Public APIs and default behavior remain unchanged.
---
Cargo.lock | 3 +--
datafusion/spark/Cargo.toml | 5 ++---
datafusion/spark/src/function/string/quote.rs | 6 ++++--
datafusion/spark/src/function/table/mod.rs | 2 +-
datafusion/spark/src/lib.rs | 6 +++---
5 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 4cb6117da3..5365be12a1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2623,14 +2623,13 @@ dependencies = [
"crc32fast",
"criterion",
"datafusion",
- "datafusion-catalog",
"datafusion-common",
- "datafusion-execution",
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-aggregate-common",
"datafusion-functions-nested",
+ "datafusion-session",
"log",
"num-traits",
"percent-encoding",
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index 93987b553f..01fd9ec9b3 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -49,14 +49,13 @@ chrono = { workspace = true }
crc32fast = "1.4"
# Optional dependency for SessionStateBuilderSpark extension trait
datafusion = { workspace = true, optional = true, default-features = false }
-datafusion-catalog = { workspace = true }
datafusion-common = { workspace = true }
-datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
-datafusion-functions = { workspace = true, features = ["crypto_expressions"] }
+datafusion-functions = { workspace = true }
datafusion-functions-aggregate = { workspace = true }
datafusion-functions-aggregate-common = { workspace = true }
datafusion-functions-nested = { workspace = true }
+datafusion-session = { workspace = true }
log = { workspace = true }
num-traits = { workspace = true }
percent-encoding = "2.3.2"
diff --git a/datafusion/spark/src/function/string/quote.rs b/datafusion/spark/src/function/string/quote.rs
index 39ad8bf841..55d827fb7a 100644
--- a/datafusion/spark/src/function/string/quote.rs
+++ b/datafusion/spark/src/function/string/quote.rs
@@ -17,12 +17,14 @@
use arrow::array::{ArrayRef, OffsetSizeTrait, StringArray};
use arrow::datatypes::DataType;
-use datafusion::logical_expr::{Coercion, ColumnarValue, Signature, TypeSignatureClass};
use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
use datafusion_common::types::{NativeType, logical_string};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, exec_err};
-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Volatility};
+use datafusion_expr::{
+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ TypeSignatureClass, Volatility,
+};
use datafusion_functions::utils::make_scalar_function;
use std::sync::Arc;
diff --git a/datafusion/spark/src/function/table/mod.rs b/datafusion/spark/src/function/table/mod.rs
index aba7b7ceb7..5ccf8d91df 100644
--- a/datafusion/spark/src/function/table/mod.rs
+++ b/datafusion/spark/src/function/table/mod.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use datafusion_catalog::TableFunction;
+use datafusion_session::TableFunction;
use std::sync::Arc;
pub mod expr_fn {}
diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs
index 6cd4678da7..c75f94142f 100644
--- a/datafusion/spark/src/lib.rs
+++ b/datafusion/spark/src/lib.rs
@@ -42,7 +42,7 @@
//! Spark functions taking priority.
//!
//! ```
-//! # use datafusion_execution::FunctionRegistry;
+//! # use datafusion_expr::registry::FunctionRegistry;
//! # use datafusion_expr::{ScalarUDF, AggregateUDF, WindowUDF, HigherOrderUDF};
//! # use datafusion_expr::planner::ExprPlanner;
//! # use datafusion_common::Result;
@@ -137,10 +137,10 @@ mod session_state;
#[cfg(feature = "core")]
pub use session_state::SessionStateBuilderSpark;
-use datafusion_catalog::TableFunction;
use datafusion_common::Result;
-use datafusion_execution::FunctionRegistry;
+use datafusion_expr::registry::FunctionRegistry;
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
+use datafusion_session::TableFunction;
use log::debug;
use std::sync::Arc;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org