You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/08/14 13:45:29 UTC
(datafusion) branch main updated: feat(cli): add opt-in Spark function registration (#24179)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 571477c4a7 feat(cli): add opt-in Spark function registration (#24179)
571477c4a7 is described below
commit 571477c4a7a532fcba67d3e272a3791755dedd79
Author: Goutam Adwant <86...@users.noreply.github.com>
AuthorDate: Fri Aug 14 13:36:59 2026 +0000
feat(cli): add opt-in Spark function registration (#24179)
## Which issue does this PR close?
- Closes #24146.
## Rationale for this change
`datafusion-cli` does not currently enable the Spark SQL dialect or
register the `datafusion-spark` function library, so Spark-compatible
syntax and functions cannot be explored interactively. Enabling them by
default could override core DataFusion functions with the same names and
change existing CLI behavior.
## What changes are included in this PR?
- Add a `--spark` CLI flag.
- Select the Spark SQL dialect while constructing the session
configuration.
- Register Spark-compatible scalar, aggregate, and window functions only
when the flag is enabled.
- Add CLI integration coverage for both default and opt-in behavior.
- Document the flag and its function-override behavior in the CLI and
Spark documentation.
## Are these changes tested?
Yes. The following checks pass:
- `cargo test -p datafusion-cli --test cli_integration
spark_features_require_opt_in`
- `cargo test -p datafusion-cli --all-targets`
- `cargo fmt --all -- --check`
- `cargo clippy --all-targets --all-features -- -D warnings`
- `./ci/scripts/doc_prettier_check.sh`
## Are there any user-facing changes?
Yes. Passing `--spark` enables the Spark SQL dialect, Spark expression
planner, and Spark-compatible functions for the CLI session. The option
is disabled by default, so existing CLI behavior remains unchanged.
---
Cargo.lock | 1 +
datafusion-cli/Cargo.toml | 1 +
datafusion-cli/src/main.rs | 24 +++++++++++++++--
datafusion-cli/tests/cli_integration.rs | 31 ++++++++++++++++++++++
datafusion/spark/README.md | 12 +++++++++
datafusion/sqllogictest/test_files/spark/README.md | 4 +++
docs/source/user-guide/cli/usage.md | 14 ++++++++++
7 files changed, 85 insertions(+), 2 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 5365be12a1..d7576659f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1870,6 +1870,7 @@ dependencies = [
"ctor",
"datafusion",
"datafusion-common",
+ "datafusion-spark",
"dirs",
"env_logger",
"futures",
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index 62eedafe79..50b4002541 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -56,6 +56,7 @@ datafusion = { workspace = true, features = [
"unicode_expressions",
] }
datafusion-common = { workspace = true }
+datafusion-spark = { workspace = true, features = ["core"] }
dirs = "6.0.0"
env_logger = { workspace = true }
futures = { workspace = true }
diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs
index 20a2537d7c..4be6058aef 100644
--- a/datafusion-cli/src/main.rs
+++ b/datafusion-cli/src/main.rs
@@ -23,6 +23,7 @@ use std::process::ExitCode;
use std::sync::{Arc, LazyLock};
use datafusion::error::{DataFusionError, Result};
+use datafusion::execution::SessionStateBuilder;
use datafusion::execution::context::SessionConfig;
use datafusion::execution::memory_pool::{
FairSpillPool, GreedyMemoryPool, MemoryPool, TrackConsumersPool,
@@ -46,9 +47,11 @@ use datafusion_cli::{
};
use clap::Parser;
+use datafusion::common::config::Dialect;
use datafusion::common::config_err;
use datafusion::config::ConfigOptions;
use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
+use datafusion_spark::SessionStateBuilderSpark;
use mimalloc::MiMalloc;
#[global_allocator]
@@ -143,6 +146,12 @@ struct Args {
#[clap(long, help = "Enables console syntax highlighting")]
color: bool,
+ #[clap(
+ long,
+ help = "Enable Apache Spark-compatible SQL dialect, functions, and expression planning"
+ )]
+ spark: bool,
+
#[clap(
short = 'd',
long,
@@ -243,9 +252,16 @@ async fn main_inner() -> Result<()> {
let runtime_env = rt_builder.build_arc()?;
+ let mut state_builder = SessionStateBuilder::new()
+ .with_config(session_config)
+ .with_runtime_env(runtime_env)
+ .with_default_features();
+ if args.spark {
+ state_builder = state_builder.with_spark_features();
+ }
+
// enable dynamic file query
- let ctx = SessionContext::new_with_config_rt(session_config, runtime_env)
- .enable_url_table();
+ let ctx = SessionContext::new_with_state(state_builder.build()).enable_url_table();
ctx.refresh_catalogs().await?;
// install dynamic catalog provider that can register required object stores
ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new(
@@ -350,6 +366,10 @@ fn get_session_config(args: &Args) -> Result<SessionConfig> {
config_options.format.null = String::from("NULL");
}
+ if args.spark {
+ config_options.sql_parser.dialect = Dialect::Spark;
+ }
+
let mut session_config =
SessionConfig::from(config_options).with_information_schema(true);
diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs
index 4dc244445a..9bfeb65278 100644
--- a/datafusion-cli/tests/cli_integration.rs
+++ b/datafusion-cli/tests/cli_integration.rs
@@ -173,6 +173,37 @@ fn cli_quick_test<'a>(
assert_cmd_snapshot!(cmd);
}
+#[test]
+fn spark_features_require_opt_in() {
+ let dialect_query = "SELECT CAST(1 AS LONG);";
+ let spark_query = "SELECT concat_ws(',', array(1, 2)), CAST(1 AS LONG);";
+
+ let output = cli()
+ .args(["-q", "--command", dialect_query])
+ .output()
+ .expect("failed to run datafusion-cli without --spark");
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ assert!(!output.status.success(), "query unexpectedly succeeded");
+ assert!(
+ stdout.contains("Unsupported SQL type LONG"),
+ "expected Spark SQL syntax to be unavailable without --spark, got:\n{stdout}"
+ );
+
+ let output = cli()
+ .args(["-q", "--spark", "--format", "csv", "--command", spark_query])
+ .output()
+ .expect("failed to run datafusion-cli with --spark");
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ assert!(
+ output.status.success(),
+ "query failed with --spark, got:\n{stdout}"
+ );
+ assert!(
+ stdout.lines().any(|line| line == "\"1,2\",1"),
+ "expected Spark planner and dialect result, got:\n{stdout}"
+ );
+}
+
/// Read data piped into the CLI via the `/dev/stdin` pseudo-path.
///
/// Unix-only: `/dev/stdin` does not exist on Windows. This drives the real
diff --git a/datafusion/spark/README.md b/datafusion/spark/README.md
index 7cb24084cd..0ad21604ce 100644
--- a/datafusion/spark/README.md
+++ b/datafusion/spark/README.md
@@ -23,6 +23,18 @@ under the License.
This crate is a submodule of DataFusion that provides [Apache Spark] compatible expressions for use with DataFusion.
+## Using the DataFusion CLI
+
+Pass `--spark` to `datafusion-cli` to enable the Spark SQL dialect and register
+the Spark-compatible scalar, aggregate, and window functions for the session:
+
+```shell
+datafusion-cli --spark
+```
+
+The option is disabled by default because Spark-compatible functions can
+override DataFusion functions with the same name.
+
[apache arrow]: https://arrow.apache.org/
[apache datafusion]: https://datafusion.apache.org/
[apache spark]: https://spark.apache.org/
diff --git a/datafusion/sqllogictest/test_files/spark/README.md b/datafusion/sqllogictest/test_files/spark/README.md
index e61001c6e4..a4550213fa 100644
--- a/datafusion/sqllogictest/test_files/spark/README.md
+++ b/datafusion/sqllogictest/test_files/spark/README.md
@@ -52,6 +52,10 @@ statement ok
set datafusion.execution.enable_ansi_mode = false;
```
+Run `datafusion-cli --spark` to explore Spark-compatible functions
+interactively. This enables the Spark SQL dialect and registers the
+Spark-compatible function library for the CLI session.
+
### Finding Test Cases
To verify and compare function behavior at a minimum, you can refer to the following documentation sources:
diff --git a/docs/source/user-guide/cli/usage.md b/docs/source/user-guide/cli/usage.md
index 75c6698f00..24811a3ce9 100644
--- a/docs/source/user-guide/cli/usage.md
+++ b/docs/source/user-guide/cli/usage.md
@@ -52,6 +52,8 @@ Options:
[possible values: numbers(0/10/...), inf(no limit)] [default: 40]
--color
Enables console syntax highlighting
+ --spark
+ Enable Apache Spark-compatible SQL dialect, functions, and expression planning
-d, --disk-limit <DISK_LIMIT>
Available disk space for spilling queries (e.g. '10g'), default to None (uses DataFusion's default value of '100g')
--object-store-profiling <OBJECT_STORE_PROFILING>
@@ -63,6 +65,18 @@ Options:
Print version
```
+## Apache Spark compatibility
+
+Use `--spark` to enable the Apache Spark SQL dialect, expression planner, and
+function library for the CLI session:
+
+```bash
+datafusion-cli --spark -c "SELECT next_day('2015-07-27'::DATE, 'Sun'::STRING);"
+```
+
+Spark compatibility is disabled by default. Enabling it overrides DataFusion
+functions with the same name for the duration of the session.
+
## Commands
Available commands inside DataFusion CLI are:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org