You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mi...@apache.org on 2026/04/27 08:22:55 UTC
(datafusion-ballista) branch main updated: fix: propagate session config from Python client to Ballista cluster (#1592)
This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 62b18711c fix: propagate session config from Python client to Ballista cluster (#1592)
62b18711c is described below
commit 62b18711ceb0adef67ab78c4a36ddf6b57a8b97b
Author: Andy Grove <ag...@apache.org>
AuthorDate: Mon Apr 27 02:22:48 2026 -0600
fix: propagate session config from Python client to Ballista cluster (#1592)
* fix: propagate session config from Python client to Ballista cluster
Closes #1591
The Python `BallistaSessionContext` did not forward DataFusion / Ballista
session configuration (e.g. `datafusion.execution.target_partitions`) to
the cluster-side session. Worse, the Rust entry point built the remote
session from `SessionStateBuilder::new_with_default_features()` whose
`target_partitions` defaults to the client pod's CPU count and was then
shipped wholesale to the scheduler, overriding the scheduler's defaults.
Start the remote session from `SessionConfig::new_with_ballista()` and
add an optional `cluster_config` dict on `BallistaSessionContext` whose
key/value pairs are applied via `update_from_key_value_pair_mut` before
the remote session is created.
* fix: also apply cluster_config to local SessionContext
Settings such as datafusion.execution.listing_table_factory_infer_partitions
are read during table registration / SQL planning on the local context,
before the logical plan is shipped to the scheduler. Apply each override
via SessionConfig.set() in addition to forwarding it to the remote ctx.
---
python/python/ballista/extension.py | 61 ++++++++++++++++++++++++++++++++-----
python/python/tests/test_context.py | 18 +++++++++++
python/src/lib.rs | 28 +++++++++++++++--
3 files changed, 97 insertions(+), 10 deletions(-)
diff --git a/python/python/ballista/extension.py b/python/python/ballista/extension.py
index 26b96f16a..51ffc0156 100644
--- a/python/python/ballista/extension.py
+++ b/python/python/ballista/extension.py
@@ -15,10 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-from datafusion import SessionContext, DataFrame, ParquetWriterOptions
+from datafusion import SessionConfig, SessionContext, DataFrame, ParquetWriterOptions
from datafusion.dataframe import Compression
from typing import (
+ Dict,
List,
Union,
Optional,
@@ -100,10 +101,14 @@ class RedefiningSessionContextMeta(type):
def __new__(cls, name, bases, attrs):
def __wrap_dataframe_result(func):
def method_wrapper(*args, **kwargs):
- address = args[0].address
- session_id = args[0].session_id
+ ctx = args[0]
df = func(*args, **kwargs)
- return DistributedDataFrame(df, session_id, address)
+ return DistributedDataFrame(
+ df,
+ ctx.session_id,
+ ctx.address,
+ ctx.cluster_config,
+ )
return method_wrapper
@@ -131,17 +136,29 @@ class RedefiningSessionContextMeta(type):
#
# this class keeps reference to remote ballista
class DistributedDataFrame(DataFrame, metaclass=RedefiningDataFrameMeta):
- def __init__(self, df: DataFrame, session_id: str, address: str):
+ def __init__(
+ self,
+ df: DataFrame,
+ session_id: str,
+ address: str,
+ cluster_config: Optional[Dict[str, str]] = None,
+ ):
super().__init__(df.df)
self.address = address
self._session_id = session_id
+ self.cluster_config = cluster_config
#
# this will create a ballista dataframe, which has ballista
# session context, and ballista planner.
#
def _to_internal_df(self):
blob_plan = self.optimized_logical_plan().to_proto()
- df = create_ballista_data_frame(blob_plan, self.address, self._session_id)
+ df = create_ballista_data_frame(
+ blob_plan,
+ self.address,
+ self._session_id,
+ self.cluster_config,
+ )
return df
def write_csv(self, path, with_header=False):
@@ -517,16 +534,44 @@ class BallistaSessionContext(SessionContext, metaclass=RedefiningSessionContextM
>>> df = ctx.sql("SELECT * FROM my_table LIMIT 10")
>>> df.show()
+ To override DataFusion / Ballista session settings on the cluster
+ (e.g. the number of target partitions used by the scheduler):
+
+ >>> ctx = BallistaSessionContext(
+ ... "df://localhost:50050",
+ ... cluster_config={"datafusion.execution.target_partitions": "256"},
+ ... )
+
For Jupyter notebook users:
>>> %load_ext ballista.jupyter
>>> %ballista connect df://localhost:50050
>>> %sql SELECT * FROM my_table
"""
- def __init__(self, address: str, config=None, runtime=None):
+ def __init__(
+ self,
+ address: str,
+ config=None,
+ runtime=None,
+ cluster_config: Optional[Dict[str, str]] = None,
+ ):
+ self.cluster_config = (
+ {str(k): str(v) for k, v in cluster_config.items()}
+ if cluster_config is not None
+ else None
+ )
+ # Apply overrides to the local SessionContext too: some settings
+ # (e.g. datafusion.execution.listing_table_factory_infer_partitions)
+ # are consulted during local table registration / planning, before
+ # the plan is shipped to the scheduler.
+ if self.cluster_config:
+ if config is None:
+ config = SessionConfig()
+ for key, value in self.cluster_config.items():
+ config = config.set(key, value)
super().__init__(config, runtime)
self.address = address
- self.session_id_internal = super().session_id()
+ self.session_id_internal = super().session_id()
@property
def session_id(self):
diff --git a/python/python/tests/test_context.py b/python/python/tests/test_context.py
index 1fe71563e..d4d9c4594 100644
--- a/python/python/tests/test_context.py
+++ b/python/python/tests/test_context.py
@@ -73,3 +73,21 @@ def test_read_dataframe_api(ctx):
assert result.column(0) == pa.array([3, 4, 5])
assert result.column(1) == pa.array([-4, -5, -6])
+
+
+def test_cluster_config_propagates_to_distributed_dataframe():
+ """The cluster_config dict should be passed to DistributedDataFrame
+ instances created through the BallistaSessionContext, so it can be
+ forwarded to the scheduler-side session.
+ """
+ (address, port) = setup_test_cluster()
+ overrides = {"datafusion.execution.target_partitions": "256"}
+ ctx = BallistaSessionContext(
+ address=f"df://{address}:{port}",
+ cluster_config=overrides,
+ )
+
+ assert ctx.cluster_config == overrides
+
+ df = ctx.sql("SELECT 1")
+ assert df.cluster_config == overrides
diff --git a/python/src/lib.rs b/python/src/lib.rs
index b0deb70b4..097f2213a 100644
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -17,11 +17,14 @@
use crate::utils::wait_for_future;
use ballista::prelude::*;
+use ballista_core::extension::SessionConfigHelperExt;
+use ballista_core::serde::protobuf::KeyValuePair;
use cluster::{PyExecutor, PyScheduler};
use datafusion::execution::SessionStateBuilder;
use datafusion::prelude::*;
use datafusion_proto::bytes::logical_plan_from_bytes;
use pyo3::prelude::*;
+use std::collections::HashMap;
mod cluster;
mod utils;
@@ -50,15 +53,36 @@ fn _internal_ballista(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
/// Creates a DataFrame which runs on ballista session context.
///
-/// Returned DataFrame will executed plan on ballista.
+/// `config_overrides` is an optional dictionary of DataFusion / Ballista
+/// configuration keys (e.g. `datafusion.execution.target_partitions`) that
+/// will be applied on top of `SessionConfig::new_with_ballista()` and
+/// propagated to the scheduler-side session.
+///
+/// Returned DataFrame will execute its plan on ballista.
#[pyfunction]
+#[pyo3(signature = (plan_blob, url, session_id, config_overrides=None))]
fn create_ballista_data_frame(
py: Python,
plan_blob: &[u8],
url: &str,
session_id: &str,
+ config_overrides: Option<HashMap<String, String>>,
) -> PyResult<datafusion_python::dataframe::PyDataFrame> {
- let state = SessionStateBuilder::new_with_default_features()
+ let mut session_config = SessionConfig::new_with_ballista();
+ if let Some(overrides) = config_overrides {
+ let pairs: Vec<KeyValuePair> = overrides
+ .into_iter()
+ .map(|(key, value)| KeyValuePair {
+ key,
+ value: Some(value),
+ })
+ .collect();
+ session_config.update_from_key_value_pair_mut(&pairs);
+ }
+
+ let state = SessionStateBuilder::new()
+ .with_default_features()
+ .with_config(session_config)
.with_session_id(session_id.to_string())
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org