You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mi...@apache.org on 2026/04/27 08:22:55 UTC

(datafusion-ballista) branch main updated: fix: propagate session config from Python client to Ballista cluster (#1592)

This is an automated email from the ASF dual-hosted git repository.

milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git


The following commit(s) were added to refs/heads/main by this push:
     new 62b18711c fix: propagate session config from Python client to Ballista cluster (#1592)
62b18711c is described below

commit 62b18711ceb0adef67ab78c4a36ddf6b57a8b97b
Author: Andy Grove <ag...@apache.org>
AuthorDate: Mon Apr 27 02:22:48 2026 -0600

    fix: propagate session config from Python client to Ballista cluster (#1592)
    
    * fix: propagate session config from Python client to Ballista cluster
    
    Closes #1591
    
    The Python `BallistaSessionContext` did not forward DataFusion / Ballista
    session configuration (e.g. `datafusion.execution.target_partitions`) to
    the cluster-side session. Worse, the Rust entry point built the remote
    session from `SessionStateBuilder::new_with_default_features()` whose
    `target_partitions` defaults to the client pod's CPU count and was then
    shipped wholesale to the scheduler, overriding the scheduler's defaults.
    
    Start the remote session from `SessionConfig::new_with_ballista()` and
    add an optional `cluster_config` dict on `BallistaSessionContext` whose
    key/value pairs are applied via `update_from_key_value_pair_mut` before
    the remote session is created.
    
    * fix: also apply cluster_config to local SessionContext
    
    Settings such as datafusion.execution.listing_table_factory_infer_partitions
    are read during table registration / SQL planning on the local context,
    before the logical plan is shipped to the scheduler. Apply each override
    via SessionConfig.set() in addition to forwarding it to the remote ctx.
---
 python/python/ballista/extension.py | 61 ++++++++++++++++++++++++++++++++-----
 python/python/tests/test_context.py | 18 +++++++++++
 python/src/lib.rs                   | 28 +++++++++++++++--
 3 files changed, 97 insertions(+), 10 deletions(-)

diff --git a/python/python/ballista/extension.py b/python/python/ballista/extension.py
index 26b96f16a..51ffc0156 100644
--- a/python/python/ballista/extension.py
+++ b/python/python/ballista/extension.py
@@ -15,10 +15,11 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from datafusion import SessionContext, DataFrame, ParquetWriterOptions
+from datafusion import SessionConfig, SessionContext, DataFrame, ParquetWriterOptions
 from datafusion.dataframe import Compression
 
 from typing import (
+    Dict,
     List,
     Union,
     Optional,
@@ -100,10 +101,14 @@ class RedefiningSessionContextMeta(type):
     def __new__(cls, name, bases, attrs):
         def __wrap_dataframe_result(func):
             def method_wrapper(*args, **kwargs):
-                address = args[0].address
-                session_id = args[0].session_id
+                ctx = args[0]
                 df = func(*args, **kwargs)
-                return DistributedDataFrame(df, session_id, address)
+                return DistributedDataFrame(
+                    df,
+                    ctx.session_id,
+                    ctx.address,
+                    ctx.cluster_config,
+                )
 
             return method_wrapper
 
@@ -131,17 +136,29 @@ class RedefiningSessionContextMeta(type):
 #
 # this class keeps reference to remote ballista
 class DistributedDataFrame(DataFrame, metaclass=RedefiningDataFrameMeta):
-    def __init__(self, df: DataFrame, session_id: str, address: str):
+    def __init__(
+        self,
+        df: DataFrame,
+        session_id: str,
+        address: str,
+        cluster_config: Optional[Dict[str, str]] = None,
+    ):
         super().__init__(df.df)
         self.address = address
         self._session_id = session_id
+        self.cluster_config = cluster_config
     #
     # this will create a ballista dataframe, which has ballista
     # session context, and ballista planner.
     #
     def _to_internal_df(self):
         blob_plan = self.optimized_logical_plan().to_proto()
-        df = create_ballista_data_frame(blob_plan, self.address, self._session_id)
+        df = create_ballista_data_frame(
+            blob_plan,
+            self.address,
+            self._session_id,
+            self.cluster_config,
+        )
         return df
 
     def write_csv(self, path, with_header=False):
@@ -517,16 +534,44 @@ class BallistaSessionContext(SessionContext, metaclass=RedefiningSessionContextM
         >>> df = ctx.sql("SELECT * FROM my_table LIMIT 10")
         >>> df.show()
 
+    To override DataFusion / Ballista session settings on the cluster
+    (e.g. the number of target partitions used by the scheduler):
+
+        >>> ctx = BallistaSessionContext(
+        ...     "df://localhost:50050",
+        ...     cluster_config={"datafusion.execution.target_partitions": "256"},
+        ... )
+
     For Jupyter notebook users:
         >>> %load_ext ballista.jupyter
         >>> %ballista connect df://localhost:50050
         >>> %sql SELECT * FROM my_table
     """
 
-    def __init__(self, address: str, config=None, runtime=None):
+    def __init__(
+        self,
+        address: str,
+        config=None,
+        runtime=None,
+        cluster_config: Optional[Dict[str, str]] = None,
+    ):
+        self.cluster_config = (
+            {str(k): str(v) for k, v in cluster_config.items()}
+            if cluster_config is not None
+            else None
+        )
+        # Apply overrides to the local SessionContext too: some settings
+        # (e.g. datafusion.execution.listing_table_factory_infer_partitions)
+        # are consulted during local table registration / planning, before
+        # the plan is shipped to the scheduler.
+        if self.cluster_config:
+            if config is None:
+                config = SessionConfig()
+            for key, value in self.cluster_config.items():
+                config = config.set(key, value)
         super().__init__(config, runtime)
         self.address = address
-        self.session_id_internal = super().session_id() 
+        self.session_id_internal = super().session_id()
 
     @property
     def session_id(self):
diff --git a/python/python/tests/test_context.py b/python/python/tests/test_context.py
index 1fe71563e..d4d9c4594 100644
--- a/python/python/tests/test_context.py
+++ b/python/python/tests/test_context.py
@@ -73,3 +73,21 @@ def test_read_dataframe_api(ctx):
 
     assert result.column(0) == pa.array([3, 4, 5])
     assert result.column(1) == pa.array([-4, -5, -6])
+
+
+def test_cluster_config_propagates_to_distributed_dataframe():
+    """The cluster_config dict should be passed to DistributedDataFrame
+    instances created through the BallistaSessionContext, so it can be
+    forwarded to the scheduler-side session.
+    """
+    (address, port) = setup_test_cluster()
+    overrides = {"datafusion.execution.target_partitions": "256"}
+    ctx = BallistaSessionContext(
+        address=f"df://{address}:{port}",
+        cluster_config=overrides,
+    )
+
+    assert ctx.cluster_config == overrides
+
+    df = ctx.sql("SELECT 1")
+    assert df.cluster_config == overrides
diff --git a/python/src/lib.rs b/python/src/lib.rs
index b0deb70b4..097f2213a 100644
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -17,11 +17,14 @@
 
 use crate::utils::wait_for_future;
 use ballista::prelude::*;
+use ballista_core::extension::SessionConfigHelperExt;
+use ballista_core::serde::protobuf::KeyValuePair;
 use cluster::{PyExecutor, PyScheduler};
 use datafusion::execution::SessionStateBuilder;
 use datafusion::prelude::*;
 use datafusion_proto::bytes::logical_plan_from_bytes;
 use pyo3::prelude::*;
+use std::collections::HashMap;
 
 mod cluster;
 mod utils;
@@ -50,15 +53,36 @@ fn _internal_ballista(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
 
 /// Creates a DataFrame which runs on ballista session context.
 ///
-/// Returned DataFrame will executed plan on ballista.
+/// `config_overrides` is an optional dictionary of DataFusion / Ballista
+/// configuration keys (e.g. `datafusion.execution.target_partitions`) that
+/// will be applied on top of `SessionConfig::new_with_ballista()` and
+/// propagated to the scheduler-side session.
+///
+/// Returned DataFrame will execute its plan on ballista.
 #[pyfunction]
+#[pyo3(signature = (plan_blob, url, session_id, config_overrides=None))]
 fn create_ballista_data_frame(
     py: Python,
     plan_blob: &[u8],
     url: &str,
     session_id: &str,
+    config_overrides: Option<HashMap<String, String>>,
 ) -> PyResult<datafusion_python::dataframe::PyDataFrame> {
-    let state = SessionStateBuilder::new_with_default_features()
+    let mut session_config = SessionConfig::new_with_ballista();
+    if let Some(overrides) = config_overrides {
+        let pairs: Vec<KeyValuePair> = overrides
+            .into_iter()
+            .map(|(key, value)| KeyValuePair {
+                key,
+                value: Some(value),
+            })
+            .collect();
+        session_config.update_from_key_value_pair_mut(&pairs);
+    }
+
+    let state = SessionStateBuilder::new()
+        .with_default_features()
+        .with_config(session_config)
         .with_session_id(session_id.to_string())
         .build();
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org