You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ti...@apache.org on 2026/08/07 20:07:47 UTC
(datafusion-ballista) 01/04: feat(python): integrate Ballista FFI query planner
This is an automated email from the ASF dual-hosted git repository.
timsaucer pushed a commit to branch feat/ffi-query-planner-55
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
commit f2bcf3c6a9fd07da830222588501064099c328ee
Author: Tim Saucer <ti...@gmail.com>
AuthorDate: Fri Aug 7 12:27:08 2026 -0400
feat(python): integrate Ballista FFI query planner
AI Disclosure: This code was written in part by an AI agent.
---
Cargo.lock | 185 +-
Cargo.toml | 19 +-
ballista/core/proto/ballista.proto | 8 +
.../core/src/execution_plans/distributed_query.rs | 30 +-
ballista/core/src/serde/generated/ballista.rs | 15 +-
ballista/core/src/serde/mod.rs | 321 ++-
python/Cargo.lock | 2034 +++++---------------
python/Cargo.toml | 25 +-
python/pyproject.toml | 3 +-
python/python/ballista/extension.py | 41 +-
python/python/tests/test_context.py | 96 +-
python/src/lib.rs | 10 +
python/src/query_planner.rs | 352 ++++
python/uv.lock | 17 +-
14 files changed, 1478 insertions(+), 1678 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 1c76fe0c9..64838e9bf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -179,9 +179,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
[[package]]
name = "arrow"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b"
+checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -200,9 +200,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff"
+checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -214,9 +214,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192"
+checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97"
dependencies = [
"ahash",
"arrow-buffer",
@@ -233,9 +233,9 @@ dependencies = [
[[package]]
name = "arrow-avro"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84"
+checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -257,21 +257,21 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434"
+checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90"
dependencies = [
"bytes",
"half",
- "num-bigint",
+ "num-bigint 0.5.1",
"num-traits",
]
[[package]]
name = "arrow-cast"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a"
+checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -280,7 +280,7 @@ dependencies = [
"arrow-schema",
"arrow-select",
"atoi",
- "base64 0.22.1",
+ "base64 0.23.1",
"chrono",
"comfy-table",
"half",
@@ -291,9 +291,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784"
+checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -306,9 +306,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66"
+checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -319,9 +319,9 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42115e09dbb694b5955da998912121451c6910b338228cb80a5701370dba43ff"
+checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -334,11 +334,10 @@ dependencies = [
"arrow-schema",
"arrow-select",
"arrow-string",
- "base64 0.22.1",
+ "base64 0.23.1",
"bytes",
"futures",
"once_cell",
- "paste",
"prost",
"prost-types",
"tonic",
@@ -347,9 +346,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c"
+checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -363,9 +362,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6"
+checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -388,9 +387,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b"
+checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -401,9 +400,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592"
+checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -414,9 +413,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5"
+checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e"
dependencies = [
"serde_core",
"serde_json",
@@ -424,9 +423,9 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9"
+checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb"
dependencies = [
"ahash",
"arrow-array",
@@ -438,9 +437,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3"
+checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -1331,7 +1330,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695"
dependencies = [
"autocfg",
"libm",
- "num-bigint",
+ "num-bigint 0.4.8",
"num-integer",
"num-traits",
]
@@ -2197,7 +2196,7 @@ dependencies = [
[[package]]
name = "datafusion"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-schema",
@@ -2250,7 +2249,7 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"async-trait",
@@ -2274,7 +2273,7 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"async-trait",
@@ -2297,7 +2296,7 @@ dependencies = [
[[package]]
name = "datafusion-cli"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"async-trait",
@@ -2324,7 +2323,7 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2351,7 +2350,7 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"futures",
"log",
@@ -2361,7 +2360,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"async-compression",
@@ -2397,7 +2396,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2420,7 +2419,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-avro"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-avro",
@@ -2438,7 +2437,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"async-trait",
@@ -2450,6 +2449,7 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"futures",
"object_store",
@@ -2460,7 +2460,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"async-trait",
@@ -2472,6 +2472,7 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"futures",
"object_store",
@@ -2482,7 +2483,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-parquet"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-schema",
@@ -2499,6 +2500,7 @@ dependencies = [
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-pruning",
"datafusion-session",
"futures",
@@ -2513,12 +2515,12 @@ dependencies = [
[[package]]
name = "datafusion-doc"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
[[package]]
name = "datafusion-execution"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2544,7 +2546,7 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-schema",
@@ -2566,7 +2568,7 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2577,7 +2579,7 @@ dependencies = [
[[package]]
name = "datafusion-functions"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2608,7 +2610,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2628,7 +2630,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2639,7 +2641,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-ord",
@@ -2663,7 +2665,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"async-trait",
@@ -2678,7 +2680,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2694,7 +2696,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2703,7 +2705,7 @@ dependencies = [
[[package]]
name = "datafusion-macros"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"datafusion-doc",
"quote",
@@ -2713,7 +2715,7 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"chrono",
@@ -2732,7 +2734,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2754,7 +2756,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2768,7 +2770,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"chrono",
@@ -2785,7 +2787,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2804,7 +2806,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"arrow-data",
@@ -2840,7 +2842,7 @@ dependencies = [
[[package]]
name = "datafusion-proto"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"chrono",
@@ -2867,7 +2869,7 @@ dependencies = [
[[package]]
name = "datafusion-proto-common"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2877,7 +2879,7 @@ dependencies = [
[[package]]
name = "datafusion-proto-models"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"datafusion-proto-common",
"prost",
@@ -2886,7 +2888,7 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2901,7 +2903,7 @@ dependencies = [
[[package]]
name = "datafusion-session"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow-schema",
"async-trait",
@@ -2915,7 +2917,7 @@ dependencies = [
[[package]]
name = "datafusion-spark"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"bigdecimal",
@@ -2944,7 +2946,7 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"arrow",
"bigdecimal",
@@ -2963,7 +2965,7 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"async-recursion",
"async-trait",
@@ -4471,9 +4473,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lz4_flex"
-version = "0.13.1"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
+checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
dependencies = [
"twox-hash",
]
@@ -4680,7 +4682,7 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
dependencies = [
- "num-bigint",
+ "num-bigint 0.4.8",
"num-complex",
"num-integer",
"num-iter",
@@ -4698,6 +4700,16 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "num-bigint"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
[[package]]
name = "num-complex"
version = "0.4.6"
@@ -4749,7 +4761,7 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
- "num-bigint",
+ "num-bigint 0.4.8",
"num-integer",
"num-traits",
]
@@ -4950,9 +4962,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd"
+checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c"
dependencies = [
"ahash",
"arrow-array",
@@ -4961,7 +4973,7 @@ dependencies = [
"arrow-ipc",
"arrow-schema",
"arrow-select",
- "base64 0.22.1",
+ "base64 0.23.1",
"brotli",
"bytes",
"chrono",
@@ -4970,11 +4982,10 @@ dependencies = [
"half",
"hashbrown 0.17.1",
"lz4_flex",
- "num-bigint",
+ "num-bigint 0.5.1",
"num-integer",
"num-traits",
"object_store",
- "paste",
"ring",
"seq-macro",
"simdutf8",
@@ -5009,12 +5020,6 @@ dependencies = [
"syn 2.0.119",
]
-[[package]]
-name = "paste"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
-
[[package]]
name = "pathdiff"
version = "0.2.3"
diff --git a/Cargo.toml b/Cargo.toml
index 71e7c374d..b8ab443a5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -35,19 +35,20 @@ edition = "2024"
rust-version = "1.88.0"
[workspace.dependencies]
-arrow = { version = "59.1.0", features = ["ipc_compression"] }
-arrow-flight = { version = "59.1.0", features = ["flight-sql-experimental"] }
+arrow = { version = "59.2.0", features = ["ipc_compression"] }
+arrow-flight = { version = "59.2.0", features = ["flight-sql-experimental"] }
clap = { version = "4.5", features = ["derive", "cargo"] }
# DataFusion crates are pinned to a git rev of apache/datafusion main rather than
# a crates.io release. Bump this rev to advance to newer DataFusion commits.
-datafusion = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" }
-datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" }
-datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" }
-datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" }
-datafusion-proto-common = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" }
-datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" }
-datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "70c26a06716eb552f0fc1d2115957a4089504f11" }
+datafusion = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-cli = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-proto-common = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
ctor = { version = "1.0" }
insta = "1.47"
diff --git a/ballista/core/proto/ballista.proto b/ballista/core/proto/ballista.proto
index 99fceb91e..08f540e21 100644
--- a/ballista/core/proto/ballista.proto
+++ b/ballista/core/proto/ballista.proto
@@ -56,9 +56,17 @@ message BallistaPhysicalPlanNode {
UnorderedRangeRepartitionExecNode unordered_range_repartition = 8;
OrderedRangeRepartitionExecNode ordered_range_repartition = 9;
PerPartitionFilterExecNode per_partition_filter = 10;
+ DistributedQueryExecNode distributed_query = 11;
}
}
+message DistributedQueryExecNode {
+ string scheduler_url = 1;
+ repeated KeyValuePair settings = 2;
+ bytes logical_plan = 3;
+ string session_id = 4;
+}
+
// Value-range router over N locally-sorted overlapping input partitions.
// Redistributes them into K range-disjoint output partitions where each
// output is fully sorted on the routing expression. Uses N × K scatter
diff --git a/ballista/core/src/execution_plans/distributed_query.rs b/ballista/core/src/execution_plans/distributed_query.rs
index 20f3234cf..62673c398 100644
--- a/ballista/core/src/execution_plans/distributed_query.rs
+++ b/ballista/core/src/execution_plans/distributed_query.rs
@@ -136,6 +136,26 @@ impl<T: 'static + AsLogicalPlan> DistributedQueryExec<T> {
}
}
+ /// Returns the scheduler URL used by this query.
+ pub fn scheduler_url(&self) -> &str {
+ &self.scheduler_url
+ }
+
+ /// Returns the Ballista configuration used by this query.
+ pub fn config(&self) -> &BallistaConfig {
+ &self.config
+ }
+
+ /// Returns the logical plan submitted to the scheduler.
+ pub fn logical_plan(&self) -> &LogicalPlan {
+ &self.plan
+ }
+
+ /// Returns the client session identifier.
+ pub fn session_id(&self) -> &str {
+ &self.session_id
+ }
+
/// Returns the scheduler job id after the query has been accepted.
pub fn job_id(&self) -> Option<JobId> {
self.job_id.lock().clone()
@@ -227,8 +247,14 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedQueryExec<T> {
DataFusionError::Execution(format!("failed to encode logical plan: {e:?}"))
})?;
- let settings = context
+ // The FFI planner carries Ballista options in this node because the
+ // destination datafusion-python TaskContext does not know Ballista's
+ // ConfigExtension. Restore them before executing the query.
+ let session_config = context
.session_config()
+ .clone()
+ .with_option_extension(self.config.clone());
+ let settings = session_config
.options()
.entries()
.iter()
@@ -255,8 +281,6 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedQueryExec<T> {
let metric_total_bytes =
MetricBuilder::new(&self.metrics).counter("transferred_bytes", partition);
- let session_config = context.session_config().clone();
-
if session_config.ballista_config().client_pull() {
let stream = futures::stream::once(
execute_query_pull(
diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs
index 5bf6964eb..326f95052 100644
--- a/ballista/core/src/serde/generated/ballista.rs
+++ b/ballista/core/src/serde/generated/ballista.rs
@@ -31,7 +31,7 @@ pub struct LogicalPlanCacheNode {
pub struct BallistaPhysicalPlanNode {
#[prost(
oneof = "ballista_physical_plan_node::PhysicalPlanType",
- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10"
+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11"
)]
pub physical_plan_type: ::core::option::Option<
ballista_physical_plan_node::PhysicalPlanType,
@@ -61,8 +61,21 @@ pub mod ballista_physical_plan_node {
OrderedRangeRepartition(super::OrderedRangeRepartitionExecNode),
#[prost(message, tag = "10")]
PerPartitionFilter(super::PerPartitionFilterExecNode),
+ #[prost(message, tag = "11")]
+ DistributedQuery(super::DistributedQueryExecNode),
}
}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct DistributedQueryExecNode {
+ #[prost(string, tag = "1")]
+ pub scheduler_url: ::prost::alloc::string::String,
+ #[prost(message, repeated, tag = "2")]
+ pub settings: ::prost::alloc::vec::Vec<KeyValuePair>,
+ #[prost(bytes = "vec", tag = "3")]
+ pub logical_plan: ::prost::alloc::vec::Vec<u8>,
+ #[prost(string, tag = "4")]
+ pub session_id: ::prost::alloc::string::String,
+}
/// Value-range router over N locally-sorted overlapping input partitions.
/// Redistributes them into K range-disjoint output partitions where each
/// output is fully sorted on the routing expression. Uses N × K scatter
diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs
index eb2ca41d1..5f1623c06 100644
--- a/ballista/core/src/serde/mod.rs
+++ b/ballista/core/src/serde/mod.rs
@@ -18,15 +18,20 @@
//! This crate contains code generated from the Ballista Protocol Buffer Definition as well
//! as convenience code for interacting with the generated code.
-use crate::extension::BallistaCacheNode;
+use crate::extension::{BallistaCacheNode, SessionConfigExt, SessionConfigHelperExt};
use crate::{error::BallistaError, serde::scheduler::Action as BallistaAction};
use arrow_flight::sql::ProstMessageExt;
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::common::{DataFusionError, Result};
use datafusion::execution::TaskContext;
+use datafusion::execution::config::SessionConfig;
use datafusion::logical_expr::Extension;
use datafusion::physical_plan::{ExecutionPlan, Partitioning};
+use datafusion_proto::bytes::{
+ logical_plan_from_bytes_with_extension_codec,
+ logical_plan_to_bytes_with_extension_codec,
+};
use datafusion_proto::logical_plan::file_formats::{
ArrowLogicalExtensionCodec, AvroLogicalExtensionCodec, CsvLogicalExtensionCodec,
JsonLogicalExtensionCodec, ParquetLogicalExtensionCodec,
@@ -56,10 +61,10 @@ use std::{convert::TryInto, io::Cursor};
use crate::execution_plans::sort_shuffle::SortShuffleConfig;
use crate::execution_plans::{
- BufferExec, BufferMode, ChaosExec, CoalescePlan, OrderedRangeRepartitionExec,
- PartitionGroup, PerPartitionFilterExec, RuntimeStatsExec, ShuffleReaderExec,
- ShuffleWriterExec, SortShuffleWriterExec, UnorderedRangeRepartitionExec,
- UnresolvedShuffleExec,
+ BufferExec, BufferMode, ChaosExec, CoalescePlan, DistributedQueryExec,
+ OrderedRangeRepartitionExec, PartitionGroup, PerPartitionFilterExec,
+ RuntimeStatsExec, ShuffleReaderExec, ShuffleWriterExec, SortShuffleWriterExec,
+ UnorderedRangeRepartitionExec, UnresolvedShuffleExec,
};
use crate::serde::protobuf::{
ballista_logical_plan_node::LogicalPlanType,
@@ -192,6 +197,21 @@ pub struct BallistaLogicalExtensionCodec {
}
impl BallistaLogicalExtensionCodec {
+ /// Creates a Ballista logical codec that delegates non-Ballista objects to
+ /// `default_codec` while retaining Ballista's file-format codecs.
+ pub fn new(default_codec: Arc<dyn LogicalExtensionCodec>) -> Self {
+ Self {
+ default_codec,
+ file_format_codecs: vec![
+ Arc::new(ParquetLogicalExtensionCodec {}),
+ Arc::new(CsvLogicalExtensionCodec {}),
+ Arc::new(JsonLogicalExtensionCodec {}),
+ Arc::new(ArrowLogicalExtensionCodec {}),
+ Arc::new(AvroLogicalExtensionCodec {}),
+ ],
+ }
+ }
+
/// looks for a codec which can operate on this node
/// returns a position of codec in the list and result.
///
@@ -220,18 +240,7 @@ impl BallistaLogicalExtensionCodec {
impl Default for BallistaLogicalExtensionCodec {
fn default() -> Self {
- Self {
- default_codec: Arc::new(DefaultLogicalExtensionCodec {}),
- // Position in this list is important as it will be used for decoding.
- // If new codec is added it should go to last position.
- file_format_codecs: vec![
- Arc::new(ParquetLogicalExtensionCodec {}),
- Arc::new(CsvLogicalExtensionCodec {}),
- Arc::new(JsonLogicalExtensionCodec {}),
- Arc::new(ArrowLogicalExtensionCodec {}),
- Arc::new(AvroLogicalExtensionCodec {}),
- ],
- }
+ Self::new(Arc::new(DefaultLogicalExtensionCodec {}))
}
}
@@ -349,22 +358,98 @@ impl LogicalExtensionCodec for BallistaLogicalExtensionCodec {
.encode(buf)
.map_err(|e| DataFusionError::Internal(e.to_string()))
}
+
+ fn try_decode_udf(
+ &self,
+ name: &str,
+ buf: &[u8],
+ ) -> Result<Arc<datafusion::logical_expr::ScalarUDF>> {
+ self.default_codec.try_decode_udf(name, buf)
+ }
+
+ fn try_encode_udf(
+ &self,
+ node: &datafusion::logical_expr::ScalarUDF,
+ buf: &mut Vec<u8>,
+ ) -> Result<()> {
+ self.default_codec.try_encode_udf(node, buf)
+ }
+
+ fn try_decode_higher_order_function(
+ &self,
+ name: &str,
+ buf: &[u8],
+ ) -> Result<Arc<datafusion::logical_expr::HigherOrderUDF>> {
+ self.default_codec
+ .try_decode_higher_order_function(name, buf)
+ }
+
+ fn try_encode_higher_order_function(
+ &self,
+ node: &datafusion::logical_expr::HigherOrderUDF,
+ buf: &mut Vec<u8>,
+ ) -> Result<()> {
+ self.default_codec
+ .try_encode_higher_order_function(node, buf)
+ }
+
+ fn try_decode_udaf(
+ &self,
+ name: &str,
+ buf: &[u8],
+ ) -> Result<Arc<datafusion::logical_expr::AggregateUDF>> {
+ self.default_codec.try_decode_udaf(name, buf)
+ }
+
+ fn try_encode_udaf(
+ &self,
+ node: &datafusion::logical_expr::AggregateUDF,
+ buf: &mut Vec<u8>,
+ ) -> Result<()> {
+ self.default_codec.try_encode_udaf(node, buf)
+ }
+
+ fn try_decode_udwf(
+ &self,
+ name: &str,
+ buf: &[u8],
+ ) -> Result<Arc<datafusion::logical_expr::WindowUDF>> {
+ self.default_codec.try_decode_udwf(name, buf)
+ }
+
+ fn try_encode_udwf(
+ &self,
+ node: &datafusion::logical_expr::WindowUDF,
+ buf: &mut Vec<u8>,
+ ) -> Result<()> {
+ self.default_codec.try_encode_udwf(node, buf)
+ }
}
/// Physical extension codec for Ballista-specific execution plan nodes.
#[derive(Debug)]
pub struct BallistaPhysicalExtensionCodec {
default_codec: Arc<dyn PhysicalExtensionCodec>,
+ logical_codec: Arc<dyn LogicalExtensionCodec>,
}
-impl Default for BallistaPhysicalExtensionCodec {
- fn default() -> Self {
+impl BallistaPhysicalExtensionCodec {
+ /// Creates a physical codec that uses `logical_codec` for logical plans
+ /// embedded in client-side Ballista execution nodes.
+ pub fn new(logical_codec: Arc<dyn LogicalExtensionCodec>) -> Self {
Self {
default_codec: Arc::new(DefaultPhysicalExtensionCodec {}),
+ logical_codec,
}
}
}
+impl Default for BallistaPhysicalExtensionCodec {
+ fn default() -> Self {
+ Self::new(Arc::new(BallistaLogicalExtensionCodec::default()))
+ }
+}
+
impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
fn try_decode(
&self,
@@ -663,6 +748,31 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
predicates,
)?))
}
+ PhysicalPlanType::DistributedQuery(distributed_query) => {
+ if !inputs.is_empty() {
+ return Err(DataFusionError::Internal(format!(
+ "DistributedQueryExec expects no inputs, got {}",
+ inputs.len()
+ )));
+ }
+ let plan = logical_plan_from_bytes_with_extension_codec(
+ &distributed_query.logical_plan,
+ ctx,
+ self.logical_codec.as_ref(),
+ )?;
+ let config = SessionConfig::new_with_ballista()
+ .update_from_key_value_pair(&distributed_query.settings)
+ .ballista_config();
+ Ok(Arc::new(
+ DistributedQueryExec::<LogicalPlanNode>::with_extension(
+ distributed_query.scheduler_url.clone(),
+ config,
+ plan,
+ Arc::clone(&self.logical_codec),
+ distributed_query.session_id.clone(),
+ ),
+ ))
+ }
}
}
@@ -924,6 +1034,38 @@ impl PhysicalExtensionCodec for BallistaPhysicalExtensionCodec {
))
})?;
Ok(())
+ } else if let Some(exec) =
+ node.downcast_ref::<DistributedQueryExec<LogicalPlanNode>>()
+ {
+ let logical_plan = logical_plan_to_bytes_with_extension_codec(
+ exec.logical_plan(),
+ self.logical_codec.as_ref(),
+ )?;
+ let settings = exec
+ .config()
+ .settings()
+ .iter()
+ .map(|(key, value)| protobuf::KeyValuePair {
+ key: key.clone(),
+ value: Some(value.clone()),
+ })
+ .collect();
+ let proto = protobuf::BallistaPhysicalPlanNode {
+ physical_plan_type: Some(PhysicalPlanType::DistributedQuery(
+ protobuf::DistributedQueryExecNode {
+ scheduler_url: exec.scheduler_url().to_string(),
+ settings,
+ logical_plan: logical_plan.to_vec(),
+ session_id: exec.session_id().to_string(),
+ },
+ )),
+ };
+ proto.encode(buf).map_err(|e| {
+ DataFusionError::Internal(format!(
+ "failed to encode distributed query execution plan: {e:?}"
+ ))
+ })?;
+ Ok(())
} else {
Err(DataFusionError::Internal(format!(
"Unsupported plan node, name: [{}] ",
@@ -968,6 +1110,109 @@ mod test {
use datafusion_proto::{logical_plan::AsLogicalPlan, protobuf::LogicalPlanNode};
use std::sync::Arc;
+ #[derive(Debug)]
+ struct MarkerLogicalCodec;
+
+ impl LogicalExtensionCodec for MarkerLogicalCodec {
+ fn try_decode(
+ &self,
+ _buf: &[u8],
+ _inputs: &[LogicalPlan],
+ _ctx: &TaskContext,
+ ) -> Result<Extension> {
+ Err(DataFusionError::Plan("marker-plan-decode".to_string()))
+ }
+
+ fn try_encode(&self, _node: &Extension, _buf: &mut Vec<u8>) -> Result<()> {
+ Err(DataFusionError::Plan("marker-plan-encode".to_string()))
+ }
+
+ fn try_decode_table_provider(
+ &self,
+ _buf: &[u8],
+ _table_ref: &datafusion::common::TableReference,
+ _schema: SchemaRef,
+ _ctx: &TaskContext,
+ ) -> Result<Arc<dyn datafusion::catalog::TableProvider>> {
+ Err(DataFusionError::Plan("marker-table-decode".to_string()))
+ }
+
+ fn try_encode_table_provider(
+ &self,
+ _table_ref: &datafusion::common::TableReference,
+ _node: Arc<dyn datafusion::catalog::TableProvider>,
+ _buf: &mut Vec<u8>,
+ ) -> Result<()> {
+ Err(DataFusionError::Plan("marker-table-encode".to_string()))
+ }
+
+ fn try_decode_udf(
+ &self,
+ _name: &str,
+ _buf: &[u8],
+ ) -> Result<Arc<datafusion::logical_expr::ScalarUDF>> {
+ Err(DataFusionError::Plan("marker-scalar".to_string()))
+ }
+
+ fn try_decode_higher_order_function(
+ &self,
+ _name: &str,
+ _buf: &[u8],
+ ) -> Result<Arc<datafusion::logical_expr::HigherOrderUDF>> {
+ Err(DataFusionError::Plan("marker-higher-order".to_string()))
+ }
+
+ fn try_decode_udaf(
+ &self,
+ _name: &str,
+ _buf: &[u8],
+ ) -> Result<Arc<datafusion::logical_expr::AggregateUDF>> {
+ Err(DataFusionError::Plan("marker-aggregate".to_string()))
+ }
+
+ fn try_decode_udwf(
+ &self,
+ _name: &str,
+ _buf: &[u8],
+ ) -> Result<Arc<datafusion::logical_expr::WindowUDF>> {
+ Err(DataFusionError::Plan("marker-window".to_string()))
+ }
+ }
+
+ #[test]
+ fn logical_codec_delegates_function_decoding() {
+ let codec = BallistaLogicalExtensionCodec::new(Arc::new(MarkerLogicalCodec));
+
+ assert!(
+ codec
+ .try_decode_udf("udf", &[])
+ .unwrap_err()
+ .to_string()
+ .contains("marker-scalar")
+ );
+ assert!(
+ codec
+ .try_decode_udaf("udaf", &[])
+ .unwrap_err()
+ .to_string()
+ .contains("marker-aggregate")
+ );
+ assert!(
+ codec
+ .try_decode_udwf("udwf", &[])
+ .unwrap_err()
+ .to_string()
+ .contains("marker-window")
+ );
+ assert!(
+ codec
+ .try_decode_higher_order_function("higher", &[])
+ .unwrap_err()
+ .to_string()
+ .contains("marker-higher-order")
+ );
+ }
+
#[tokio::test]
async fn file_format_serialization_roundtrip() {
let ctx = SessionContext::new().task_ctx();
@@ -1093,6 +1338,44 @@ mod test {
);
}
+ #[tokio::test]
+ async fn test_distributed_query_exec_roundtrip() {
+ let logical_plan = LogicalPlan::EmptyRelation(EmptyRelation {
+ produce_one_row: true,
+ schema: Arc::new(DFSchema::empty()),
+ });
+ let config = SessionConfig::new_with_ballista()
+ .set_bool(crate::config::BALLISTA_CLIENT_PULL, true)
+ .ballista_config();
+ let original_exec = DistributedQueryExec::<LogicalPlanNode>::new(
+ "http://localhost:50050".to_string(),
+ config,
+ logical_plan.clone(),
+ "test-session".to_string(),
+ );
+
+ let codec = BallistaPhysicalExtensionCodec::default();
+ let converter = DefaultPhysicalProtoConverter {};
+ let mut buf = Vec::new();
+ codec
+ .try_encode(Arc::new(original_exec), &mut buf, &converter)
+ .unwrap();
+
+ let ctx = SessionContext::new().task_ctx();
+ let decoded_plan = codec.try_decode(&buf, &[], &ctx, &converter).unwrap();
+ let decoded_exec = decoded_plan
+ .downcast_ref::<DistributedQueryExec<LogicalPlanNode>>()
+ .expect("Expected DistributedQueryExec");
+
+ assert_eq!(decoded_exec.scheduler_url(), "http://localhost:50050");
+ assert_eq!(decoded_exec.session_id(), "test-session");
+ assert!(decoded_exec.config().client_pull());
+ assert_eq!(
+ decoded_exec.logical_plan().display_indent().to_string(),
+ logical_plan.display_indent().to_string()
+ );
+ }
+
#[tokio::test]
async fn test_unresolved_shuffle_exec_roundtrip() {
let schema = create_test_schema();
diff --git a/python/Cargo.lock b/python/Cargo.lock
index 9f8e3776b..1a2121812 100644
--- a/python/Cargo.lock
+++ b/python/Cargo.lock
@@ -99,108 +99,55 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
[[package]]
name = "arrow"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3"
-dependencies = [
- "arrow-arith 58.4.0",
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-cast 58.4.0",
- "arrow-csv 58.4.0",
- "arrow-data 58.4.0",
- "arrow-ipc 58.4.0",
- "arrow-json 58.4.0",
- "arrow-ord 58.4.0",
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf"
+dependencies = [
+ "arrow-arith",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-csv",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-json",
+ "arrow-ord",
"arrow-pyarrow",
- "arrow-row 58.4.0",
- "arrow-schema 58.4.0",
- "arrow-select 58.4.0",
- "arrow-string 58.4.0",
-]
-
-[[package]]
-name = "arrow"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b"
-dependencies = [
- "arrow-arith 59.1.0",
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-cast 59.1.0",
- "arrow-csv 59.1.0",
- "arrow-data 59.1.0",
- "arrow-ipc 59.1.0",
- "arrow-json 59.1.0",
- "arrow-ord 59.1.0",
- "arrow-row 59.1.0",
- "arrow-schema 59.1.0",
- "arrow-select 59.1.0",
- "arrow-string 59.1.0",
+ "arrow-row",
+ "arrow-schema",
+ "arrow-select",
+ "arrow-string",
]
[[package]]
name = "arrow-arith"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a"
+checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4"
dependencies = [
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-schema 58.4.0",
- "chrono",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-arith"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff"
-dependencies = [
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-schema 59.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"chrono",
"num-traits",
]
[[package]]
name = "arrow-array"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69"
+checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97"
dependencies = [
"ahash",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-schema 58.4.0",
- "chrono",
- "chrono-tz",
- "half",
- "hashbrown 0.17.1",
- "num-complex",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-array"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192"
-dependencies = [
- "ahash",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-schema 59.1.0",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"chrono",
"chrono-tz",
"half",
"hashbrown 0.17.1",
+ "libc",
"num-complex",
"num-integer",
"num-traits",
@@ -208,13 +155,13 @@ dependencies = [
[[package]]
name = "arrow-avro"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8108cbcae32141577bc55ab490449944c63cfc93d47384906b475d13d89d43d"
+checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61"
dependencies = [
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-schema 58.4.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-schema",
"bytes",
"bzip2",
"crc",
@@ -232,64 +179,30 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5"
-dependencies = [
- "bytes",
- "half",
- "num-bigint",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-buffer"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434"
+checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90"
dependencies = [
"bytes",
"half",
- "num-bigint",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-cast"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34"
-dependencies = [
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-ord 58.4.0",
- "arrow-schema 58.4.0",
- "arrow-select 58.4.0",
- "atoi",
- "base64 0.22.1",
- "chrono",
- "comfy-table",
- "half",
- "lexical-core",
+ "num-bigint 0.5.1",
"num-traits",
- "ryu",
]
[[package]]
name = "arrow-cast"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a"
+checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79"
dependencies = [
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-ord 59.1.0",
- "arrow-schema 59.1.0",
- "arrow-select 59.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-ord",
+ "arrow-schema",
+ "arrow-select",
"atoi",
- "base64 0.22.1",
+ "base64 0.23.1",
"chrono",
"comfy-table",
"half",
@@ -300,55 +213,27 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26"
+checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36"
dependencies = [
- "arrow-array 58.4.0",
- "arrow-cast 58.4.0",
- "arrow-schema 58.4.0",
+ "arrow-array",
+ "arrow-cast",
+ "arrow-schema",
"chrono",
"csv",
"csv-core",
"regex",
]
-[[package]]
-name = "arrow-csv"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784"
-dependencies = [
- "arrow-array 59.1.0",
- "arrow-cast 59.1.0",
- "arrow-schema 59.1.0",
- "chrono",
- "csv",
- "csv-core",
- "regex",
-]
-
-[[package]]
-name = "arrow-data"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b"
-dependencies = [
- "arrow-buffer 58.4.0",
- "arrow-schema 58.4.0",
- "half",
- "num-integer",
- "num-traits",
-]
-
[[package]]
name = "arrow-data"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66"
+checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d"
dependencies = [
- "arrow-buffer 59.1.0",
- "arrow-schema 59.1.0",
+ "arrow-buffer",
+ "arrow-schema",
"half",
"num-integer",
"num-traits",
@@ -356,26 +241,25 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42115e09dbb694b5955da998912121451c6910b338228cb80a5701370dba43ff"
-dependencies = [
- "arrow-arith 59.1.0",
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-cast 59.1.0",
- "arrow-data 59.1.0",
- "arrow-ipc 59.1.0",
- "arrow-ord 59.1.0",
- "arrow-row 59.1.0",
- "arrow-schema 59.1.0",
- "arrow-select 59.1.0",
- "arrow-string 59.1.0",
- "base64 0.22.1",
+version = "59.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64"
+dependencies = [
+ "arrow-arith",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-ord",
+ "arrow-row",
+ "arrow-schema",
+ "arrow-select",
+ "arrow-string",
+ "base64 0.23.1",
"bytes",
"futures",
"once_cell",
- "paste",
"prost",
"prost-types",
"tonic",
@@ -384,31 +268,15 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b"
-dependencies = [
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-schema 58.4.0",
- "arrow-select 58.4.0",
- "flatbuffers",
- "lz4_flex",
- "zstd",
-]
-
-[[package]]
-name = "arrow-ipc"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c"
+checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b"
dependencies = [
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-schema 59.1.0",
- "arrow-select 59.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
"flatbuffers",
"lz4_flex",
"zstd",
@@ -416,41 +284,16 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14"
-dependencies = [
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-cast 58.4.0",
- "arrow-ord 58.4.0",
- "arrow-schema 58.4.0",
- "arrow-select 58.4.0",
- "chrono",
- "half",
- "indexmap",
- "itoa",
- "lexical-core",
- "memchr",
- "num-traits",
- "ryu",
- "serde_core",
- "serde_json",
- "simdutf8",
-]
-
-[[package]]
-name = "arrow-json"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6"
+checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e"
dependencies = [
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-cast 59.1.0",
- "arrow-ord 59.1.0",
- "arrow-schema 59.1.0",
- "arrow-select 59.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-ord",
+ "arrow-schema",
+ "arrow-select",
"chrono",
"half",
"indexmap",
@@ -466,145 +309,78 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0"
+checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979"
dependencies = [
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-schema 58.4.0",
- "arrow-select 58.4.0",
-]
-
-[[package]]
-name = "arrow-ord"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b"
-dependencies = [
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-schema 59.1.0",
- "arrow-select 59.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
]
[[package]]
name = "arrow-pyarrow"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec"
+checksum = "c196ecc25b3a8dcbc1d842f2619cee653dcfa2fb8b56a291bc0481c3cf5c3821"
dependencies = [
- "arrow-array 58.4.0",
- "arrow-data 58.4.0",
- "arrow-schema 58.4.0",
+ "arrow-array",
+ "arrow-data",
+ "arrow-schema",
"pyo3",
]
[[package]]
name = "arrow-row"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c"
+checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2"
dependencies = [
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-schema 58.4.0",
- "half",
-]
-
-[[package]]
-name = "arrow-row"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592"
-dependencies = [
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-schema 59.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"half",
]
[[package]]
name = "arrow-schema"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e"
+checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e"
dependencies = [
"bitflags",
"serde_core",
"serde_json",
]
-[[package]]
-name = "arrow-schema"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5"
-dependencies = [
- "serde_core",
- "serde_json",
-]
-
[[package]]
name = "arrow-select"
-version = "58.4.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad"
+checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb"
dependencies = [
"ahash",
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-schema 58.4.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"num-traits",
]
-[[package]]
-name = "arrow-select"
-version = "59.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9"
-dependencies = [
- "ahash",
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-schema 59.1.0",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-string"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c"
-dependencies = [
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-schema 58.4.0",
- "arrow-select 58.4.0",
- "memchr",
- "num-traits",
- "regex",
- "regex-syntax",
-]
-
[[package]]
name = "arrow-string"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3"
+checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1"
dependencies = [
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-schema 59.1.0",
- "arrow-select 59.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
"memchr",
"num-traits",
"regex",
@@ -721,7 +497,7 @@ dependencies = [
"ballista-core",
"ballista-executor",
"ballista-scheduler",
- "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion",
"log",
"tokio",
"url",
@@ -734,10 +510,10 @@ dependencies = [
"arrow-flight",
"async-trait",
"chrono",
- "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion",
+ "datafusion-functions-aggregate-common",
+ "datafusion-proto",
+ "datafusion-proto-common",
"futures",
"itertools 0.15.0",
"log",
@@ -762,14 +538,14 @@ dependencies = [
name = "ballista-executor"
version = "54.0.0"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"arrow-flight",
"async-trait",
"ballista-core",
"bytesize",
"dashmap",
- "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion",
+ "datafusion-proto",
"futures",
"libc",
"log",
@@ -795,8 +571,8 @@ dependencies = [
"axum",
"ballista-core",
"dashmap",
- "datafusion 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-proto 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion",
+ "datafusion-proto",
"futures",
"http",
"insta",
@@ -836,7 +612,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695"
dependencies = [
"autocfg",
"libm",
- "num-bigint",
+ "num-bigint 0.4.8",
"num-integer",
"num-traits",
]
@@ -915,12 +691,6 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
[[package]]
name = "bytes"
version = "1.12.1"
@@ -1201,92 +971,39 @@ dependencies = [
[[package]]
name = "datafusion"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
- "arrow-schema 58.4.0",
+ "arrow",
+ "arrow-schema",
"async-trait",
"bzip2",
"chrono",
- "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-catalog-listing 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource-arrow 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "datafusion-catalog",
+ "datafusion-catalog-listing",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-datasource-arrow",
"datafusion-datasource-avro",
- "datafusion-datasource-csv 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource-json 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource-parquet 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-table 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-window 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-sql 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "flate2",
- "futures",
- "indexmap",
- "itertools 0.14.0",
- "liblzma",
- "log",
- "object_store",
- "parking_lot",
- "parquet 58.4.0",
- "sqlparser",
- "tempfile",
- "tokio",
- "url",
- "uuid",
- "zstd",
-]
-
-[[package]]
-name = "datafusion"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "arrow-schema 59.1.0",
- "async-trait",
- "bzip2",
- "chrono",
- "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-window 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-optimizer 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-sql 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-datasource-csv",
+ "datafusion-datasource-json",
+ "datafusion-datasource-parquet",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-functions",
+ "datafusion-functions-aggregate",
+ "datafusion-functions-nested",
+ "datafusion-functions-table",
+ "datafusion-functions-window",
+ "datafusion-optimizer",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-adapter",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-optimizer",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "datafusion-sql",
"flate2",
"futures",
"indexmap",
@@ -1295,7 +1012,7 @@ dependencies = [
"log",
"object_store",
"parking_lot",
- "parquet 59.1.0",
+ "parquet",
"sqlparser",
"tempfile",
"tokio",
@@ -1307,44 +1024,19 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
+ "arrow",
"async-trait",
"dashmap",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
- "parking_lot",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-catalog"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "async-trait",
- "dashmap",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-plan",
+ "datafusion-session",
"futures",
"itertools 0.15.0",
"log",
@@ -1356,42 +1048,19 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
-dependencies = [
- "arrow 58.4.0",
- "async-trait",
- "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
-]
-
-[[package]]
-name = "datafusion-catalog-listing"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"async-trait",
- "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-adapter",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
"futures",
"itertools 0.15.0",
"log",
@@ -1402,37 +1071,11 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
- "arrow-ipc 58.4.0",
- "arrow-schema 58.4.0",
- "chrono",
- "foldhash 0.2.0",
- "half",
- "hashbrown 0.17.1",
- "indexmap",
- "itertools 0.14.0",
- "libc",
- "log",
- "object_store",
- "parquet 58.4.0",
- "recursive",
- "sqlparser",
- "tokio",
- "uuid",
- "web-time",
-]
-
-[[package]]
-name = "datafusion-common"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "arrow-ipc 59.1.0",
- "arrow-schema 59.1.0",
+ "arrow",
+ "arrow-ipc",
+ "arrow-schema",
"chrono",
"foldhash 0.2.0",
"half",
@@ -1443,7 +1086,7 @@ dependencies = [
"log",
"num-traits",
"object_store",
- "parquet 59.1.0",
+ "parquet",
"recursive",
"sqlparser",
"tokio",
@@ -1454,81 +1097,34 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
"futures",
"log",
"tokio",
]
-[[package]]
-name = "datafusion-common-runtime"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "futures",
- "log",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-datasource"
-version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
-dependencies = [
- "arrow 58.4.0",
- "async-compression",
- "async-trait",
- "bytes",
- "bzip2",
- "chrono",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "flate2",
- "futures",
- "glob",
- "itertools 0.14.0",
- "liblzma",
- "log",
- "object_store",
- "parking_lot",
- "rand 0.9.5",
- "tokio",
- "tokio-util",
- "url",
- "zstd",
-]
-
[[package]]
name = "datafusion-datasource"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"async-compression",
"async-trait",
"bytes",
"bzip2",
"chrono",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-adapter",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
"datafusion-proto-models",
- "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-session",
"flate2",
"futures",
"glob",
@@ -1547,44 +1143,20 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
- "arrow-ipc 58.4.0",
+ "arrow",
+ "arrow-ipc",
"async-trait",
"bytes",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures",
- "itertools 0.14.0",
- "object_store",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-datasource-arrow"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "arrow-ipc 59.1.0",
- "async-trait",
- "bytes",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
"futures",
"itertools 0.15.0",
"object_store",
@@ -1594,18 +1166,17 @@ dependencies = [
[[package]]
name = "datafusion-datasource-avro"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
+ "arrow",
"arrow-avro",
"async-trait",
"bytes",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-physical-expr-adapter",
+ "datafusion-physical-plan",
+ "datafusion-session",
"futures",
"object_store",
]
@@ -1613,42 +1184,20 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
+ "arrow",
"async-trait",
"bytes",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures",
- "object_store",
- "regex",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-datasource-csv"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "async-trait",
- "bytes",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-proto-models",
+ "datafusion-session",
"futures",
"object_store",
"regex",
@@ -1658,42 +1207,20 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
-dependencies = [
- "arrow 58.4.0",
- "async-trait",
- "bytes",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures",
- "object_store",
- "tokio",
- "tokio-stream",
-]
-
-[[package]]
-name = "datafusion-datasource-json"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"async-trait",
"bytes",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-proto-models",
+ "datafusion-session",
"futures",
"object_store",
"tokio",
@@ -1703,111 +1230,53 @@ dependencies = [
[[package]]
name = "datafusion-datasource-parquet"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
+ "arrow",
+ "arrow-schema",
"async-trait",
"bytes",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-adapter 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-pruning 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
- "parking_lot",
- "parquet 58.4.0",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-datasource-parquet"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "arrow-schema 59.1.0",
- "async-trait",
- "bytes",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-adapter 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-functions-aggregate-common",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-adapter",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-proto-models",
+ "datafusion-pruning",
+ "datafusion-session",
"futures",
"itertools 0.15.0",
"log",
"object_store",
"parking_lot",
- "parquet 59.1.0",
+ "parquet",
"tokio",
]
[[package]]
name = "datafusion-doc"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"
-
-[[package]]
-name = "datafusion-doc"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
[[package]]
name = "datafusion-execution"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
- "arrow-buffer 58.4.0",
- "async-trait",
- "dashmap",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures",
- "log",
- "object_store",
- "parking_lot",
- "rand 0.9.5",
- "tempfile",
- "url",
-]
-
-[[package]]
-name = "datafusion-execution"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "arrow-buffer 59.1.0",
+ "arrow",
+ "arrow-buffer",
"async-trait",
"bytes",
"dashmap",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-physical-expr-common",
"futures",
"log",
"object_store",
@@ -1823,41 +1292,18 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
- "arrow-schema 58.4.0",
+ "arrow",
+ "arrow-schema",
"async-trait",
"chrono",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "indexmap",
- "itertools 0.14.0",
- "recursive",
- "serde_json",
- "sqlparser",
-]
-
-[[package]]
-name = "datafusion-expr"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "arrow-schema 59.1.0",
- "async-trait",
- "chrono",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-expr-common",
+ "datafusion-functions-aggregate-common",
+ "datafusion-functions-window-common",
+ "datafusion-physical-expr-common",
"indexmap",
"itertools 0.15.0",
"recursive",
@@ -1868,22 +1314,10 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "indexmap",
- "itertools 0.14.0",
-]
-
-[[package]]
-name = "datafusion-expr-common"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "arrow",
+ "datafusion-common",
"indexmap",
"itertools 0.15.0",
]
@@ -1891,27 +1325,26 @@ dependencies = [
[[package]]
name = "datafusion-ffi"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9a4a09f24f1387408810acca070207a9cf6c7fc2368122982a8e7671f643d1e"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
- "arrow-schema 58.4.0",
+ "arrow",
+ "arrow-schema",
"async-ffi",
"async-trait",
"chrono",
- "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-optimizer 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-proto-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-session 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions-aggregate-common",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-optimizer",
+ "datafusion-physical-plan",
+ "datafusion-proto",
+ "datafusion-proto-common",
+ "datafusion-session",
"futures",
"libloading",
"log",
@@ -1924,101 +1357,48 @@ dependencies = [
[[package]]
name = "datafusion-functions"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
-dependencies = [
- "arrow 58.4.0",
- "arrow-buffer 58.4.0",
- "base64 0.22.1",
- "blake2",
- "blake3",
- "chrono",
- "chrono-tz",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hex",
- "itertools 0.14.0",
- "log",
- "md-5 0.11.0",
- "memchr",
- "num-traits",
- "rand 0.9.5",
- "regex",
- "sha2",
- "uuid",
-]
-
-[[package]]
-name = "datafusion-functions"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
- "arrow-buffer 59.1.0",
+ "arrow",
+ "arrow-buffer",
"base64 0.23.1",
"blake2",
"blake3",
"chrono",
"chrono-tz",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-macros",
+ "datafusion-physical-expr-common",
"hex",
"itertools 0.15.0",
"log",
"md-5 0.11.0",
- "memchr",
- "num-traits",
- "rand 0.9.5",
- "regex",
- "sha2",
- "uuid",
-]
-
-[[package]]
-name = "datafusion-functions-aggregate"
-version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "foldhash 0.2.0",
- "half",
- "log",
+ "memchr",
"num-traits",
+ "rand 0.9.5",
+ "regex",
+ "sha2",
+ "uuid",
]
[[package]]
name = "datafusion-functions-aggregate"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions-aggregate-common",
+ "datafusion-macros",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
"half",
"hashbrown 0.17.1",
"log",
@@ -2028,68 +1408,31 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "datafusion-functions-aggregate-common"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
-]
-
-[[package]]
-name = "datafusion-functions-nested"
-version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
-dependencies = [
- "arrow 58.4.0",
- "arrow-ord 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hashbrown 0.17.1",
- "itertools 0.14.0",
- "itoa",
- "log",
- "memchr",
+ "arrow",
+ "datafusion-common",
+ "datafusion-expr-common",
+ "datafusion-physical-expr-common",
]
[[package]]
name = "datafusion-functions-nested"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "arrow-ord 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
+dependencies = [
+ "arrow",
+ "arrow-ord",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-functions",
+ "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
+ "datafusion-macros",
+ "datafusion-physical-expr-common",
"hashbrown 0.17.1",
"itertools 0.15.0",
"itoa",
@@ -2100,103 +1443,49 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
-dependencies = [
- "arrow 58.4.0",
- "async-trait",
- "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "parking_lot",
-]
-
-[[package]]
-name = "datafusion-functions-table"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"async-trait",
- "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-plan",
"parking_lot",
]
[[package]]
name = "datafusion-functions-window"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-macros 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "log",
-]
-
-[[package]]
-name = "datafusion-functions-window"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-macros 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "datafusion-doc",
+ "datafusion-expr",
+ "datafusion-functions-window-common",
+ "datafusion-macros",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
"log",
]
[[package]]
name = "datafusion-functions-window-common"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
-dependencies = [
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "datafusion-functions-window-common"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
-]
-
-[[package]]
-name = "datafusion-macros"
-version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "datafusion-doc 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote",
- "syn 2.0.119",
+ "datafusion-common",
+ "datafusion-physical-expr-common",
]
[[package]]
name = "datafusion-macros"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "datafusion-doc 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-doc",
"quote",
"syn 3.0.3",
]
@@ -2204,34 +1493,14 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
-dependencies = [
- "arrow 58.4.0",
- "chrono",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "indexmap",
- "itertools 0.14.0",
- "log",
- "recursive",
- "regex",
- "regex-syntax",
-]
-
-[[package]]
-name = "datafusion-optimizer"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"chrono",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-physical-expr",
"indexmap",
"itertools 0.15.0",
"log",
@@ -2243,36 +1512,14 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "half",
- "hashbrown 0.17.1",
- "indexmap",
- "itertools 0.14.0",
- "parking_lot",
- "petgraph",
- "recursive",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-physical-expr"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-functions-aggregate-common",
+ "datafusion-physical-expr-common",
"datafusion-proto-models",
"half",
"hashbrown 0.17.1",
@@ -2287,58 +1534,26 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "itertools 0.14.0",
-]
-
-[[package]]
-name = "datafusion-physical-expr-adapter"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
"itertools 0.15.0",
]
[[package]]
name = "datafusion-physical-expr-common"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
-dependencies = [
- "arrow 58.4.0",
- "chrono",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hashbrown 0.17.1",
- "indexmap",
- "itertools 0.14.0",
- "parking_lot",
- "pin-project",
-]
-
-[[package]]
-name = "datafusion-physical-expr-common"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"chrono",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-expr-common",
"datafusion-proto-models",
"hashbrown 0.17.1",
"indexmap",
@@ -2350,37 +1565,18 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-pruning 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "itertools 0.14.0",
- "recursive",
-]
-
-[[package]]
-name = "datafusion-physical-optimizer"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-pruning 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-session 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-pruning",
+ "datafusion-session",
"itertools 0.15.0",
"recursive",
]
@@ -2388,58 +1584,25 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
-dependencies = [
- "arrow 58.4.0",
- "arrow-data 58.4.0",
- "arrow-ipc 58.4.0",
- "arrow-ord 58.4.0",
- "arrow-schema 58.4.0",
- "async-trait",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common-runtime 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-window-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "futures",
- "half",
- "hashbrown 0.17.1",
- "indexmap",
- "itertools 0.14.0",
- "log",
- "num-traits",
- "parking_lot",
- "pin-project-lite",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-physical-plan"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
- "arrow-data 59.1.0",
- "arrow-ipc 59.1.0",
- "arrow-ord 59.1.0",
- "arrow-schema 59.1.0",
+ "arrow",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-ord",
+ "arrow-schema",
"async-trait",
"bytes",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common-runtime 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-aggregate-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-window-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-functions-aggregate-common",
+ "datafusion-functions-window-common",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-proto-common",
"datafusion-proto-models",
"futures",
"half",
@@ -2457,52 +1620,25 @@ dependencies = [
[[package]]
name = "datafusion-proto"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e"
-dependencies = [
- "arrow 58.4.0",
- "chrono",
- "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-catalog-listing 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource-arrow 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource-csv 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource-json 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource-parquet 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-table 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-proto-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "object_store",
- "prost",
-]
-
-[[package]]
-name = "datafusion-proto"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"chrono",
- "datafusion-catalog 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-catalog-listing 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource-arrow 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource-csv 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource-json 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource-parquet 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-table 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-catalog",
+ "datafusion-catalog-listing",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-datasource-arrow",
+ "datafusion-datasource-csv",
+ "datafusion-datasource-json",
+ "datafusion-datasource-parquet",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions-table",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-proto-common",
"datafusion-proto-models",
"object_store",
"prost",
@@ -2511,79 +1647,52 @@ dependencies = [
[[package]]
name = "datafusion-proto-common"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "prost",
-]
-
-[[package]]
-name = "datafusion-proto-common"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "arrow",
+ "datafusion-common",
"prost",
]
[[package]]
name = "datafusion-proto-models"
version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "datafusion-proto-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-proto-common",
"prost",
]
[[package]]
name = "datafusion-pruning"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
-dependencies = [
- "arrow 58.4.0",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-datasource 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-expr-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "log",
-]
-
-[[package]]
-name = "datafusion-pruning"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
-dependencies = [
- "arrow 59.1.0",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-datasource 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-expr-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-expr-common",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
"log",
]
[[package]]
name = "datafusion-python"
version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b5da6aab44da83d488e6fb05855f5cd4480c7ab6b44f164166b495cba9fbbb8"
+source = "git+https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca"
dependencies = [
- "arrow 58.4.0",
- "arrow-select 58.4.0",
+ "arrow",
+ "arrow-select",
"async-trait",
"chrono",
"cstr",
- "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "datafusion",
"datafusion-ffi",
- "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "datafusion-proto",
"datafusion-python-util",
+ "datafusion-session",
"datafusion-spark",
"futures",
"log",
@@ -2593,7 +1702,7 @@ dependencies = [
"prost-types",
"pyo3",
"pyo3-async-runtimes",
- "pyo3-build-config 0.28.3",
+ "pyo3-build-config",
"pyo3-log",
"serde_json",
"tokio",
@@ -2604,13 +1713,12 @@ dependencies = [
[[package]]
name = "datafusion-python-util"
version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba491c15adfebc9ae53aa11f65231814b5199627e9815e2fb62d2f81f4a17fb2"
+source = "git+https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca"
dependencies = [
- "arrow 58.4.0",
- "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "arrow",
+ "datafusion",
"datafusion-ffi",
- "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "datafusion-proto",
"prost",
"pyo3",
"tokio",
@@ -2619,49 +1727,35 @@ dependencies = [
[[package]]
name = "datafusion-session"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
-dependencies = [
- "async-trait",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-physical-plan 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "parking_lot",
-]
-
-[[package]]
-name = "datafusion-session"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow-schema 59.1.0",
+ "arrow-schema",
"async-trait",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-execution 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-physical-plan 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-plan",
"parking_lot",
]
[[package]]
name = "datafusion-spark"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 58.4.0",
+ "arrow",
"bigdecimal",
"chrono",
"crc32fast",
- "datafusion-catalog 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-execution 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-aggregate-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "datafusion",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
+ "datafusion-functions-nested",
"log",
"num-traits",
"percent-encoding",
@@ -2676,33 +1770,14 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "54.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
-dependencies = [
- "arrow 58.4.0",
- "bigdecimal",
- "chrono",
- "datafusion-common 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-expr 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-functions-nested 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "indexmap",
- "log",
- "recursive",
- "regex",
- "sqlparser",
-]
-
-[[package]]
-name = "datafusion-sql"
-version = "54.1.0"
-source = "git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11#70c26a06716eb552f0fc1d2115957a4089504f11"
+source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
dependencies = [
- "arrow 59.1.0",
+ "arrow",
"bigdecimal",
"chrono",
- "datafusion-common 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-expr 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
- "datafusion-functions-nested 54.1.0 (git+https://github.com/apache/datafusion.git?rev=70c26a06716eb552f0fc1d2115957a4089504f11)",
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-functions-nested",
"indexmap",
"log",
"recursive",
@@ -3333,12 +2408,6 @@ dependencies = [
"tempfile",
]
-[[package]]
-name = "integer-encoding"
-version = "3.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
-
[[package]]
name = "ipnet"
version = "2.12.0"
@@ -3539,9 +2608,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lz4_flex"
-version = "0.13.1"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
+checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
dependencies = [
"twox-hash",
]
@@ -3640,6 +2709,16 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "num-bigint"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
[[package]]
name = "num-complex"
version = "0.4.6"
@@ -3748,15 +2827,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
-[[package]]
-name = "ordered-float"
-version = "2.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
-dependencies = [
- "num-traits",
-]
-
[[package]]
name = "parking_lot"
version = "0.12.5"
@@ -3782,54 +2852,18 @@ dependencies = [
[[package]]
name = "parquet"
-version = "58.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704"
-dependencies = [
- "ahash",
- "arrow-array 58.4.0",
- "arrow-buffer 58.4.0",
- "arrow-data 58.4.0",
- "arrow-ipc 58.4.0",
- "arrow-schema 58.4.0",
- "arrow-select 58.4.0",
- "base64 0.22.1",
- "brotli",
- "bytes",
- "chrono",
- "flate2",
- "futures",
- "half",
- "hashbrown 0.17.1",
- "lz4_flex",
- "num-bigint",
- "num-integer",
- "num-traits",
- "object_store",
- "paste",
- "seq-macro",
- "simdutf8",
- "snap",
- "thrift",
- "tokio",
- "twox-hash",
- "zstd",
-]
-
-[[package]]
-name = "parquet"
-version = "59.1.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd"
+checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c"
dependencies = [
"ahash",
- "arrow-array 59.1.0",
- "arrow-buffer 59.1.0",
- "arrow-data 59.1.0",
- "arrow-ipc 59.1.0",
- "arrow-schema 59.1.0",
- "arrow-select 59.1.0",
- "base64 0.22.1",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-schema",
+ "arrow-select",
+ "base64 0.23.1",
"brotli",
"bytes",
"chrono",
@@ -3838,11 +2872,10 @@ dependencies = [
"half",
"hashbrown 0.17.1",
"lz4_flex",
- "num-bigint",
+ "num-bigint 0.5.1",
"num-integer",
"num-traits",
"object_store",
- "paste",
"seq-macro",
"simdutf8",
"snap",
@@ -3851,12 +2884,6 @@ dependencies = [
"zstd",
]
-[[package]]
-name = "paste"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
-
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -4069,36 +3096,38 @@ dependencies = [
"ballista-core",
"ballista-executor",
"ballista-scheduler",
- "datafusion 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "datafusion-proto 54.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "datafusion",
+ "datafusion-ffi",
+ "datafusion-proto",
"datafusion-python",
"log",
"pyo3",
- "pyo3-build-config 0.29.0",
+ "pyo3-build-config",
"pyo3-log",
"tokio",
"tonic",
+ "url",
]
[[package]]
name = "pyo3"
-version = "0.28.3"
+version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
+checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
dependencies = [
"libc",
"once_cell",
"portable-atomic",
- "pyo3-build-config 0.28.3",
+ "pyo3-build-config",
"pyo3-ffi",
"pyo3-macros",
]
[[package]]
name = "pyo3-async-runtimes"
-version = "0.28.0"
+version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e7364a95bf00e8377bbf9b0f09d7ff9715a29d8fcf93b47d1a967363b973178"
+checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046"
dependencies = [
"futures-channel",
"futures-util",
@@ -4110,31 +3139,21 @@ dependencies = [
[[package]]
name = "pyo3-build-config"
-version = "0.28.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
-dependencies = [
- "python3-dll-a",
- "target-lexicon",
-]
-
-[[package]]
-name = "pyo3-build-config"
-version = "0.29.0"
+version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
+checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
dependencies = [
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
-version = "0.28.3"
+version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
+checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
dependencies = [
"libc",
- "pyo3-build-config 0.28.3",
+ "pyo3-build-config",
]
[[package]]
@@ -4150,9 +3169,9 @@ dependencies = [
[[package]]
name = "pyo3-macros"
-version = "0.28.3"
+version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813"
+checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
@@ -4162,26 +3181,16 @@ dependencies = [
[[package]]
name = "pyo3-macros-backend"
-version = "0.28.3"
+version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
+checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
dependencies = [
"heck",
"proc-macro2",
- "pyo3-build-config 0.28.3",
"quote",
"syn 2.0.119",
]
-[[package]]
-name = "python3-dll-a"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b"
-dependencies = [
- "cc",
-]
-
[[package]]
name = "quick-xml"
version = "0.39.4"
@@ -4935,17 +3944,6 @@ dependencies = [
"syn 3.0.3",
]
-[[package]]
-name = "thrift"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
-dependencies = [
- "byteorder",
- "integer-encoding",
- "ordered-float",
-]
-
[[package]]
name = "tiny-keccak"
version = "2.0.2"
diff --git a/python/Cargo.toml b/python/Cargo.toml
index 071260583..10527a71b 100644
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -35,14 +35,16 @@ ballista = { path = "../ballista/client", version = "54.0.0" }
ballista-core = { path = "../ballista/core", version = "54.0.0" }
ballista-executor = { path = "../ballista/executor", version = "54.0.0", default-features = false }
ballista-scheduler = { path = "../ballista/scheduler", version = "54.0.0", default-features = false }
-datafusion = { version = "54", features = ["avro"] }
-datafusion-proto = { version = "54" }
-datafusion-python = { version = "54", default-features = false }
+datafusion = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e", features = ["avro"] }
+datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-python = { git = "https://github.com/apache/datafusion-python.git", rev = "55f43b5e231da5591e7bbd632dac85f304a22cca", default-features = false }
log = { version = "0.4" }
-pyo3 = { version = "0.28", features = ["extension-module", "abi3", "abi3-py310"] }
+pyo3 = { version = "0.29", features = ["extension-module", "abi3", "abi3-py310"] }
pyo3-log = "0.13"
tokio = { version = "1.48", features = ["macros", "rt", "rt-multi-thread", "sync"] }
tonic = { version = "0.14" }
+url = { version = "2.5" }
[lib]
crate-type = ["cdylib"]
@@ -50,3 +52,18 @@ name = "ballista"
[build-dependencies]
pyo3-build-config = "0.29"
+
+# datafusion-python's workspace patch does not apply when it is consumed as a
+# dependency. Repeat its exact DataFusion patch here so the standalone Python
+# crate uses one ABI-compatible DataFusion source family.
+[patch.crates-io]
+datafusion = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-catalog = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-functions-window = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-session = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
+datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 081c504b4..4e85fbb07 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -41,7 +41,8 @@ classifiers = [
]
dependencies = [
"pyarrow>=23.0.1",
- "datafusion==54",
+ # Temporary PR #1672 pin. Use its released version before publishing.
+ "datafusion @ git+https://github.com/apache/datafusion-python.git@55f43b5e231da5591e7bbd632dac85f304a22cca",
"typing-extensions;python_version<'3.13'",
]
dynamic = ["version"]
diff --git a/python/python/ballista/extension.py b/python/python/ballista/extension.py
index 645c99518..3455da013 100644
--- a/python/python/ballista/extension.py
+++ b/python/python/ballista/extension.py
@@ -26,7 +26,9 @@ from datafusion.dataframe import Compression
from typing import Dict, List, Union, Optional, Callable
import warnings
+from ._internal_ballista import ballista_datafusion_config_defaults
from ._internal_ballista import create_ballista_data_frame
+from ._internal_ballista import with_ballista_query_planner
from ._internal_ballista import ParquetColumnOptions as ParquetColumnOptionsInternal
from ._internal_ballista import ParquetWriterOptions as ParquetWriterOptionsInternal
@@ -53,9 +55,9 @@ class RedefiningDataFrameMeta(type):
def __wrap_dataframe_execution(func):
def method_wrapper(*args, **kwargs):
slf, *argz = args
- df = slf._to_internal_df()
-
- return getattr(df, func)(*argz, **kwargs)
+ # The DataFrame already carries the FFI-installed Ballista
+ # planner, so normal terminal operations execute it directly.
+ return getattr(DataFrame, func)(slf, *argz, **kwargs)
return method_wrapper
@@ -66,7 +68,12 @@ class RedefiningDataFrameMeta(type):
address = args[0].address
session_id = args[0].session_id
df = func(*args, **kwargs)
- return DistributedDataFrame(df, session_id, address)
+ return DistributedDataFrame(
+ df,
+ session_id,
+ address,
+ args[0].cluster_config,
+ )
return method_wrapper
@@ -531,6 +538,9 @@ class ExecutionPlanVisualization:
return f"ExecutionPlanVisualization(analyze={self.analyze})\n{self.plan_str}"
+# Keep the compatibility wrapper for write_* methods because DataFusion's
+# logical codec FFI does not yet transport the file-format factories used by COPY.
+# Normal terminal operations execute through the installed FFI planner.
class BallistaSessionContext(SessionContext, metaclass=RedefiningSessionContextMeta):
"""
A session context for connecting to and querying a Ballista cluster.
@@ -571,20 +581,17 @@ class BallistaSessionContext(SessionContext, metaclass=RedefiningSessionContextM
if cluster_config is not None
else None
)
- # Apply overrides to the local SessionContext too: some settings
- # (e.g. datafusion.execution.listing_table_factory_infer_partitions)
- # are consulted during local table registration / planning, before
- # the plan is shipped to the scheduler. Ballista-namespaced keys
- # are not understood by the local SessionConfig and are forwarded
- # to the scheduler only.
- if self.cluster_config:
- if config is None:
- config = SessionConfig()
- for key, value in self.cluster_config.items():
- if key.startswith("ballista."):
- continue
+ if config is None:
+ config = SessionConfig()
+ for key, value in ballista_datafusion_config_defaults().items():
config = config.set(key, value)
- super().__init__(config, runtime)
+ source_ctx = SessionContext(config, runtime)
+ configured_ctx = with_ballista_query_planner(
+ source_ctx,
+ address,
+ self.cluster_config,
+ )
+ self.ctx = configured_ctx.ctx
self.address = address
self.session_id_internal = super().session_id()
diff --git a/python/python/tests/test_context.py b/python/python/tests/test_context.py
index 6e5180378..e823a5ed8 100644
--- a/python/python/tests/test_context.py
+++ b/python/python/tests/test_context.py
@@ -16,8 +16,19 @@
# under the License.
from ballista import BallistaSessionContext, setup_test_cluster
-from ballista.extension import DataFrame, DistributedDataFrame, SessionContext
+from ballista._internal_ballista import (
+ BallistaQueryPlanner,
+ ballista_datafusion_config_defaults,
+ with_ballista_query_planner,
+)
+from ballista.extension import (
+ DataFrame,
+ DistributedDataFrame,
+ SessionConfig,
+ SessionContext,
+)
from datafusion import col, lit
+import ctypes
import pytest
import pyarrow as pa
@@ -28,14 +39,82 @@ def ctx():
return BallistaSessionContext(address=f"df://{address}:{port}")
+def assert_uses_ballista(df):
+ assert "DistributedQueryExec" in str(df.execution_plan())
+
+
+def test_query_planner_capsules_use_current_datafusion_abi():
+ source_ctx = SessionContext()
+ planner = BallistaQueryPlanner("df://localhost:50050", source_ctx)
+ is_valid = ctypes.pythonapi.PyCapsule_IsValid
+ is_valid.argtypes = [ctypes.py_object, ctypes.c_char_p]
+ is_valid.restype = ctypes.c_int
+
+ capsules = {
+ b"datafusion_query_planner": planner.__datafusion_query_planner__(),
+ b"datafusion_logical_extension_codec": (
+ planner.__datafusion_logical_extension_codec__()
+ ),
+ b"datafusion_physical_extension_codec": (
+ planner.__datafusion_physical_extension_codec__()
+ ),
+ }
+ for name, capsule in capsules.items():
+ assert is_valid(capsule, name) == 1
+
+
+def test_low_level_helper_returns_standard_distributed_dataframe():
+ address, port = setup_test_cluster()
+ source_ctx = SessionContext()
+ source_ctx.register_csv("registered", "testdata/test.csv", has_header=True)
+ configured_ctx = with_ballista_query_planner(
+ source_ctx,
+ f"df://{address}:{port}",
+ {"datafusion.execution.target_partitions": "7"},
+ )
+
+ assert type(configured_ctx) is SessionContext
+ assert "datafusion.execution.target_partitions = 7" in str(source_ctx)
+ assert "datafusion.execution.target_partitions = 7" in str(configured_ctx)
+ df = configured_ctx.sql("SELECT COUNT(*) FROM registered")
+ assert type(df) is DataFrame
+ assert_uses_ballista(df)
+ assert df.collect()[0].column(0).to_pylist() == [5]
+
+
+def test_ballista_datafusion_defaults_are_applied(ctx):
+ config_text = str(ctx)
+
+ for key, value in ballista_datafusion_config_defaults().items():
+ assert f"{key} = {value}" in config_text
+
+
+def test_caller_session_config_is_not_overwritten():
+ key = "datafusion.execution.target_partitions"
+ config = SessionConfig().set(key, "7")
+ ctx = BallistaSessionContext("df://localhost:50050", config=config)
+
+ assert f"{key} = 7" in str(ctx)
+
+ override_config = SessionConfig().set(key, "7")
+ override_ctx = BallistaSessionContext(
+ "df://localhost:50050",
+ config=override_config,
+ cluster_config={key: "8"},
+ )
+ assert f"{key} = 8" in str(override_ctx)
+
+
def test_select_one(ctx):
df = ctx.sql("SELECT 1")
+ assert_uses_ballista(df)
batches = df.collect()
assert len(batches) == 1
def test_read_csv(ctx):
df = ctx.read_csv("testdata/test.csv", has_header=True)
+ assert_uses_ballista(df)
batches = df.collect()
assert len(batches) == 1
assert len(batches[0]) == 5
@@ -44,6 +123,7 @@ def test_read_csv(ctx):
def test_register_csv(ctx):
ctx.register_csv("test", "testdata/test.csv", has_header=True)
df = ctx.sql("SELECT * FROM test")
+ assert_uses_ballista(df)
batches = df.collect()
assert len(batches) == 1
assert len(batches[0]) == 5
@@ -51,6 +131,7 @@ def test_register_csv(ctx):
def test_read_parquet(ctx):
df = ctx.read_parquet("testdata/test.parquet")
+ assert_uses_ballista(df)
batches = df.collect()
assert len(batches) == 1
assert len(batches[0]) == 8
@@ -59,6 +140,7 @@ def test_read_parquet(ctx):
def test_register_parquet(ctx):
ctx.register_parquet("test", "testdata/test.parquet")
df = ctx.sql("SELECT * FROM test")
+ assert_uses_ballista(df)
batches = df.collect()
assert len(batches) == 1
assert len(batches[0]) == 8
@@ -70,6 +152,7 @@ def test_read_dataframe_api(ctx):
.select("a", "b")
.filter(col("a") > lit(2))
)
+ assert_uses_ballista(df)
result = df.collect()[0]
assert result.column(0) == pa.array([3, 4, 5])
@@ -92,6 +175,8 @@ def test_cluster_config_propagates_to_distributed_dataframe():
df = ctx.sql("SELECT 1")
assert df.cluster_config == overrides
+ assert_uses_ballista(df)
+ assert len(df.collect()) == 1
def test_cluster_config_accepts_ballista_namespaced_keys():
@@ -114,9 +199,18 @@ def test_cluster_config_accepts_ballista_namespaced_keys():
df = ctx.sql("SELECT 1")
assert df.cluster_config == overrides
+ assert_uses_ballista(df)
assert len(df.collect()) == 1
+def test_malformed_scheduler_url_fails_lazily():
+ ctx = BallistaSessionContext(address="not a scheduler URL")
+ ctx.register_csv("test", "testdata/test.csv", has_header=True)
+
+ with pytest.raises(Exception, match="relative URL without a base"):
+ ctx.sql("SELECT * FROM test").collect()
+
+
def test_write_csv(ctx, tmp_path):
df = ctx.read_csv("testdata/test.csv", has_header=True)
out_dir = str(tmp_path / "out")
diff --git a/python/src/lib.rs b/python/src/lib.rs
index d927311be..ba58070d3 100644
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -28,6 +28,7 @@ use pyo3::prelude::*;
use std::collections::HashMap;
mod cluster;
+mod query_planner;
mod utils;
pub(crate) struct TokioRuntime(tokio::runtime::Runtime);
@@ -38,12 +39,21 @@ fn _internal_ballista(_py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyScheduler>()?;
m.add_class::<PyExecutor>()?;
+ m.add_class::<query_planner::PyBallistaQueryPlanner>()?;
m.add_class::<datafusion_python::dataframe::PyParquetWriterOptions>()?;
m.add_class::<datafusion_python::dataframe::PyParquetColumnOptions>()?;
m.add_class::<datafusion_python::dataframe::PyDataFrame>()?;
m.add_function(wrap_pyfunction!(create_ballista_data_frame, m.clone())?)?;
+ m.add_function(wrap_pyfunction!(
+ query_planner::ballista_datafusion_config_defaults,
+ m.clone()
+ )?)?;
+ m.add_function(wrap_pyfunction!(
+ query_planner::with_ballista_query_planner,
+ m.clone()
+ )?)?;
m.add_function(wrap_pyfunction!(
crate::cluster::setup_test_cluster,
m.clone()
diff --git a/python/src/query_planner.rs b/python/src/query_planner.rs
new file mode 100644
index 000000000..403ca99ae
--- /dev/null
+++ b/python/src/query_planner.rs
@@ -0,0 +1,352 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use ballista_core::extension::SessionConfigExt;
+use ballista_core::planner::BallistaQueryPlanner;
+use ballista_core::serde::{
+ BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec,
+};
+use datafusion::catalog::Session;
+use datafusion::common::{DataFusionError, Result};
+use datafusion::execution::config::SessionConfig;
+use datafusion::execution::context::QueryPlanner;
+use datafusion::logical_expr::LogicalPlan;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion_ffi::execution::FFI_TaskContextProvider;
+use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;
+use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec;
+use datafusion_ffi::query_planner::FFI_QueryPlanner;
+use datafusion_proto::logical_plan::{
+ DefaultLogicalExtensionCodec, LogicalExtensionCodec,
+};
+use datafusion_proto::physical_plan::PhysicalExtensionCodec;
+use datafusion_proto::protobuf::LogicalPlanNode;
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+use pyo3::types::PyCapsule;
+use std::collections::HashMap;
+use std::ptr::NonNull;
+use std::sync::{Arc, OnceLock};
+use tokio::runtime::{Handle, Runtime};
+use url::Url;
+
+const DEFAULT_SCHEDULER_PORT: u16 = 50050;
+
+#[pyclass(
+ name = "BallistaQueryPlanner",
+ module = "ballista._internal_ballista",
+ skip_from_py_object
+)]
+pub(crate) struct PyBallistaQueryPlanner {
+ address: String,
+ config: ballista_core::config::BallistaConfig,
+ logical_codec: Arc<dyn LogicalExtensionCodec>,
+ ffi_logical_codec: FFI_LogicalExtensionCodec,
+ ffi_physical_codec: FFI_PhysicalExtensionCodec,
+ // The FFI codecs hold a weak task-context provider. Retain its source
+ // context for as long as Python retains this planner.
+ _session_ctx: Py<PyAny>,
+}
+
+#[pymethods]
+impl PyBallistaQueryPlanner {
+ #[new]
+ #[pyo3(signature = (address, session_ctx, config_overrides=None))]
+ fn new(
+ address: String,
+ session_ctx: Bound<'_, PyAny>,
+ config_overrides: Option<HashMap<String, String>>,
+ ) -> PyResult<Self> {
+ let user_logical_codec: Arc<dyn LogicalExtensionCodec> =
+ (&ffi_logical_codec_from_python(session_ctx.clone())?).into();
+ // The planner-side codec must retain the source context's Python-aware
+ // codecs because DistributedQueryExec embeds a serialized logical plan.
+ let logical_codec: Arc<dyn LogicalExtensionCodec> =
+ Arc::new(BallistaLogicalExtensionCodec::new(user_logical_codec));
+ let physical_codec: Arc<dyn PhysicalExtensionCodec> = Arc::new(
+ BallistaPhysicalExtensionCodec::new(Arc::clone(&logical_codec)),
+ );
+
+ // The receiving SessionContext composes an installed codec with its own
+ // codecs. Export only the Ballista layer here; wrapping the source codec
+ // again would make the receiver traverse the same codec stack twice.
+ let installed_logical_codec: Arc<dyn LogicalExtensionCodec> = Arc::new(
+ BallistaLogicalExtensionCodec::new(Arc::new(
+ DefaultLogicalExtensionCodec {},
+ )),
+ );
+ let task_ctx_provider = ffi_task_ctx_provider_from_python(session_ctx.clone())?;
+ let runtime = tokio_runtime_handle();
+ let ffi_logical_codec = FFI_LogicalExtensionCodec::new(
+ installed_logical_codec,
+ Some(runtime.clone()),
+ task_ctx_provider.clone(),
+ );
+ let ffi_physical_codec = FFI_PhysicalExtensionCodec::new(
+ physical_codec,
+ Some(runtime),
+ task_ctx_provider.clone(),
+ );
+
+ let mut session_config = SessionConfig::new_with_ballista();
+ if let Some(overrides) = config_overrides {
+ for (key, value) in overrides {
+ if !key.starts_with("ballista.") {
+ return Err(PyValueError::new_err(format!(
+ "unsupported Ballista configuration key: {key}"
+ )));
+ }
+ session_config
+ .options_mut()
+ .set(&key, &value)
+ .map_err(|error| PyValueError::new_err(error.to_string()))?;
+ }
+ }
+
+ Ok(Self {
+ address,
+ config: session_config.ballista_config(),
+ logical_codec,
+ ffi_logical_codec,
+ ffi_physical_codec,
+ _session_ctx: session_ctx.unbind(),
+ })
+ }
+
+ fn __datafusion_logical_extension_codec__<'py>(
+ &self,
+ py: Python<'py>,
+ ) -> PyResult<Bound<'py, PyCapsule>> {
+ PyCapsule::new_with_value(
+ py,
+ self.ffi_logical_codec.clone(),
+ cr"datafusion_logical_extension_codec",
+ )
+ }
+
+ fn __datafusion_physical_extension_codec__<'py>(
+ &self,
+ py: Python<'py>,
+ ) -> PyResult<Bound<'py, PyCapsule>> {
+ PyCapsule::new_with_value(
+ py,
+ self.ffi_physical_codec.clone(),
+ cr"datafusion_physical_extension_codec",
+ )
+ }
+
+ fn __datafusion_query_planner__<'py>(
+ &self,
+ py: Python<'py>,
+ ) -> PyResult<Bound<'py, PyCapsule>> {
+ let planner: Arc<dyn QueryPlanner + Send + Sync> =
+ Arc::new(LazyBallistaQueryPlanner {
+ address: self.address.clone(),
+ config: self.config.clone(),
+ logical_codec: Arc::clone(&self.logical_codec),
+ _session_ctx: self._session_ctx.clone_ref(py),
+ });
+ let ffi = FFI_QueryPlanner::new_with_ffi_codecs(
+ planner,
+ self.ffi_logical_codec.clone(),
+ self.ffi_physical_codec.clone(),
+ );
+
+ // This exact name is the ABI consumed by datafusion-python 55f43b5e.
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_query_planner")
+ }
+}
+
+struct LazyBallistaQueryPlanner {
+ address: String,
+ config: ballista_core::config::BallistaConfig,
+ logical_codec: Arc<dyn LogicalExtensionCodec>,
+ _session_ctx: Py<PyAny>,
+}
+
+impl std::fmt::Debug for LazyBallistaQueryPlanner {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("LazyBallistaQueryPlanner")
+ .field("address", &self.address)
+ .field("config", &self.config)
+ .finish_non_exhaustive()
+ }
+}
+
+#[async_trait::async_trait]
+impl QueryPlanner for LazyBallistaQueryPlanner {
+ async fn create_physical_plan(
+ &self,
+ logical_plan: &LogicalPlan,
+ session: &dyn Session,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ if matches!(logical_plan, LogicalPlan::Analyze(_)) {
+ return Err(DataFusionError::NotImplemented(
+ "EXPLAIN ANALYZE is not yet supported by the Ballista Python FFI planner"
+ .to_string(),
+ ));
+ }
+
+ // Parse only when DataFusion asks for a physical plan so malformed
+ // addresses fail lazily, not during context construction.
+ let scheduler_url = parse_scheduler_url(&self.address)?;
+ let planner = BallistaQueryPlanner::<LogicalPlanNode>::with_extension(
+ scheduler_url,
+ self.config.clone(),
+ Arc::clone(&self.logical_codec),
+ );
+ planner.create_physical_plan(logical_plan, session).await
+ }
+}
+
+#[pyfunction]
+pub(crate) fn ballista_datafusion_config_defaults() -> HashMap<String, String> {
+ let defaults = SessionConfig::new();
+ let ballista = SessionConfig::new_with_ballista();
+ let default_values = defaults
+ .options()
+ .entries()
+ .into_iter()
+ .map(|entry| (entry.key, entry.value))
+ .collect::<HashMap<_, _>>();
+
+ ballista
+ .options()
+ .entries()
+ .into_iter()
+ .filter_map(|entry| {
+ if !entry.key.starts_with("datafusion.") {
+ return None;
+ }
+ let value = entry.value?;
+ (default_values.get(&entry.key) != Some(&Some(value.clone())))
+ .then_some((entry.key, value))
+ })
+ .collect()
+}
+
+#[pyfunction]
+#[pyo3(signature = (session_ctx, address, config_overrides=None))]
+pub(crate) fn with_ballista_query_planner(
+ py: Python<'_>,
+ session_ctx: Bound<'_, PyAny>,
+ address: String,
+ config_overrides: Option<HashMap<String, String>>,
+) -> PyResult<Py<PyAny>> {
+ let mut ballista_overrides = HashMap::new();
+ if let Some(overrides) = config_overrides {
+ for (key, value) in overrides {
+ if key.starts_with("datafusion.") {
+ apply_datafusion_override(&session_ctx, &key, &value)?;
+ } else if key.starts_with("ballista.") {
+ ballista_overrides.insert(key, value);
+ } else {
+ return Err(PyValueError::new_err(format!(
+ "configuration key must start with 'datafusion.' or 'ballista.': {key}"
+ )));
+ }
+ }
+ }
+
+ let planner = PyBallistaQueryPlanner::new(
+ address,
+ session_ctx.clone(),
+ Some(ballista_overrides),
+ )?;
+ let planner = Py::new(py, planner)?;
+
+ // datafusion-python deliberately rebinds imported planners to the codecs
+ // installed on the receiving context. Install Ballista's codecs first so
+ // that rebind keeps DistributedQueryExec serialization active.
+ let ctx = session_ctx.call_method1(
+ "with_logical_extension_codec",
+ (planner.bind(py),),
+ )?;
+ let ctx = ctx.call_method1(
+ "with_physical_extension_codec",
+ (planner.bind(py),),
+ )?;
+ Ok(ctx
+ .call_method1("with_query_planner", (planner.bind(py),))?
+ .unbind())
+}
+
+fn apply_datafusion_override(
+ session_ctx: &Bound<'_, PyAny>,
+ key: &str,
+ value: &str,
+) -> PyResult<()> {
+ if !key
+ .bytes()
+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_'))
+ {
+ return Err(PyValueError::new_err(format!(
+ "invalid DataFusion configuration key: {key}"
+ )));
+ }
+
+ // SET mutates the shared session state. The contexts derived below retain
+ // that state, including registered tables and functions.
+ let escaped_value = value.replace('\'', "''");
+ let statement = format!("SET {key} = '{escaped_value}'");
+ session_ctx.call_method1("sql", (statement,))?;
+ Ok(())
+}
+
+fn parse_scheduler_url(address: &str) -> Result<String> {
+ let url =
+ Url::parse(address).map_err(|e| DataFusionError::Configuration(e.to_string()))?;
+ let host = url.host().ok_or_else(|| {
+ DataFusionError::Configuration("hostname should be provided".to_string())
+ })?;
+ let port = url.port().unwrap_or(DEFAULT_SCHEDULER_PORT);
+ Ok(format!("http://{host}:{port}"))
+}
+
+fn tokio_runtime_handle() -> Handle {
+ static RUNTIME: OnceLock<Runtime> = OnceLock::new();
+ RUNTIME
+ .get_or_init(|| Runtime::new().expect("tokio runtime for Ballista Python FFI"))
+ .handle()
+ .clone()
+}
+
+fn ffi_logical_codec_from_python(
+ obj: Bound<'_, PyAny>,
+) -> PyResult<FFI_LogicalExtensionCodec> {
+ let capsule = obj
+ .getattr("__datafusion_logical_extension_codec__")?
+ .call0()?;
+ let capsule = capsule.cast::<PyCapsule>()?;
+ let data: NonNull<FFI_LogicalExtensionCodec> = capsule
+ .pointer_checked(Some(c"datafusion_logical_extension_codec"))?
+ .cast();
+ Ok(unsafe { data.as_ref().clone() })
+}
+
+fn ffi_task_ctx_provider_from_python(
+ obj: Bound<'_, PyAny>,
+) -> PyResult<FFI_TaskContextProvider> {
+ let capsule = obj
+ .getattr("__datafusion_task_context_provider__")?
+ .call0()?;
+ let capsule = capsule.cast::<PyCapsule>()?;
+ let data: NonNull<FFI_TaskContextProvider> = capsule
+ .pointer_checked(Some(c"datafusion_task_context_provider"))?
+ .cast();
+ Ok(unsafe { data.as_ref().clone() })
+}
diff --git a/python/uv.lock b/python/uv.lock
index 81ee8f45b..f467ad317 100644
--- a/python/uv.lock
+++ b/python/uv.lock
@@ -52,7 +52,7 @@ dev = [
[package.metadata]
requires-dist = [
- { name = "datafusion", specifier = "==54" },
+ { name = "datafusion", git = "https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca" },
{ name = "ipython", marker = "extra == 'jupyter'", specifier = ">=8.0.0" },
{ name = "pyarrow", specifier = ">=23.0.1" },
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
@@ -328,25 +328,12 @@ wheels = [
[[package]]
name = "datafusion"
version = "54.0.0"
-source = { registry = "https://pypi.org/simple" }
+source = { git = "https://github.com/apache/datafusion-python.git?rev=55f43b5e231da5591e7bbd632dac85f304a22cca#55f43b5e231da5591e7bbd632dac85f304a22cca" }
dependencies = [
{ name = "cloudpickle" },
{ name = "pyarrow" },
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" },
- { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" },
- { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" },
- { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" },
- { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" },
- { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" },
- { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" },
- { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" },
- { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" },
- { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" },
-]
[[package]]
name = "decorator"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org