You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2022/05/22 14:50:45 UTC

[arrow-datafusion] branch master updated: Build against `arrow-ballista` in CI & remove ballista code from this repo (#2582)

This is an automated email from the ASF dual-hosted git repository.

agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git


The following commit(s) were added to refs/heads/master by this push:
     new 907504c5a Build against `arrow-ballista` in CI & remove ballista code from this repo (#2582)
907504c5a is described below

commit 907504c5aa768601f9d70ad2c8f928bedfa9b069
Author: Andy Grove <ag...@apache.org>
AuthorDate: Sun May 22 08:50:39 2022 -0600

    Build against `arrow-ballista` in CI & remove ballista code from this repo (#2582)
---
 .github/pull_request_template.md                   |    20 +
 .github/workflows/dev.yml                          |     7 +-
 .github/workflows/dev_pr/labeler.yml               |     5 -
 .github/workflows/rust.yml                         |    18 +-
 .gitignore                                         |     3 +
 Cargo.toml                                         |     7 +-
 ballista-cli/Cargo.lock                            |  2557 ----
 ballista-cli/Cargo.toml                            |    41 -
 ballista-cli/Dockerfile                            |    38 -
 ballista-cli/README.md                             |    94 -
 ballista-cli/src/command.rs                        |   222 -
 ballista-cli/src/context.rs                        |    90 -
 ballista-cli/src/exec.rs                           |   170 -
 ballista-cli/src/lib.rs                            |    25 -
 ballista-cli/src/main.rs                           |   165 -
 ballista-examples/Cargo.toml                       |    44 -
 ballista-examples/README.md                        |    55 -
 ballista-examples/examples/test_sql.rs             |    46 -
 ballista-examples/src/bin/ballista-dataframe.rs    |    45 -
 ballista-examples/src/bin/ballista-sql.rs          |    54 -
 ballista/CHANGELOG.md                              |   549 -
 ballista/README.md                                 |    71 -
 ballista/docs/README.md                            |    34 -
 ballista/docs/architecture.md                      |    75 -
 ballista/docs/dev-env.md                           |    51 -
 ballista/docs/images/query-execution.png           |   Bin 11378 -> 0 bytes
 ballista/docs/integration-testing.md               |    29 -
 ballista/rust/.dockerignore                        |    23 -
 ballista/rust/.gitignore                           |     2 -
 ballista/rust/client/Cargo.toml                    |    45 -
 ballista/rust/client/README.md                     |   133 -
 ballista/rust/client/src/columnar_batch.rs         |   163 -
 ballista/rust/client/src/context.rs                |   947 --
 ballista/rust/client/src/lib.rs                    |    22 -
 ballista/rust/client/src/prelude.rs                |    26 -
 ballista/rust/core/Cargo.toml                      |    67 -
 ballista/rust/core/README.md                       |    23 -
 ballista/rust/core/build.rs                        |    30 -
 ballista/rust/core/proto/ballista.proto            |   918 --
 ballista/rust/core/proto/datafusion.proto          |   542 -
 ballista/rust/core/src/client.rs                   |   178 -
 ballista/rust/core/src/config.rs                   |   323 -
 ballista/rust/core/src/error.rs                    |   185 -
 ballista/rust/core/src/event_loop.rs               |   141 -
 .../core/src/execution_plans/distributed_query.rs  |   329 -
 ballista/rust/core/src/execution_plans/mod.rs      |    29 -
 .../core/src/execution_plans/shuffle_reader.rs     |   294 -
 .../core/src/execution_plans/shuffle_writer.rs     |   559 -
 .../core/src/execution_plans/unresolved_shuffle.rs |   129 -
 ballista/rust/core/src/lib.rs                      |    35 -
 ballista/rust/core/src/plugin/mod.rs               |   127 -
 ballista/rust/core/src/plugin/plugin_manager.rs    |   150 -
 ballista/rust/core/src/plugin/udf.rs               |   152 -
 .../rust/core/src/serde/logical_plan/from_proto.rs |    52 -
 ballista/rust/core/src/serde/logical_plan/mod.rs   |  1413 ---
 ballista/rust/core/src/serde/mod.rs                |   745 --
 .../core/src/serde/physical_plan/from_proto.rs     |   396 -
 ballista/rust/core/src/serde/physical_plan/mod.rs  |  1396 ---
 .../rust/core/src/serde/physical_plan/to_proto.rs  |   446 -
 .../rust/core/src/serde/scheduler/from_proto.rs    |   106 -
 ballista/rust/core/src/serde/scheduler/mod.rs      |   423 -
 ballista/rust/core/src/serde/scheduler/to_proto.rs |   107 -
 ballista/rust/core/src/utils.rs                    |   315 -
 ballista/rust/executor/Cargo.toml                  |    63 -
 ballista/rust/executor/README.md                   |    22 -
 ballista/rust/executor/build.rs                    |    24 -
 .../executor/examples/example_executor_config.toml |    22 -
 ballista/rust/executor/executor_config_spec.toml   |   104 -
 ballista/rust/executor/src/collect.rs              |   135 -
 ballista/rust/executor/src/cpu_bound_executor.rs   |   378 -
 ballista/rust/executor/src/execution_loop.rs       |   213 -
 ballista/rust/executor/src/executor.rs             |   119 -
 ballista/rust/executor/src/executor_server.rs      |   353 -
 ballista/rust/executor/src/flight_service.rs       |   245 -
 ballista/rust/executor/src/lib.rs                  |    68 -
 ballista/rust/executor/src/main.rs                 |   295 -
 ballista/rust/executor/src/metrics/mod.rs          |    58 -
 ballista/rust/executor/src/standalone.rs           |    95 -
 ballista/rust/scheduler/Cargo.toml                 |    70 -
 ballista/rust/scheduler/README.md                  |    22 -
 ballista/rust/scheduler/build.rs                   |    29 -
 ballista/rust/scheduler/proto/keda.proto           |    63 -
 ballista/rust/scheduler/scheduler_config_spec.toml |    73 -
 ballista/rust/scheduler/src/api/handlers.rs        |    56 -
 ballista/rust/scheduler/src/api/mod.rs             |    91 -
 ballista/rust/scheduler/src/lib.rs                 |    28 -
 ballista/rust/scheduler/src/main.rs                |   199 -
 ballista/rust/scheduler/src/planner.rs             |   619 -
 .../rust/scheduler/src/scheduler_server/event.rs   |    33 -
 .../scheduler/src/scheduler_server/event_loop.rs   |   169 -
 .../src/scheduler_server/external_scaler.rs        |    65 -
 .../rust/scheduler/src/scheduler_server/grpc.rs    |   634 -
 .../rust/scheduler/src/scheduler_server/mod.rs     |   617 -
 .../src/scheduler_server/query_stage_scheduler.rs  |   476 -
 ballista/rust/scheduler/src/standalone.rs          |    58 -
 ballista/rust/scheduler/src/state/backend/etcd.rs  |   187 -
 ballista/rust/scheduler/src/state/backend/mod.rs   |    94 -
 .../rust/scheduler/src/state/backend/standalone.rs |   216 -
 .../rust/scheduler/src/state/executor_manager.rs   |   145 -
 ballista/rust/scheduler/src/state/mod.rs           |   258 -
 .../rust/scheduler/src/state/persistent_state.rs   |   528 -
 ballista/rust/scheduler/src/state/stage_manager.rs |   783 --
 .../rust/scheduler/src/state/task_scheduler.rs     |   211 -
 ballista/rust/scheduler/src/test_utils.rs          |   138 -
 .../rust/scheduler/testdata/customer/customer.tbl  |    10 -
 .../scheduler/testdata/lineitem/partition0.tbl     |    10 -
 .../scheduler/testdata/lineitem/partition1.tbl     |    10 -
 ballista/rust/scheduler/testdata/nation/nation.tbl |    10 -
 ballista/rust/scheduler/testdata/orders/orders.tbl |    10 -
 ballista/rust/scheduler/testdata/part/part.tbl     |    10 -
 .../rust/scheduler/testdata/partsupp/partsupp.tbl  |    10 -
 ballista/rust/scheduler/testdata/region/region.tbl |     5 -
 .../rust/scheduler/testdata/supplier/supplier.tbl  |    10 -
 ballista/ui/scheduler/.dockerignore                |     2 -
 ballista/ui/scheduler/.gitignore                   |    23 -
 ballista/ui/scheduler/README.md                    |    55 -
 ballista/ui/scheduler/index.d.ts                   |    18 -
 ballista/ui/scheduler/package.json                 |    58 -
 ballista/ui/scheduler/public/favicon.ico           |   Bin 3870 -> 0 bytes
 ballista/ui/scheduler/public/index.html            |    62 -
 ballista/ui/scheduler/public/logo192.png           |   Bin 5347 -> 0 bytes
 ballista/ui/scheduler/public/logo512.png           |   Bin 9664 -> 0 bytes
 ballista/ui/scheduler/public/manifest.json         |    25 -
 ballista/ui/scheduler/react-table-config.d.ts      |   146 -
 ballista/ui/scheduler/src/App.css                  |    18 -
 ballista/ui/scheduler/src/App.test.tsx             |    26 -
 ballista/ui/scheduler/src/App.tsx                  |    97 -
 ballista/ui/scheduler/src/components/DataTable.tsx |   198 -
 ballista/ui/scheduler/src/components/Empty.tsx     |    36 -
 ballista/ui/scheduler/src/components/Footer.tsx    |    34 -
 ballista/ui/scheduler/src/components/Header.tsx    |    95 -
 ballista/ui/scheduler/src/components/NodesList.tsx |    71 -
 .../ui/scheduler/src/components/QueriesList.tsx    |   141 -
 ballista/ui/scheduler/src/components/Summary.tsx   |    92 -
 ballista/ui/scheduler/src/components/logo.svg      |    25 -
 ballista/ui/scheduler/src/index.css                |    32 -
 ballista/ui/scheduler/src/index.tsx                |    38 -
 ballista/ui/scheduler/src/react-app-env.d.ts       |    18 -
 ballista/ui/scheduler/src/reportWebVitals.ts       |    32 -
 ballista/ui/scheduler/src/setupTests.ts            |    22 -
 ballista/ui/scheduler/tsconfig.json                |    28 -
 ballista/ui/scheduler/yarn.lock                    | 12431 -------------------
 benchmarks/Cargo.toml                              |     6 +-
 benchmarks/src/bin/tpch.rs                         |   467 -
 datafusion-cli/Cargo.lock                          |   701 +-
 datafusion-cli/Cargo.toml                          |     2 +-
 datafusion-cli/src/context.rs                      |    44 +-
 datafusion/core/src/sql/planner.rs                 |     9 +-
 .../robots.txt => dev/build-arrow-ballista.sh      |    20 +-
 dev/make-ballista-deps-local.py                    |    52 +
 150 files changed, 121 insertions(+), 38630 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index e71d245c7..2c740dede 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -25,3 +25,23 @@ If there are user-facing changes then we may require documentation to be updated
 <!--
 If there are any breaking changes to public APIs, please add the `api change` label.
 -->
+
+# Does this PR break compatibility with Ballista?
+
+<!--
+The CI checks will attempt to build [arrow-ballista](https://github.com/apache/arrow-ballista) against this PR. If 
+this check fails then it indicates that this PR makes a breaking change to the DataFusion API.
+
+If possible, try to make the change in a way that is not a breaking API change. For example, if code has moved 
+ around, try adding `pub use` from the original location to preserve the current API.
+
+If it is not possible to avoid a breaking change (such as when adding enum variants) then follow this process:
+
+- Make a corresponding PR against `arrow-ballista` with the changes required there
+- Update `dev/build-arrow-ballista.sh` to clone the appropriate `arrow-ballista` repo & branch
+- Merge this PR when CI passes
+- Merge the Ballista PR
+- Create a new PR here to reset `dev/build-arrow-ballista.sh` to point to `arrow-ballista` master again
+
+_If you would like to help improve this process, please see https://github.com/apache/arrow-datafusion/issues/2583_
+-->
\ No newline at end of file
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index f0f17c0a9..060224968 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -66,9 +66,8 @@ jobs:
           #
           # ignore subproject CHANGELOG.md because they are machine generated
           npx prettier@2.3.2 --write \
-            '{ballista,datafusion,datafusion-examples,docs,python}/**/*.md' \
-            '!{ballista,datafusion,python}/CHANGELOG.md' \
+            '{datafusion,datafusion-examples,docs,python}/**/*.md' \
+            '!{datafusion,python}/CHANGELOG.md' \
             README.md \
-            CONTRIBUTING.md \
-            'ballista/**/*.{ts,tsx}'
+            CONTRIBUTING.md
           git diff --exit-code
diff --git a/.github/workflows/dev_pr/labeler.yml b/.github/workflows/dev_pr/labeler.yml
index 5b956a18a..9fdafc640 100644
--- a/.github/workflows/dev_pr/labeler.yml
+++ b/.github/workflows/dev_pr/labeler.yml
@@ -20,10 +20,6 @@ datafusion:
   - datafusion-cli/**/*
   - datafusion-examples/**/*
 
-ballista:
-  - ballista/**/*
-  - ballista-examples/**/*
-
 python:
   - python/**/*
 
@@ -41,5 +37,4 @@ documentation:
   - README.md
   - ./**/README.md
   - DEVELOPERS.md
-  - ballista/docs/**.*
   - datafusion/docs/**.*
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index ce1bc32f8..d2142bd7b 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -121,8 +121,6 @@ jobs:
           cargo run --example csv_sql
           cargo run --example parquet_sql
           cargo run --example avro_sql --features=datafusion/avro
-          cd ../ballista-examples
-          cargo run --example test_sql --features=ballista/standalone
         env:
           CARGO_HOME: "/github/home/.cargo"
           CARGO_TARGET_DIR: "/github/home/target"
@@ -146,6 +144,9 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.x'
       - name: Cache Cargo
         uses: actions/cache@v2
         with:
@@ -162,16 +163,9 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: ${{ matrix.rust }}
-      # Ballista is currently not part of the main workspace so requires a separate test step
-      - name: Run Ballista tests
+      - name: Run tests
         run: |
-          export ARROW_TEST_DATA=$(pwd)/testing/data
-          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
-          cd ballista/rust
-          # snmalloc requires cmake so build without default features
-          cargo test --no-default-features --features sled
-          # Ensure also compiles in standalone mode
-          cargo test --no-default-features --features standalone
+          ./dev/build-arrow-ballista.sh
         env:
           CARGO_HOME: "/github/home/.cargo"
           CARGO_TARGET_DIR: "/github/home/target"
@@ -237,8 +231,6 @@ jobs:
           POSTGRES_PASSWORD: postgres
       - name: Build datafusion-cli
         run: (cd datafusion-cli && cargo build)
-      - name: Build ballista-cli
-        run: (cd ballista-cli && cargo build)
       - name: Test Psql Parity
         run: python -m pytest -v integration-tests/test_psql_parity.py
         env:
diff --git a/.gitignore b/.gitignore
index 1cfeb1f0b..529c590f7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -96,3 +96,6 @@ venv/*
 
 # apache release artifacts
 dev/dist
+
+# CI
+arrow-ballista
diff --git a/Cargo.toml b/Cargo.toml
index ef7dd30c2..813d9e94f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,13 +27,8 @@ members = [
     "datafusion/row",
     "datafusion-examples",
     "benchmarks",
-    "ballista/rust/client",
-    "ballista/rust/core",
-    "ballista/rust/executor",
-    "ballista/rust/scheduler",
-    "ballista-examples",
 ]
-exclude = ["ballista-cli", "datafusion-cli"]
+exclude = ["datafusion-cli"]
 
 [profile.release]
 codegen-units = 1
diff --git a/ballista-cli/Cargo.lock b/ballista-cli/Cargo.lock
deleted file mode 100644
index 528148c9b..000000000
--- a/ballista-cli/Cargo.lock
+++ /dev/null
@@ -1,2557 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
-[[package]]
-name = "ahash"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
-dependencies = [
- "getrandom",
- "once_cell",
- "version_check",
-]
-
-[[package]]
-name = "aho-corasick"
-version = "0.7.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "alloc-no-stdlib"
-version = "2.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3"
-
-[[package]]
-name = "alloc-stdlib"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2"
-dependencies = [
- "alloc-no-stdlib",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.57"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
-
-[[package]]
-name = "arrayref"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
-
-[[package]]
-name = "arrayvec"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
-
-[[package]]
-name = "arrow"
-version = "13.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c6bee230122beb516ead31935a61f683715f987c6f003eff44ad6986624105a"
-dependencies = [
- "bitflags",
- "chrono",
- "comfy-table",
- "csv",
- "flatbuffers",
- "half",
- "hex",
- "indexmap",
- "lazy_static",
- "lexical-core",
- "multiversion",
- "num",
- "rand",
- "regex",
- "serde",
- "serde_derive",
- "serde_json",
-]
-
-[[package]]
-name = "arrow-flight"
-version = "13.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a3666d2dbc637fa979d1f0bf3031d39a80e709f3b9ec88e3d573c1d666bf553"
-dependencies = [
- "arrow",
- "base64",
- "bytes",
- "futures",
- "proc-macro2",
- "prost",
- "prost-derive",
- "tokio",
- "tonic",
- "tonic-build",
-]
-
-[[package]]
-name = "async-stream"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"
-dependencies = [
- "async-stream-impl",
- "futures-core",
-]
-
-[[package]]
-name = "async-stream-impl"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "async-trait"
-version = "0.1.53"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi",
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-
-[[package]]
-name = "axum"
-version = "0.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a"
-dependencies = [
- "async-trait",
- "axum-core",
- "bitflags",
- "bytes",
- "futures-util",
- "http",
- "http-body",
- "hyper",
- "itoa 1.0.1",
- "matchit",
- "memchr",
- "mime",
- "percent-encoding",
- "pin-project-lite",
- "serde",
- "sync_wrapper",
- "tokio",
- "tower",
- "tower-http",
- "tower-layer",
- "tower-service",
-]
-
-[[package]]
-name = "axum-core"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da31c0ed7b4690e2c78fe4b880d21cd7db04a346ebc658b4270251b695437f17"
-dependencies = [
- "async-trait",
- "bytes",
- "futures-util",
- "http",
- "http-body",
- "mime",
-]
-
-[[package]]
-name = "ballista"
-version = "0.6.0"
-dependencies = [
- "ballista-core",
- "datafusion",
- "futures",
- "log",
- "parking_lot",
- "sqlparser",
- "tempfile",
- "tokio",
-]
-
-[[package]]
-name = "ballista-cli"
-version = "0.6.0"
-dependencies = [
- "arrow",
- "ballista",
- "clap",
- "datafusion",
- "datafusion-cli",
- "dirs",
- "env_logger",
- "mimalloc",
- "rustyline",
- "tokio",
-]
-
-[[package]]
-name = "ballista-core"
-version = "0.6.0"
-dependencies = [
- "ahash",
- "arrow-flight",
- "async-trait",
- "chrono",
- "clap",
- "datafusion",
- "datafusion-proto",
- "futures",
- "hashbrown 0.12.1",
- "libloading",
- "log",
- "once_cell",
- "parking_lot",
- "parse_arg",
- "prost",
- "prost-types",
- "rustc_version",
- "serde",
- "sqlparser",
- "tokio",
- "tonic",
- "tonic-build",
- "uuid",
- "walkdir",
-]
-
-[[package]]
-name = "base64"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "blake2"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388"
-dependencies = [
- "digest",
-]
-
-[[package]]
-name = "blake3"
-version = "1.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f"
-dependencies = [
- "arrayref",
- "arrayvec",
- "cc",
- "cfg-if",
- "constant_time_eq",
- "digest",
-]
-
-[[package]]
-name = "block-buffer"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "brotli"
-version = "3.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
- "brotli-decompressor",
-]
-
-[[package]]
-name = "brotli-decompressor"
-version = "2.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
-]
-
-[[package]]
-name = "bstr"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
-dependencies = [
- "lazy_static",
- "memchr",
- "regex-automata",
- "serde",
-]
-
-[[package]]
-name = "byteorder"
-version = "1.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
-
-[[package]]
-name = "bytes"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
-
-[[package]]
-name = "cc"
-version = "1.0.73"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
-dependencies = [
- "jobserver",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "chrono"
-version = "0.4.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
-dependencies = [
- "libc",
- "num-integer",
- "num-traits",
- "winapi",
-]
-
-[[package]]
-name = "clap"
-version = "3.1.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85a35a599b11c089a7f49105658d089b8f2cf0882993c17daf6de15285c2c35d"
-dependencies = [
- "atty",
- "bitflags",
- "clap_derive",
- "clap_lex",
- "indexmap",
- "lazy_static",
- "strsim",
- "termcolor",
- "textwrap",
-]
-
-[[package]]
-name = "clap_derive"
-version = "3.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1"
-dependencies = [
- "heck 0.4.0",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "clap_lex"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
-dependencies = [
- "os_str_bytes",
-]
-
-[[package]]
-name = "clipboard-win"
-version = "4.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f3e1238132dc01f081e1cbb9dace14e5ef4c3a51ee244bd982275fb514605db"
-dependencies = [
- "error-code",
- "str-buf",
- "winapi",
-]
-
-[[package]]
-name = "cmake"
-version = "0.1.48"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "comfy-table"
-version = "5.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b103d85ca6e209388771bfb7aa6b68a7aeec4afbf6f0a0264bfbf50360e5212e"
-dependencies = [
- "strum",
- "strum_macros",
- "unicode-width",
-]
-
-[[package]]
-name = "constant_time_eq"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
-
-[[package]]
-name = "cpufeatures"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "crc32fast"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "crypto-common"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
-[[package]]
-name = "csv"
-version = "1.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
-dependencies = [
- "bstr",
- "csv-core",
- "itoa 0.4.8",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "datafusion"
-version = "7.0.0"
-dependencies = [
- "ahash",
- "arrow",
- "async-trait",
- "chrono",
- "datafusion-common",
- "datafusion-data-access",
- "datafusion-expr",
- "datafusion-physical-expr",
- "datafusion-row",
- "futures",
- "hashbrown 0.12.1",
- "lazy_static",
- "log",
- "num_cpus",
- "ordered-float 3.0.0",
- "parking_lot",
- "parquet",
- "paste",
- "pin-project-lite",
- "rand",
- "smallvec",
- "sqlparser",
- "tempfile",
- "tokio",
- "tokio-stream",
- "uuid",
-]
-
-[[package]]
-name = "datafusion-cli"
-version = "7.0.0"
-dependencies = [
- "arrow",
- "clap",
- "datafusion",
- "dirs",
- "env_logger",
- "mimalloc",
- "rustyline",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-common"
-version = "7.0.0"
-dependencies = [
- "arrow",
- "ordered-float 3.0.0",
- "parquet",
- "sqlparser",
-]
-
-[[package]]
-name = "datafusion-data-access"
-version = "7.0.0"
-dependencies = [
- "async-trait",
- "chrono",
- "futures",
- "glob",
- "parking_lot",
- "tempfile",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-expr"
-version = "7.0.0"
-dependencies = [
- "ahash",
- "arrow",
- "datafusion-common",
- "sqlparser",
-]
-
-[[package]]
-name = "datafusion-physical-expr"
-version = "7.0.0"
-dependencies = [
- "ahash",
- "arrow",
- "blake2",
- "blake3",
- "chrono",
- "datafusion-common",
- "datafusion-expr",
- "datafusion-row",
- "hashbrown 0.12.1",
- "lazy_static",
- "md-5",
- "ordered-float 3.0.0",
- "paste",
- "rand",
- "regex",
- "sha2",
- "unicode-segmentation",
-]
-
-[[package]]
-name = "datafusion-proto"
-version = "7.0.0"
-dependencies = [
- "datafusion",
- "prost",
- "tonic-build",
-]
-
-[[package]]
-name = "datafusion-row"
-version = "7.0.0"
-dependencies = [
- "arrow",
- "datafusion-common",
- "paste",
- "rand",
-]
-
-[[package]]
-name = "digest"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
-dependencies = [
- "block-buffer",
- "crypto-common",
- "subtle",
-]
-
-[[package]]
-name = "dirs"
-version = "4.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
-dependencies = [
- "dirs-sys",
-]
-
-[[package]]
-name = "dirs-next"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
-dependencies = [
- "cfg-if",
- "dirs-sys-next",
-]
-
-[[package]]
-name = "dirs-sys"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
-[[package]]
-name = "dirs-sys-next"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
-[[package]]
-name = "either"
-version = "1.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
-
-[[package]]
-name = "endian-type"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
-
-[[package]]
-name = "env_logger"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3"
-dependencies = [
- "atty",
- "humantime",
- "log",
- "regex",
- "termcolor",
-]
-
-[[package]]
-name = "errno"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
-dependencies = [
- "errno-dragonfly",
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
-dependencies = [
- "cc",
- "libc",
-]
-
-[[package]]
-name = "error-code"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21"
-dependencies = [
- "libc",
- "str-buf",
-]
-
-[[package]]
-name = "fastrand"
-version = "1.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf"
-dependencies = [
- "instant",
-]
-
-[[package]]
-name = "fd-lock"
-version = "3.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46e245f4c8ec30c6415c56cb132c07e69e74f1942f6b4a4061da748b49f486ca"
-dependencies = [
- "cfg-if",
- "rustix",
- "windows-sys 0.30.0",
-]
-
-[[package]]
-name = "fixedbitset"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"
-
-[[package]]
-name = "flatbuffers"
-version = "2.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86b428b715fdbdd1c364b84573b5fdc0f84f8e423661b9f398735278bc7f2b6a"
-dependencies = [
- "bitflags",
- "smallvec",
- "thiserror",
-]
-
-[[package]]
-name = "flate2"
-version = "1.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af"
-dependencies = [
- "cfg-if",
- "crc32fast",
- "libc",
- "miniz_oxide",
-]
-
-[[package]]
-name = "fnv"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
-
-[[package]]
-name = "futures"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e"
-dependencies = [
- "futures-channel",
- "futures-core",
- "futures-executor",
- "futures-io",
- "futures-sink",
- "futures-task",
- "futures-util",
-]
-
-[[package]]
-name = "futures-channel"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010"
-dependencies = [
- "futures-core",
- "futures-sink",
-]
-
-[[package]]
-name = "futures-core"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
-
-[[package]]
-name = "futures-executor"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6"
-dependencies = [
- "futures-core",
- "futures-task",
- "futures-util",
-]
-
-[[package]]
-name = "futures-io"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
-
-[[package]]
-name = "futures-macro"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "futures-sink"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868"
-
-[[package]]
-name = "futures-task"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a"
-
-[[package]]
-name = "futures-util"
-version = "0.3.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a"
-dependencies = [
- "futures-channel",
- "futures-core",
- "futures-io",
- "futures-macro",
- "futures-sink",
- "futures-task",
- "memchr",
- "pin-project-lite",
- "pin-utils",
- "slab",
-]
-
-[[package]]
-name = "generic-array"
-version = "0.14.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
-dependencies = [
- "typenum",
- "version_check",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.10.2+wasi-snapshot-preview1",
-]
-
-[[package]]
-name = "glob"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
-
-[[package]]
-name = "h2"
-version = "0.3.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57"
-dependencies = [
- "bytes",
- "fnv",
- "futures-core",
- "futures-sink",
- "futures-util",
- "http",
- "indexmap",
- "slab",
- "tokio",
- "tokio-util",
- "tracing",
-]
-
-[[package]]
-name = "half"
-version = "1.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
-
-[[package]]
-name = "hashbrown"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
-
-[[package]]
-name = "hashbrown"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3"
-dependencies = [
- "ahash",
-]
-
-[[package]]
-name = "heck"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
-dependencies = [
- "unicode-segmentation",
-]
-
-[[package]]
-name = "heck"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
-[[package]]
-name = "http"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb"
-dependencies = [
- "bytes",
- "fnv",
- "itoa 1.0.1",
-]
-
-[[package]]
-name = "http-body"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
-dependencies = [
- "bytes",
- "http",
- "pin-project-lite",
-]
-
-[[package]]
-name = "http-range-header"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"
-
-[[package]]
-name = "httparse"
-version = "1.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c"
-
-[[package]]
-name = "httpdate"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
-
-[[package]]
-name = "humantime"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
-
-[[package]]
-name = "hyper"
-version = "0.14.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2"
-dependencies = [
- "bytes",
- "futures-channel",
- "futures-core",
- "futures-util",
- "h2",
- "http",
- "http-body",
- "httparse",
- "httpdate",
- "itoa 1.0.1",
- "pin-project-lite",
- "socket2",
- "tokio",
- "tower-service",
- "tracing",
- "want",
-]
-
-[[package]]
-name = "hyper-timeout"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
-dependencies = [
- "hyper",
- "pin-project-lite",
- "tokio",
- "tokio-io-timeout",
-]
-
-[[package]]
-name = "indexmap"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
-dependencies = [
- "autocfg",
- "hashbrown 0.11.2",
-]
-
-[[package]]
-name = "instant"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "integer-encoding"
-version = "1.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f"
-
-[[package]]
-name = "io-lifetimes"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9448015e586b611e5d322f6703812bbca2f1e709d5773ecd38ddb4e3bb649504"
-
-[[package]]
-name = "itertools"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itoa"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
-
-[[package]]
-name = "itoa"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
-
-[[package]]
-name = "jobserver"
-version = "0.1.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
-[[package]]
-name = "lexical-core"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92912c4af2e7d9075be3e5e3122c4d7263855fa6cce34fbece4dd08e5884624d"
-dependencies = [
- "lexical-parse-float",
- "lexical-parse-integer",
- "lexical-util",
- "lexical-write-float",
- "lexical-write-integer",
-]
-
-[[package]]
-name = "lexical-parse-float"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f518eed87c3be6debe6d26b855c97358d8a11bf05acec137e5f53080f5ad2dd8"
-dependencies = [
- "lexical-parse-integer",
- "lexical-util",
- "static_assertions",
-]
-
-[[package]]
-name = "lexical-parse-integer"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afc852ec67c6538bbb2b9911116a385b24510e879a69ab516e6a151b15a79168"
-dependencies = [
- "lexical-util",
- "static_assertions",
-]
-
-[[package]]
-name = "lexical-util"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c72a9d52c5c4e62fa2cdc2cb6c694a39ae1382d9c2a17a466f18e272a0930eb1"
-dependencies = [
- "static_assertions",
-]
-
-[[package]]
-name = "lexical-write-float"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a89ec1d062e481210c309b672f73a0567b7855f21e7d2fae636df44d12e97f9"
-dependencies = [
- "lexical-util",
- "lexical-write-integer",
- "static_assertions",
-]
-
-[[package]]
-name = "lexical-write-integer"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "094060bd2a7c2ff3a16d5304a6ae82727cb3cc9d1c70f813cc73f744c319337e"
-dependencies = [
- "lexical-util",
- "static_assertions",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.125"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b"
-
-[[package]]
-name = "libloading"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd"
-dependencies = [
- "cfg-if",
- "winapi",
-]
-
-[[package]]
-name = "libmimalloc-sys"
-version = "0.1.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11ca136052550448f55df7898c6dbe651c6b574fe38a0d9ea687a9f8088a2e2c"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.0.46"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"
-
-[[package]]
-name = "lock_api"
-version = "0.4.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
-dependencies = [
- "autocfg",
- "scopeguard",
-]
-
-[[package]]
-name = "log"
-version = "0.4.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "lz4"
-version = "1.23.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4edcb94251b1c375c459e5abe9fb0168c1c826c3370172684844f8f3f8d1a885"
-dependencies = [
- "libc",
- "lz4-sys",
-]
-
-[[package]]
-name = "lz4-sys"
-version = "1.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7be8908e2ed6f31c02db8a9fa962f03e36c53fbfde437363eae3306b85d7e17"
-dependencies = [
- "cc",
- "libc",
-]
-
-[[package]]
-name = "matchit"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
-
-[[package]]
-name = "md-5"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582"
-dependencies = [
- "digest",
-]
-
-[[package]]
-name = "memchr"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-
-[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "mimalloc"
-version = "0.1.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f64ad83c969af2e732e907564deb0d0ed393cec4af80776f77dd77a1a427698"
-dependencies = [
- "libmimalloc-sys",
-]
-
-[[package]]
-name = "mime"
-version = "0.3.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
-
-[[package]]
-name = "miniz_oxide"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082"
-dependencies = [
- "adler",
-]
-
-[[package]]
-name = "mio"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9"
-dependencies = [
- "libc",
- "log",
- "miow",
- "ntapi",
- "wasi 0.11.0+wasi-snapshot-preview1",
- "winapi",
-]
-
-[[package]]
-name = "miow"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "multimap"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
-
-[[package]]
-name = "multiversion"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373"
-dependencies = [
- "multiversion-macros",
-]
-
-[[package]]
-name = "multiversion-macros"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "nibble_vec"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
-dependencies = [
- "smallvec",
-]
-
-[[package]]
-name = "nix"
-version = "0.23.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6"
-dependencies = [
- "bitflags",
- "cc",
- "cfg-if",
- "libc",
- "memoffset",
-]
-
-[[package]]
-name = "ntapi"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "num"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606"
-dependencies = [
- "num-bigint",
- "num-complex",
- "num-integer",
- "num-iter",
- "num-rational",
- "num-traits",
-]
-
-[[package]]
-name = "num-bigint"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-complex"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97fbc387afefefd5e9e39493299f3069e14a140dd34dc19b4c1c1a8fddb6a790"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "num-integer"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
-dependencies = [
- "autocfg",
- "num-traits",
-]
-
-[[package]]
-name = "num-iter"
-version = "0.1.43"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-rational"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
-dependencies = [
- "autocfg",
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "num_cpus"
-version = "1.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
-dependencies = [
- "hermit-abi",
- "libc",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
-
-[[package]]
-name = "ordered-float"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "ordered-float"
-version = "3.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96bcbab4bfea7a59c2c0fe47211a1ac4e3e96bea6eb446d704f310bc5c732ae2"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
-name = "os_str_bytes"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
-
-[[package]]
-name = "parking_lot"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-sys 0.36.1",
-]
-
-[[package]]
-name = "parquet"
-version = "13.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c6d737baed48775e87a69aa262f1fa2f1d6bd074dedbe9cac244b9aabf2a0b4"
-dependencies = [
- "arrow",
- "base64",
- "brotli",
- "byteorder",
- "chrono",
- "flate2",
- "lz4",
- "num",
- "num-bigint",
- "parquet-format",
- "rand",
- "snap",
- "thrift",
- "zstd",
-]
-
-[[package]]
-name = "parquet-format"
-version = "4.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f0c06cdcd5460967c485f9c40a821746f5955ad81990533c7fae95dbd9bc0b5"
-dependencies = [
- "thrift",
-]
-
-[[package]]
-name = "parse_arg"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14248cc8eced350e20122a291613de29e4fa129ba2731818c4cdbb44fccd3e55"
-
-[[package]]
-name = "paste"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
-
-[[package]]
-name = "percent-encoding"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
-
-[[package]]
-name = "petgraph"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f"
-dependencies = [
- "fixedbitset",
- "indexmap",
-]
-
-[[package]]
-name = "pin-project"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e"
-dependencies = [
- "pin-project-internal",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pin-project-lite"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
-
-[[package]]
-name = "pin-utils"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
-
-[[package]]
-name = "prettyplease"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803"
-dependencies = [
- "proc-macro2",
- "syn",
-]
-
-[[package]]
-name = "proc-macro-error"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2",
- "quote",
- "syn",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
-dependencies = [
- "proc-macro2",
- "quote",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
-dependencies = [
- "unicode-xid",
-]
-
-[[package]]
-name = "prost"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a07b0857a71a8cb765763950499cae2413c3f9cede1133478c43600d9e146890"
-dependencies = [
- "bytes",
- "prost-derive",
-]
-
-[[package]]
-name = "prost-build"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "120fbe7988713f39d780a58cf1a7ef0d7ef66c6d87e5aa3438940c05357929f4"
-dependencies = [
- "bytes",
- "cfg-if",
- "cmake",
- "heck 0.4.0",
- "itertools",
- "lazy_static",
- "log",
- "multimap",
- "petgraph",
- "prost",
- "prost-types",
- "regex",
- "tempfile",
- "which",
-]
-
-[[package]]
-name = "prost-derive"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc"
-dependencies = [
- "anyhow",
- "itertools",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "prost-types"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68"
-dependencies = [
- "bytes",
- "prost",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "radix_trie"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
-dependencies = [
- "endian-type",
- "nibble_vec",
-]
-
-[[package]]
-name = "rand"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
-dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.2.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "redox_users"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
-dependencies = [
- "getrandom",
- "redox_syscall",
- "thiserror",
-]
-
-[[package]]
-name = "regex"
-version = "1.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
-
-[[package]]
-name = "regex-syntax"
-version = "0.6.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
-
-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "rustc_version"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
-dependencies = [
- "semver",
-]
-
-[[package]]
-name = "rustix"
-version = "0.34.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3e74b3f02f2b6eb33790923756784614f456de79d821d6b2670dc7d5fbea807"
-dependencies = [
- "bitflags",
- "errno",
- "io-lifetimes",
- "libc",
- "linux-raw-sys",
- "winapi",
-]
-
-[[package]]
-name = "rustversion"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f"
-
-[[package]]
-name = "rustyline"
-version = "9.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db7826789c0e25614b03e5a54a0717a86f9ff6e6e5247f92b369472869320039"
-dependencies = [
- "bitflags",
- "cfg-if",
- "clipboard-win",
- "dirs-next",
- "fd-lock",
- "libc",
- "log",
- "memchr",
- "nix",
- "radix_trie",
- "scopeguard",
- "smallvec",
- "unicode-segmentation",
- "unicode-width",
- "utf8parse",
- "winapi",
-]
-
-[[package]]
-name = "ryu"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
-
-[[package]]
-name = "same-file"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
-dependencies = [
- "winapi-util",
-]
-
-[[package]]
-name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
-[[package]]
-name = "semver"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd"
-
-[[package]]
-name = "serde"
-version = "1.0.137"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.137"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.80"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f972498cf015f7c0746cac89ebe1d6ef10c293b94175a243a2d9442c163d9944"
-dependencies = [
- "indexmap",
- "itoa 1.0.1",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "sha2"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
-
-[[package]]
-name = "slab"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
-
-[[package]]
-name = "smallvec"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
-
-[[package]]
-name = "snap"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451"
-
-[[package]]
-name = "socket2"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
-dependencies = [
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "sqlparser"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddc2739f3a9bfc68e2f7b7695589f6cb0181c88af73ceaee0c84215cd2a2ae28"
-dependencies = [
- "log",
-]
-
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
-[[package]]
-name = "str-buf"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a"
-
-[[package]]
-name = "strsim"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
-
-[[package]]
-name = "strum"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"
-
-[[package]]
-name = "strum_macros"
-version = "0.23.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"
-dependencies = [
- "heck 0.3.3",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn",
-]
-
-[[package]]
-name = "subtle"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
-
-[[package]]
-name = "syn"
-version = "1.0.92"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-xid",
-]
-
-[[package]]
-name = "sync_wrapper"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
-
-[[package]]
-name = "tempfile"
-version = "3.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
-dependencies = [
- "cfg-if",
- "fastrand",
- "libc",
- "redox_syscall",
- "remove_dir_all",
- "winapi",
-]
-
-[[package]]
-name = "termcolor"
-version = "1.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
-dependencies = [
- "winapi-util",
-]
-
-[[package]]
-name = "textwrap"
-version = "0.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
-
-[[package]]
-name = "thiserror"
-version = "1.0.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "threadpool"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
-dependencies = [
- "num_cpus",
-]
-
-[[package]]
-name = "thrift"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6d965454947cc7266d22716ebfd07b18d84ebaf35eec558586bbb2a8cb6b5b"
-dependencies = [
- "byteorder",
- "integer-encoding",
- "log",
- "ordered-float 1.1.1",
- "threadpool",
-]
-
-[[package]]
-name = "tokio"
-version = "1.18.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc"
-dependencies = [
- "bytes",
- "libc",
- "memchr",
- "mio",
- "num_cpus",
- "once_cell",
- "parking_lot",
- "pin-project-lite",
- "socket2",
- "tokio-macros",
- "winapi",
-]
-
-[[package]]
-name = "tokio-io-timeout"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
-dependencies = [
- "pin-project-lite",
- "tokio",
-]
-
-[[package]]
-name = "tokio-macros"
-version = "1.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "tokio-stream"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"
-dependencies = [
- "futures-core",
- "pin-project-lite",
- "tokio",
-]
-
-[[package]]
-name = "tokio-util"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764"
-dependencies = [
- "bytes",
- "futures-core",
- "futures-sink",
- "pin-project-lite",
- "tokio",
- "tracing",
-]
-
-[[package]]
-name = "tonic"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30fb54bf1e446f44d870d260d99957e7d11fb9d0a0f5bd1a662ad1411cc103f9"
-dependencies = [
- "async-stream",
- "async-trait",
- "axum",
- "base64",
- "bytes",
- "futures-core",
- "futures-util",
- "h2",
- "http",
- "http-body",
- "hyper",
- "hyper-timeout",
- "percent-encoding",
- "pin-project",
- "prost",
- "prost-derive",
- "tokio",
- "tokio-stream",
- "tokio-util",
- "tower",
- "tower-layer",
- "tower-service",
- "tracing",
- "tracing-futures",
-]
-
-[[package]]
-name = "tonic-build"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03447cdc9eaf8feffb6412dcb27baf2db11669a6c4789f29da799aabfb99547"
-dependencies = [
- "prettyplease",
- "proc-macro2",
- "prost-build",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "tower"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e"
-dependencies = [
- "futures-core",
- "futures-util",
- "indexmap",
- "pin-project",
- "pin-project-lite",
- "rand",
- "slab",
- "tokio",
- "tokio-util",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "tower-http"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e980386f06883cf4d0578d6c9178c81f68b45d77d00f2c2c1bc034b3439c2c56"
-dependencies = [
- "bitflags",
- "bytes",
- "futures-core",
- "futures-util",
- "http",
- "http-body",
- "http-range-header",
- "pin-project-lite",
- "tower",
- "tower-layer",
- "tower-service",
-]
-
-[[package]]
-name = "tower-layer"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62"
-
-[[package]]
-name = "tower-service"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
-
-[[package]]
-name = "tracing"
-version = "0.1.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09"
-dependencies = [
- "cfg-if",
- "log",
- "pin-project-lite",
- "tracing-attributes",
- "tracing-core",
-]
-
-[[package]]
-name = "tracing-attributes"
-version = "0.1.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "tracing-core"
-version = "0.1.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f"
-dependencies = [
- "lazy_static",
-]
-
-[[package]]
-name = "tracing-futures"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
-dependencies = [
- "pin-project",
- "tracing",
-]
-
-[[package]]
-name = "try-lock"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
-
-[[package]]
-name = "typenum"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
-
-[[package]]
-name = "unicode-segmentation"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
-
-[[package]]
-name = "unicode-width"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
-
-[[package]]
-name = "unicode-xid"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
-
-[[package]]
-name = "utf8parse"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
-
-[[package]]
-name = "uuid"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "version_check"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
-
-[[package]]
-name = "walkdir"
-version = "2.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
-dependencies = [
- "same-file",
- "winapi",
- "winapi-util",
-]
-
-[[package]]
-name = "want"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
-dependencies = [
- "log",
- "try-lock",
-]
-
-[[package]]
-name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
-[[package]]
-name = "which"
-version = "4.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
-dependencies = [
- "either",
- "lazy_static",
- "libc",
-]
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-util"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "windows-sys"
-version = "0.30.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "030b7ff91626e57a05ca64a07c481973cbb2db774e4852c9c7ca342408c6a99a"
-dependencies = [
- "windows_aarch64_msvc 0.30.0",
- "windows_i686_gnu 0.30.0",
- "windows_i686_msvc 0.30.0",
- "windows_x86_64_gnu 0.30.0",
- "windows_x86_64_msvc 0.30.0",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
-dependencies = [
- "windows_aarch64_msvc 0.36.1",
- "windows_i686_gnu 0.36.1",
- "windows_i686_msvc 0.36.1",
- "windows_x86_64_gnu 0.36.1",
- "windows_x86_64_msvc 0.36.1",
-]
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.30.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29277a4435d642f775f63c7d1faeb927adba532886ce0287bd985bffb16b6bca"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.30.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1145e1989da93956c68d1864f32fb97c8f561a8f89a5125f6a2b7ea75524e4b8"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.30.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4a09e3a0d4753b73019db171c1339cd4362c8c44baf1bcea336235e955954a6"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.30.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ca64fcb0220d58db4c119e050e7af03c69e6f4f415ef69ec1773d9aab422d5a"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.30.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08cabc9f0066848fef4bc6a1c1668e6efce38b661d2aeec75d18d8617eebb5f1"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
-
-[[package]]
-name = "zstd"
-version = "0.11.1+zstd.1.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a16b8414fde0414e90c612eba70985577451c4c504b99885ebed24762cb81a"
-dependencies = [
- "zstd-safe",
-]
-
-[[package]]
-name = "zstd-safe"
-version = "5.0.1+zstd.1.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c12659121420dd6365c5c3de4901f97145b79651fb1d25814020ed2ed0585ae"
-dependencies = [
- "libc",
- "zstd-sys",
-]
-
-[[package]]
-name = "zstd-sys"
-version = "2.0.1+zstd.1.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
-dependencies = [
- "cc",
- "libc",
-]
diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml
deleted file mode 100644
index 3638a6227..000000000
--- a/ballista-cli/Cargo.toml
+++ /dev/null
@@ -1,41 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "ballista-cli"
-description = "Command Line Client for Ballista distributed query engine."
-version = "0.7.0"
-authors = ["Apache Arrow <de...@arrow.apache.org>"]
-edition = "2021"
-keywords = [ "ballista", "cli", ]
-license = "Apache-2.0"
-homepage = "https://github.com/apache/arrow-datafusion"
-repository = "https://github.com/apache/arrow-datafusion"
-rust-version = "1.59"
-readme = "README.md"
-
-[dependencies]
-arrow = { version = "14.0.0" }
-ballista = { path = "../ballista/rust/client", version = "0.7.0" }
-clap = { version = "3", features = ["derive", "cargo"] }
-datafusion = { path = "../datafusion/core", version = "8.0.0" }
-datafusion-cli = { path = "../datafusion-cli", version = "8.0.0" }
-dirs = "4.0.0"
-env_logger = "0.9"
-mimalloc = { version = "0.1", default-features = false }
-rustyline = "9.0"
-tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
diff --git a/ballista-cli/Dockerfile b/ballista-cli/Dockerfile
deleted file mode 100644
index 7a35d185b..000000000
--- a/ballista-cli/Dockerfile
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-FROM rust:1.59 as builder
-
-COPY ./datafusion /usr/src/datafusion
-
-COPY ./ballista-cli /usr/src/ballista-cli
-
-COPY ./ballista /usr/src/ballista
-
-WORKDIR /usr/src/ballista-cli
-
-RUN rustup component add rustfmt
-
-RUN cargo build --release
-
-FROM debian:bullseye-slim
-
-COPY --from=builder /usr/src/ballista-cli/target/release/ballista-cli /usr/local/bin
-
-ENTRYPOINT ["ballista-cli"]
-
-CMD ["--data-path", "/data"]
diff --git a/ballista-cli/README.md b/ballista-cli/README.md
deleted file mode 100644
index 733329e0c..000000000
--- a/ballista-cli/README.md
+++ /dev/null
@@ -1,94 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Ballista Command-line Interface
-
-[Ballista](df) is a distributed query execution framework, written in Rust, that uses Apache Arrow as its in-memory format.
-
-The Ballista CLI allows SQL queries to be executed by an in-process DataFusion context, or by a distributed
-Ballista context.
-
-```ignore
-USAGE:
-    ballista-cli [FLAGS] [OPTIONS]
-
-FLAGS:
-    -h, --help       Prints help information
-    -q, --quiet      Reduce printing other than the results and work quietly
-    -V, --version    Prints version information
-
-OPTIONS:
-    -c, --batch-size <batch-size>    The batch size of each query, or use DataFusion default
-    -p, --data-path <data-path>      Path to your data, default to current directory
-    -f, --file <file>...             Execute commands from file(s), then exit
-        --format <format>            Output format [default: table]  [possible values: csv, tsv, table, json, ndjson]
-        --host <host>                Ballista scheduler host
-        --port <port>                Ballista scheduler port
-```
-
-## Example
-
-Create a CSV file to query.
-
-```bash,ignore
-$ echo "1,2" > data.csv
-```
-
-```sql,ignore
-$ ballista-cli
-
-Ballista CLI v0.6.0
-
-> CREATE EXTERNAL TABLE foo (a INT, b INT) STORED AS CSV LOCATION 'data.csv';
-0 rows in set. Query took 0.001 seconds.
-
-> SELECT * FROM foo;
-+---+---+
-| a | b |
-+---+---+
-| 1 | 2 |
-+---+---+
-1 row in set. Query took 0.017 seconds.
-```
-
-## Ballista-Cli
-
-Build the `ballista-cli` without the feature of ballista.
-
-```bash
-cd arrow-datafusion/ballista-cli
-cargo build
-```
-
-## Ballista
-
-If you want to execute the SQL in ballista by `ballista-cli`, you must build/compile the `ballista-cli` with features of "ballista" first.
-
-```bash
-cd arrow-datafusion/ballista-cli
-cargo build
-```
-
-The Ballista CLI can connect to a Ballista scheduler for query execution.
-
-```bash
-ballista-cli --host localhost --port 50050
-```
-
-[df]: https://crates.io/crates/datafusion
diff --git a/ballista-cli/src/command.rs b/ballista-cli/src/command.rs
deleted file mode 100644
index 0fd43a307..000000000
--- a/ballista-cli/src/command.rs
+++ /dev/null
@@ -1,222 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Command within CLI
-
-use crate::context::Context;
-use crate::functions::{display_all_functions, Function};
-use crate::print_format::PrintFormat;
-use crate::print_options::PrintOptions;
-use clap::ArgEnum;
-use datafusion::arrow::array::{ArrayRef, StringArray};
-use datafusion::arrow::datatypes::{DataType, Field, Schema};
-use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::error::{DataFusionError, Result};
-use std::str::FromStr;
-use std::sync::Arc;
-use std::time::Instant;
-
-/// Command
-#[derive(Debug)]
-pub enum Command {
-    Quit,
-    Help,
-    ListTables,
-    DescribeTable(String),
-    ListFunctions,
-    SearchFunctions(String),
-    QuietMode(Option<bool>),
-    OutputFormat(Option<String>),
-}
-
-pub enum OutputFormat {
-    ChangeFormat(String),
-}
-
-impl Command {
-    pub async fn execute(
-        &self,
-        ctx: &mut Context,
-        print_options: &mut PrintOptions,
-    ) -> Result<()> {
-        let now = Instant::now();
-        match self {
-            Self::Help => print_options
-                .print_batches(&[all_commands_info()], now)
-                .map_err(|e| DataFusionError::Execution(e.to_string())),
-            Self::ListTables => {
-                let df = ctx.sql("SHOW TABLES").await?;
-                let batches = df.collect().await?;
-                print_options
-                    .print_batches(&batches, now)
-                    .map_err(|e| DataFusionError::Execution(e.to_string()))
-            }
-            Self::DescribeTable(name) => {
-                let df = ctx.sql(&format!("SHOW COLUMNS FROM {}", name)).await?;
-                let batches = df.collect().await?;
-                print_options
-                    .print_batches(&batches, now)
-                    .map_err(|e| DataFusionError::Execution(e.to_string()))
-            }
-            Self::QuietMode(quiet) => {
-                if let Some(quiet) = quiet {
-                    print_options.quiet = *quiet;
-                    println!(
-                        "Quiet mode set to {}",
-                        if print_options.quiet { "true" } else { "false" }
-                    );
-                } else {
-                    println!(
-                        "Quiet mode is {}",
-                        if print_options.quiet { "true" } else { "false" }
-                    );
-                }
-                Ok(())
-            }
-            Self::Quit => Err(DataFusionError::Execution(
-                "Unexpected quit, this should be handled outside".into(),
-            )),
-            Self::ListFunctions => display_all_functions(),
-            Self::SearchFunctions(function) => {
-                if let Ok(func) = function.parse::<Function>() {
-                    let details = func.function_details()?;
-                    println!("{}", details);
-                    Ok(())
-                } else {
-                    let msg = format!("{} is not a supported function", function);
-                    Err(DataFusionError::Execution(msg))
-                }
-            }
-            Self::OutputFormat(_) => Err(DataFusionError::Execution(
-                "Unexpected change output format, this should be handled outside".into(),
-            )),
-        }
-    }
-
-    fn get_name_and_description(&self) -> (&'static str, &'static str) {
-        match self {
-            Self::Quit => ("\\q", "quit datafusion-cli"),
-            Self::ListTables => ("\\d", "list tables"),
-            Self::DescribeTable(_) => ("\\d name", "describe table"),
-            Self::Help => ("\\?", "help"),
-            Self::ListFunctions => ("\\h", "function list"),
-            Self::SearchFunctions(_) => ("\\h function", "search function"),
-            Self::QuietMode(_) => ("\\quiet (true|false)?", "print or set quiet mode"),
-            Self::OutputFormat(_) => {
-                ("\\pset [NAME [VALUE]]", "set table output option\n(format)")
-            }
-        }
-    }
-}
-
-const ALL_COMMANDS: [Command; 8] = [
-    Command::ListTables,
-    Command::DescribeTable(String::new()),
-    Command::Quit,
-    Command::Help,
-    Command::ListFunctions,
-    Command::SearchFunctions(String::new()),
-    Command::QuietMode(None),
-    Command::OutputFormat(None),
-];
-
-fn all_commands_info() -> RecordBatch {
-    let schema = Arc::new(Schema::new(vec![
-        Field::new("Command", DataType::Utf8, false),
-        Field::new("Description", DataType::Utf8, false),
-    ]));
-    let (names, description): (Vec<&str>, Vec<&str>) = ALL_COMMANDS
-        .into_iter()
-        .map(|c| c.get_name_and_description())
-        .unzip();
-    RecordBatch::try_new(
-        schema,
-        [names, description]
-            .into_iter()
-            .map(|i| Arc::new(StringArray::from(i)) as ArrayRef)
-            .collect::<Vec<_>>(),
-    )
-    .expect("This should not fail")
-}
-
-impl FromStr for Command {
-    type Err = ();
-
-    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
-        let (c, arg) = if let Some((a, b)) = s.split_once(' ') {
-            (a, Some(b))
-        } else {
-            (s, None)
-        };
-        Ok(match (c, arg) {
-            ("q", None) => Self::Quit,
-            ("d", None) => Self::ListTables,
-            ("d", Some(name)) => Self::DescribeTable(name.into()),
-            ("?", None) => Self::Help,
-            ("h", None) => Self::ListFunctions,
-            ("h", Some(function)) => Self::SearchFunctions(function.into()),
-            ("quiet", Some("true" | "t" | "yes" | "y" | "on")) => {
-                Self::QuietMode(Some(true))
-            }
-            ("quiet", Some("false" | "f" | "no" | "n" | "off")) => {
-                Self::QuietMode(Some(false))
-            }
-            ("quiet", None) => Self::QuietMode(None),
-            ("pset", Some(subcommand)) => {
-                Self::OutputFormat(Some(subcommand.to_string()))
-            }
-            ("pset", None) => Self::OutputFormat(None),
-            _ => return Err(()),
-        })
-    }
-}
-
-impl FromStr for OutputFormat {
-    type Err = ();
-
-    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
-        let (c, arg) = if let Some((a, b)) = s.split_once(' ') {
-            (a, Some(b))
-        } else {
-            (s, None)
-        };
-        Ok(match (c, arg) {
-            ("format", Some(format)) => Self::ChangeFormat(format.to_string()),
-            _ => return Err(()),
-        })
-    }
-}
-
-impl OutputFormat {
-    pub async fn execute(&self, print_options: &mut PrintOptions) -> Result<()> {
-        match self {
-            Self::ChangeFormat(format) => {
-                if let Ok(format) = format.parse::<PrintFormat>() {
-                    print_options.format = format;
-                    println!("Output format is {:?}.", print_options.format);
-                    Ok(())
-                } else {
-                    Err(DataFusionError::Execution(format!(
-                        "{:?} is not a valid format type [possible values: {:?}]",
-                        format,
-                        PrintFormat::value_variants()
-                    )))
-                }
-            }
-        }
-    }
-}
diff --git a/ballista-cli/src/context.rs b/ballista-cli/src/context.rs
deleted file mode 100644
index c96b0e766..000000000
--- a/ballista-cli/src/context.rs
+++ /dev/null
@@ -1,90 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Context (remote or local)
-
-use datafusion::dataframe::DataFrame;
-use datafusion::error::{DataFusionError, Result};
-use datafusion::execution::context::{SessionConfig, SessionContext};
-use std::sync::Arc;
-
-/// The CLI supports using a local DataFusion context or a distributed BallistaContext
-pub enum Context {
-    /// In-process execution with DataFusion
-    Local(SessionContext),
-    /// Distributed execution with Ballista (if available)
-    Remote(BallistaContext),
-}
-
-impl Context {
-    /// create a new remote context with given host and port
-    pub async fn new_remote(host: &str, port: u16) -> Result<Context> {
-        Ok(Context::Remote(BallistaContext::try_new(host, port).await?))
-    }
-
-    /// create a local context using the given config
-    pub fn new_local(config: &SessionConfig) -> Context {
-        Context::Local(SessionContext::with_config(config.clone()))
-    }
-
-    /// execute an SQL statement against the context
-    pub async fn sql(&mut self, sql: &str) -> Result<Arc<DataFrame>> {
-        match self {
-            Context::Local(datafusion) => datafusion.sql(sql).await,
-            Context::Remote(ballista) => ballista.sql(sql).await,
-        }
-    }
-}
-
-// implement wrappers around the BallistaContext to support running without ballista
-
-#[cfg(feature = "ballista")]
-pub struct BallistaContext(ballista::context::BallistaContext);
-#[cfg(feature = "ballista")]
-impl BallistaContext {
-    pub async fn try_new(host: &str, port: u16) -> Result<Self> {
-        use ballista::context::BallistaContext;
-        use ballista::prelude::BallistaConfig;
-        let builder =
-            BallistaConfig::builder().set("ballista.with_information_schema", "true");
-        let config = builder
-            .build()
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-        let remote_ctx = BallistaContext::remote(host, port, &config)
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-        Ok(Self(remote_ctx))
-    }
-    pub async fn sql(&mut self, sql: &str) -> Result<Arc<DataFrame>> {
-        self.0.sql(sql).await
-    }
-}
-
-#[cfg(not(feature = "ballista"))]
-pub struct BallistaContext();
-#[cfg(not(feature = "ballista"))]
-impl BallistaContext {
-    pub async fn try_new(_host: &str, _port: u16) -> Result<Self> {
-        Err(DataFusionError::NotImplemented(
-            "Remote execution not supported. Compile with feature 'ballista' to enable"
-                .to_string(),
-        ))
-    }
-    pub async fn sql(&mut self, _sql: &str) -> Result<Arc<DataFrame>> {
-        unreachable!()
-    }
-}
diff --git a/ballista-cli/src/exec.rs b/ballista-cli/src/exec.rs
deleted file mode 100644
index dc3c73e5a..000000000
--- a/ballista-cli/src/exec.rs
+++ /dev/null
@@ -1,170 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Execution functions
-
-use crate::{
-    command::{Command, OutputFormat},
-    context::Context,
-    helper::CliHelper,
-    print_options::PrintOptions,
-};
-use datafusion::error::Result;
-use rustyline::error::ReadlineError;
-use rustyline::Editor;
-use std::fs::File;
-use std::io::prelude::*;
-use std::io::BufReader;
-use std::time::Instant;
-
-/// run and execute SQL statements and commands from a file, against a context with the given print options
-pub async fn exec_from_lines(
-    ctx: &mut Context,
-    reader: &mut BufReader<File>,
-    print_options: &PrintOptions,
-) {
-    let mut query = "".to_owned();
-
-    for line in reader.lines() {
-        match line {
-            Ok(line) if line.starts_with("--") => {
-                continue;
-            }
-            Ok(line) => {
-                let line = line.trim_end();
-                query.push_str(line);
-                if line.ends_with(';') {
-                    match exec_and_print(ctx, print_options, query).await {
-                        Ok(_) => {}
-                        Err(err) => println!("{:?}", err),
-                    }
-                    query = "".to_owned();
-                } else {
-                    query.push('\n');
-                }
-            }
-            _ => {
-                break;
-            }
-        }
-    }
-
-    // run the left over query if the last statement doesn't contain ‘;’
-    if !query.is_empty() {
-        match exec_and_print(ctx, print_options, query).await {
-            Ok(_) => {}
-            Err(err) => println!("{:?}", err),
-        }
-    }
-}
-
-pub async fn exec_from_files(
-    files: Vec<String>,
-    ctx: &mut Context,
-    print_options: &PrintOptions,
-) {
-    let files = files
-        .into_iter()
-        .map(|file_path| File::open(file_path).unwrap())
-        .collect::<Vec<_>>();
-    for file in files {
-        let mut reader = BufReader::new(file);
-        exec_from_lines(ctx, &mut reader, print_options).await;
-    }
-}
-
-/// run and execute SQL statements and commands against a context with the given print options
-pub async fn exec_from_repl(ctx: &mut Context, print_options: &mut PrintOptions) {
-    let mut rl = Editor::<CliHelper>::new();
-    rl.set_helper(Some(CliHelper::default()));
-    rl.load_history(".history").ok();
-
-    let mut print_options = print_options.clone();
-
-    loop {
-        match rl.readline("❯ ") {
-            Ok(line) if line.starts_with('\\') => {
-                rl.add_history_entry(line.trim_end());
-                let command = line.split_whitespace().collect::<Vec<_>>().join(" ");
-                if let Ok(cmd) = &command[1..].parse::<Command>() {
-                    match cmd {
-                        Command::Quit => break,
-                        Command::OutputFormat(subcommand) => {
-                            if let Some(subcommand) = subcommand {
-                                if let Ok(command) = subcommand.parse::<OutputFormat>() {
-                                    if let Err(e) =
-                                        command.execute(&mut print_options).await
-                                    {
-                                        eprintln!("{}", e)
-                                    }
-                                } else {
-                                    eprintln!(
-                                        "'\\{}' is not a valid command",
-                                        &line[1..]
-                                    );
-                                }
-                            } else {
-                                println!("Output format is {:?}.", print_options.format);
-                            }
-                        }
-                        _ => {
-                            if let Err(e) = cmd.execute(ctx, &mut print_options).await {
-                                eprintln!("{}", e)
-                            }
-                        }
-                    }
-                } else {
-                    eprintln!("'\\{}' is not a valid command", &line[1..]);
-                }
-            }
-            Ok(line) => {
-                rl.add_history_entry(line.trim_end());
-                match exec_and_print(ctx, &print_options, line).await {
-                    Ok(_) => {}
-                    Err(err) => eprintln!("{:?}", err),
-                }
-            }
-            Err(ReadlineError::Interrupted) => {
-                println!("^C");
-                continue;
-            }
-            Err(ReadlineError::Eof) => {
-                println!("\\q");
-                break;
-            }
-            Err(err) => {
-                eprintln!("Unknown error happened {:?}", err);
-                break;
-            }
-        }
-    }
-
-    rl.save_history(".history").ok();
-}
-
-async fn exec_and_print(
-    ctx: &mut Context,
-    print_options: &PrintOptions,
-    sql: String,
-) -> Result<()> {
-    let now = Instant::now();
-    let df = ctx.sql(&sql).await?;
-    let results = df.collect().await?;
-    print_options.print_batches(&results, now)?;
-
-    Ok(())
-}
diff --git a/ballista-cli/src/lib.rs b/ballista-cli/src/lib.rs
deleted file mode 100644
index a51aad013..000000000
--- a/ballista-cli/src/lib.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#![doc = include_str!("../README.md")]
-pub const BALLISTA_CLI_VERSION: &str = env!("CARGO_PKG_VERSION");
-
-pub mod command;
-pub mod context;
-pub mod exec;
-
-pub use datafusion_cli::{helper, print_format, print_options, functions};
diff --git a/ballista-cli/src/main.rs b/ballista-cli/src/main.rs
deleted file mode 100644
index 69868091d..000000000
--- a/ballista-cli/src/main.rs
+++ /dev/null
@@ -1,165 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use ballista_cli::{
-    context::Context, exec, print_format::PrintFormat, print_options::PrintOptions,
-    BALLISTA_CLI_VERSION,
-};
-use clap::Parser;
-use datafusion::error::Result;
-use datafusion::execution::context::SessionConfig;
-use mimalloc::MiMalloc;
-use std::env;
-use std::path::Path;
-
-#[global_allocator]
-static GLOBAL: MiMalloc = MiMalloc;
-
-#[derive(Debug, Parser, PartialEq)]
-#[clap(author, version, about, long_about= None)]
-struct Args {
-    #[clap(
-        short = 'p',
-        long,
-        help = "Path to your data, default to current directory",
-        validator(is_valid_data_dir)
-    )]
-    data_path: Option<String>,
-
-    #[clap(
-        short = 'c',
-        long,
-        help = "The batch size of each query, or use DataFusion default",
-        validator(is_valid_batch_size)
-    )]
-    batch_size: Option<usize>,
-
-    #[clap(
-        short,
-        long,
-        multiple_values = true,
-        help = "Execute commands from file(s), then exit",
-        validator(is_valid_file)
-    )]
-    file: Vec<String>,
-
-    #[clap(
-        short = 'r',
-        long,
-        multiple_values = true,
-        help = "Run the provided files on startup instead of ~/.datafusionrc",
-        validator(is_valid_file),
-        conflicts_with = "file"
-    )]
-    rc: Option<Vec<String>>,
-
-    #[clap(long, arg_enum, default_value_t = PrintFormat::Table)]
-    format: PrintFormat,
-
-    #[clap(long, help = "Ballista scheduler host")]
-    host: Option<String>,
-
-    #[clap(long, help = "Ballista scheduler port")]
-    port: Option<u16>,
-
-    #[clap(
-        short,
-        long,
-        help = "Reduce printing other than the results and work quietly"
-    )]
-    quiet: bool,
-}
-
-#[tokio::main]
-pub async fn main() -> Result<()> {
-    env_logger::init();
-    let args = Args::parse();
-
-    if !args.quiet {
-        println!("Ballista CLI v{}", BALLISTA_CLI_VERSION);
-    }
-
-    if let Some(ref path) = args.data_path {
-        let p = Path::new(path);
-        env::set_current_dir(&p).unwrap();
-    };
-
-    let mut session_config = SessionConfig::new().with_information_schema(true);
-
-    if let Some(batch_size) = args.batch_size {
-        session_config = session_config.with_batch_size(batch_size);
-    };
-
-    let mut ctx: Context = match (args.host, args.port) {
-        (Some(ref h), Some(p)) => Context::new_remote(h, p).await?,
-        _ => Context::new_local(&session_config),
-    };
-
-    let mut print_options = PrintOptions {
-        format: args.format,
-        quiet: args.quiet,
-    };
-
-    let files = args.file;
-    let rc = match args.rc {
-        Some(file) => file,
-        None => {
-            let mut files = Vec::new();
-            let home = dirs::home_dir();
-            if let Some(p) = home {
-                let home_rc = p.join(".datafusionrc");
-                if home_rc.exists() {
-                    files.push(home_rc.into_os_string().into_string().unwrap());
-                }
-            }
-            files
-        }
-    };
-    if !files.is_empty() {
-        exec::exec_from_files(files, &mut ctx, &print_options).await
-    } else {
-        if !rc.is_empty() {
-            exec::exec_from_files(rc, &mut ctx, &print_options).await
-        }
-        exec::exec_from_repl(&mut ctx, &mut print_options).await;
-    }
-
-    Ok(())
-}
-
-fn is_valid_file(dir: &str) -> std::result::Result<(), String> {
-    if Path::new(dir).is_file() {
-        Ok(())
-    } else {
-        Err(format!("Invalid file '{}'", dir))
-    }
-}
-
-fn is_valid_data_dir(dir: &str) -> std::result::Result<(), String> {
-    if Path::new(dir).is_dir() {
-        Ok(())
-    } else {
-        Err(format!("Invalid data directory '{}'", dir))
-    }
-}
-
-fn is_valid_batch_size(size: &str) -> std::result::Result<(), String> {
-    match size.parse::<usize>() {
-        Ok(size) if size > 0 => Ok(()),
-        _ => Err(format!("Invalid batch size '{}'", size)),
-    }
-}
diff --git a/ballista-examples/Cargo.toml b/ballista-examples/Cargo.toml
deleted file mode 100644
index f119401b0..000000000
--- a/ballista-examples/Cargo.toml
+++ /dev/null
@@ -1,44 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "ballista-examples"
-description = "Ballista usage examples"
-version = "0.6.0"
-homepage = "https://github.com/apache/arrow-datafusion"
-repository = "https://github.com/apache/arrow-datafusion"
-authors = ["Apache Arrow <de...@arrow.apache.org>"]
-license = "Apache-2.0"
-keywords = [ "arrow", "distributed", "query", "sql" ]
-edition = "2021"
-publish = false
-rust-version = "1.59"
-
-
-[[example]]
-name = "test_sql"
-path = "examples/test_sql.rs"
-required-features = ["ballista/standalone"]
-
-[dependencies]
-ballista = { path = "../ballista/rust/client", version = "0.7.0" }
-datafusion = { path = "../datafusion/core" }
-futures = "0.3"
-num_cpus = "1.13.0"
-prost = "0.10"
-tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
-tonic = "0.7"
diff --git a/ballista-examples/README.md b/ballista-examples/README.md
deleted file mode 100644
index 8dd83fc9e..000000000
--- a/ballista-examples/README.md
+++ /dev/null
@@ -1,55 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Ballista Examples
-
-This directory contains examples for executing distributed queries with Ballista.
-
-For background information on the Ballista architecture, refer to 
-the [Ballista README](../ballista/README.md).
-
-## Start a standalone cluster
-
-From the root of the arrow-datafusion project, build release binaries.
-
-```bash
-cargo build --release
-```
-
-Start a Ballista scheduler process in a new terminal session.
-
-```bash
-RUST_LOG=info ./target/release/ballista-scheduler
-```
-
-Start one or more Ballista executor processes in new terminal sessions. When starting more than one 
-executor, a unique port number must be specified for each executor.
-
-```bash
-RUST_LOG=info ./target/release/ballista-executor -c 4
-```
-
-## Running the examples
-
-The examples can be run using the `cargo run --bin` syntax. 
-
-```bash
-cargo run --release --bin ballista-dataframe
-```
-
diff --git a/ballista-examples/examples/test_sql.rs b/ballista-examples/examples/test_sql.rs
deleted file mode 100644
index 136f4f4ab..000000000
--- a/ballista-examples/examples/test_sql.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use ballista::prelude::{BallistaConfig, BallistaContext, Result};
-use datafusion::prelude::CsvReadOptions;
-
-/// This example show the udf plugin is work
-///
-#[tokio::main]
-async fn main() -> Result<()> {
-    let config = BallistaConfig::builder()
-        .set("ballista.shuffle.partitions", "1")
-        .build()?;
-
-    let ctx = BallistaContext::standalone(&config, 10).await.unwrap();
-
-    let testdata = datafusion::test_util::arrow_test_data();
-
-    // register csv file with the execution context
-    ctx.register_csv(
-        "aggregate_test_100",
-        &format!("{}/csv/aggregate_test_100.csv", testdata),
-        CsvReadOptions::new(),
-    )
-    .await?;
-
-    // test udf
-    let df = ctx.sql("select count(1) from aggregate_test_100").await?;
-
-    df.show().await?;
-    Ok(())
-}
diff --git a/ballista-examples/src/bin/ballista-dataframe.rs b/ballista-examples/src/bin/ballista-dataframe.rs
deleted file mode 100644
index a819950f4..000000000
--- a/ballista-examples/src/bin/ballista-dataframe.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use ballista::prelude::*;
-use datafusion::prelude::{col, lit, ParquetReadOptions};
-
-/// This example demonstrates executing a simple query against an Arrow data source (Parquet) and
-/// fetching results, using the DataFrame trait
-#[tokio::main]
-async fn main() -> Result<()> {
-    let config = BallistaConfig::builder()
-        .set("ballista.shuffle.partitions", "4")
-        .build()?;
-    let ctx = BallistaContext::remote("localhost", 50050, &config).await?;
-
-    let testdata = datafusion::test_util::parquet_test_data();
-
-    let filename = &format!("{}/alltypes_plain.parquet", testdata);
-
-    // define the query using the DataFrame trait
-    let df = ctx
-        .read_parquet(filename, ParquetReadOptions::default())
-        .await?
-        .select_columns(&["id", "bool_col", "timestamp_col"])?
-        .filter(col("id").gt(lit(1)))?;
-
-    // print the results
-    df.show().await?;
-
-    Ok(())
-}
diff --git a/ballista-examples/src/bin/ballista-sql.rs b/ballista-examples/src/bin/ballista-sql.rs
deleted file mode 100644
index b4a1260f6..000000000
--- a/ballista-examples/src/bin/ballista-sql.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use ballista::prelude::*;
-use datafusion::prelude::CsvReadOptions;
-
-/// This example demonstrates executing a simple query against an Arrow data source (CSV) and
-/// fetching results, using SQL
-#[tokio::main]
-async fn main() -> Result<()> {
-    let config = BallistaConfig::builder()
-        .set("ballista.shuffle.partitions", "4")
-        .build()?;
-    let ctx = BallistaContext::remote("localhost", 50050, &config).await?;
-
-    let testdata = datafusion::test_util::arrow_test_data();
-
-    // register csv file with the execution context
-    ctx.register_csv(
-        "aggregate_test_100",
-        &format!("{}/csv/aggregate_test_100.csv", testdata),
-        CsvReadOptions::new(),
-    )
-    .await?;
-
-    // execute the query
-    let df = ctx
-        .sql(
-            "SELECT c1, MIN(c12), MAX(c12) \
-        FROM aggregate_test_100 \
-        WHERE c11 > 0.1 AND c11 < 0.9 \
-        GROUP BY c1",
-        )
-        .await?;
-
-    // print the results
-    df.show().await?;
-
-    Ok(())
-}
diff --git a/ballista/CHANGELOG.md b/ballista/CHANGELOG.md
deleted file mode 100644
index 07ce062a4..000000000
--- a/ballista/CHANGELOG.md
+++ /dev/null
@@ -1,549 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Changelog
-
-## [ballista-0.7.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.7.0) (2022-05-12)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.1.0-rc1...ballista-0.7.0)
-
-**Breaking changes:**
-
-- Make `ExecutionPlan::execute` Sync [\#2434](https://github.com/apache/arrow-datafusion/pull/2434) ([tustvold](https://github.com/tustvold))
-- Add `Expr::Exists` to represent EXISTS subquery expression [\#2339](https://github.com/apache/arrow-datafusion/pull/2339) ([andygrove](https://github.com/andygrove))
-- Remove dependency from `LogicalPlan::TableScan` to `ExecutionPlan` [\#2284](https://github.com/apache/arrow-datafusion/pull/2284) ([andygrove](https://github.com/andygrove))
-- Move logical expression type-coercion code from `physical-expr` crate to `expr` crate [\#2257](https://github.com/apache/arrow-datafusion/pull/2257) ([andygrove](https://github.com/andygrove))
-- feat: 2061 create external table ddl table partition cols [\#2099](https://github.com/apache/arrow-datafusion/pull/2099) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jychen7](https://github.com/jychen7))
-- Reorganize the project folders [\#2081](https://github.com/apache/arrow-datafusion/pull/2081) ([yahoNanJing](https://github.com/yahoNanJing))
-- Support more ScalarFunction in Ballista [\#2008](https://github.com/apache/arrow-datafusion/pull/2008) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Merge dataframe and dataframe imp [\#1998](https://github.com/apache/arrow-datafusion/pull/1998) ([vchag](https://github.com/vchag))
-- Rename `ExecutionContext` to `SessionContext`, `ExecutionContextState` to `SessionState`, add `TaskContext` to support multi-tenancy configurations - Part 1 [\#1987](https://github.com/apache/arrow-datafusion/pull/1987) ([mingmwang](https://github.com/mingmwang))
-- Add Coalesce function [\#1969](https://github.com/apache/arrow-datafusion/pull/1969) ([msathis](https://github.com/msathis))
-- Add Create Schema functionality in SQL [\#1959](https://github.com/apache/arrow-datafusion/pull/1959) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
-- remove sync constraint of SendableRecordBatchStream [\#1884](https://github.com/apache/arrow-datafusion/pull/1884) ([doki23](https://github.com/doki23))
-
-**Implemented enhancements:**
-
-- Add `CREATE VIEW` [\#2279](https://github.com/apache/arrow-datafusion/pull/2279) ([matthewmturner](https://github.com/matthewmturner))
-- \[Ballista\] Support Union in ballista. [\#2098](https://github.com/apache/arrow-datafusion/pull/2098) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Add missing aggr\_expr to PhysicalExprNode for Ballista. [\#1989](https://github.com/apache/arrow-datafusion/pull/1989) ([Ted-Jiang](https://github.com/Ted-Jiang))
-
-**Fixed bugs:**
-
-- Ballista integration tests no longer work [\#2440](https://github.com/apache/arrow-datafusion/issues/2440)
-- Ballista crates cannot be released from DafaFusion 7.0.0 source release [\#1980](https://github.com/apache/arrow-datafusion/issues/1980)
-- protobuf OctetLength should be deserialized as octet\_length, not length [\#1834](https://github.com/apache/arrow-datafusion/pull/1834) ([carols10cents](https://github.com/carols10cents))
-
-**Documentation updates:**
-
-- MINOR: Make crate READMEs consistent [\#2437](https://github.com/apache/arrow-datafusion/pull/2437) ([andygrove](https://github.com/andygrove))
-- docs: Update the Ballista dev env instructions [\#2419](https://github.com/apache/arrow-datafusion/pull/2419) ([haoxins](https://github.com/haoxins))
-- Revise document of installing ballista pinned to specified version [\#2034](https://github.com/apache/arrow-datafusion/pull/2034) ([WinkerDu](https://github.com/WinkerDu))
-- Fix typos \(Datafusion -\> DataFusion\) [\#1993](https://github.com/apache/arrow-datafusion/pull/1993) ([andygrove](https://github.com/andygrove))
-
-**Performance improvements:**
-
-- Introduce StageManager for managing tasks stage by stage [\#1983](https://github.com/apache/arrow-datafusion/pull/1983) ([yahoNanJing](https://github.com/yahoNanJing))
-
-**Closed issues:**
-
-- Make expected result string in unit tests more readable [\#2412](https://github.com/apache/arrow-datafusion/issues/2412)
-- remove duplicated `fn aggregate()` in aggregate expression tests [\#2399](https://github.com/apache/arrow-datafusion/issues/2399)
-- split `distinct_expression.rs` into `count_distinct.rs` and `array_agg_distinct.rs` [\#2385](https://github.com/apache/arrow-datafusion/issues/2385)
-- move sql tests in `context.rs` to corresponding test files in `datafustion/core/tests/sql` [\#2328](https://github.com/apache/arrow-datafusion/issues/2328)
-- Date32/Date64 as join keys for merge join [\#2314](https://github.com/apache/arrow-datafusion/issues/2314)
-- Error precision and scale for decimal coercion in logic comparison [\#2232](https://github.com/apache/arrow-datafusion/issues/2232)
-- Support Multiple row layout [\#2188](https://github.com/apache/arrow-datafusion/issues/2188)
-- Discussion: Is Ballista a standalone system or framework [\#1916](https://github.com/apache/arrow-datafusion/issues/1916)
-
-**Merged pull requests:**
-
-- MINOR: Enable multi-statement benchmark queries [\#2507](https://github.com/apache/arrow-datafusion/pull/2507) ([andygrove](https://github.com/andygrove))
-- Persist session configs in scheduler [\#2501](https://github.com/apache/arrow-datafusion/pull/2501) ([thinkharderdev](https://github.com/thinkharderdev))
-- Update to `sqlparser` `0.17.0` [\#2500](https://github.com/apache/arrow-datafusion/pull/2500) ([alamb](https://github.com/alamb))
-- Limit cpu cores used when generating changelog [\#2494](https://github.com/apache/arrow-datafusion/pull/2494) ([andygrove](https://github.com/andygrove))
-- MINOR: Parameterize changelog script [\#2484](https://github.com/apache/arrow-datafusion/pull/2484) ([jychen7](https://github.com/jychen7))
-- Fix stage key extraction [\#2472](https://github.com/apache/arrow-datafusion/pull/2472) ([thinkharderdev](https://github.com/thinkharderdev))
-- Add support for list\_dir\(\) on local fs [\#2467](https://github.com/apache/arrow-datafusion/pull/2467) ([wjones127](https://github.com/wjones127))
-- minor: update versions and paths in changelog scripts [\#2429](https://github.com/apache/arrow-datafusion/pull/2429) ([andygrove](https://github.com/andygrove))
-- Fix Ballista executing during plan [\#2428](https://github.com/apache/arrow-datafusion/pull/2428) ([tustvold](https://github.com/tustvold))
-- Re-organize and rename aggregates physical plan [\#2388](https://github.com/apache/arrow-datafusion/pull/2388) ([yjshen](https://github.com/yjshen))
-- Upgrade to arrow 13 [\#2382](https://github.com/apache/arrow-datafusion/pull/2382) ([alamb](https://github.com/alamb))
-- Grouped Aggregate in row format [\#2375](https://github.com/apache/arrow-datafusion/pull/2375) ([yjshen](https://github.com/yjshen))
-- Stop optimizing queries twice [\#2369](https://github.com/apache/arrow-datafusion/pull/2369) ([andygrove](https://github.com/andygrove))
-- Bump follow-redirects from 1.13.2 to 1.14.9 in /ballista/ui/scheduler [\#2325](https://github.com/apache/arrow-datafusion/pull/2325) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Move FileType enum from sql module to logical\_plan module [\#2290](https://github.com/apache/arrow-datafusion/pull/2290) ([andygrove](https://github.com/andygrove))
-- Add BatchPartitioner \(\#2285\) [\#2287](https://github.com/apache/arrow-datafusion/pull/2287) ([tustvold](https://github.com/tustvold))
-- Update uuid requirement from 0.8 to 1.0 [\#2280](https://github.com/apache/arrow-datafusion/pull/2280) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump async from 2.6.3 to 2.6.4 in /ballista/ui/scheduler [\#2277](https://github.com/apache/arrow-datafusion/pull/2277) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump minimist from 1.2.5 to 1.2.6 in /ballista/ui/scheduler [\#2276](https://github.com/apache/arrow-datafusion/pull/2276) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump url-parse from 1.5.1 to 1.5.10 in /ballista/ui/scheduler [\#2275](https://github.com/apache/arrow-datafusion/pull/2275) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Bump nanoid from 3.1.20 to 3.3.3 in /ballista/ui/scheduler [\#2274](https://github.com/apache/arrow-datafusion/pull/2274) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Update to Arrow 12.0.0, update tonic and prost [\#2253](https://github.com/apache/arrow-datafusion/pull/2253) ([alamb](https://github.com/alamb))
-- Add ExecutorMetricsCollector interface [\#2234](https://github.com/apache/arrow-datafusion/pull/2234) ([thinkharderdev](https://github.com/thinkharderdev))
-- minor: add editor config file [\#2224](https://github.com/apache/arrow-datafusion/pull/2224) ([jackwener](https://github.com/jackwener))
-- \[Ballista\] Enable ApproxPercentileWithWeight in Ballista and fill UT  [\#2192](https://github.com/apache/arrow-datafusion/pull/2192) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- make nightly clippy happy [\#2186](https://github.com/apache/arrow-datafusion/pull/2186) ([xudong963](https://github.com/xudong963))
-- \[Ballista\]Make PhysicalAggregateExprNode has repeated PhysicalExprNode [\#2184](https://github.com/apache/arrow-datafusion/pull/2184) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Add LogicalPlan::SubqueryAlias [\#2172](https://github.com/apache/arrow-datafusion/pull/2172) ([andygrove](https://github.com/andygrove))
-- Implement fast path of with\_new\_children\(\) in ExecutionPlan [\#2168](https://github.com/apache/arrow-datafusion/pull/2168) ([mingmwang](https://github.com/mingmwang))
-- \[MINOR\] ignore suspicious slow test in Ballista [\#2167](https://github.com/apache/arrow-datafusion/pull/2167) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- enable explain for ballista [\#2163](https://github.com/apache/arrow-datafusion/pull/2163) ([doki23](https://github.com/doki23))
-- Add delimiter for create external table [\#2162](https://github.com/apache/arrow-datafusion/pull/2162) ([matthewmturner](https://github.com/matthewmturner))
-- Update sqlparser requirement from 0.15 to 0.16 [\#2152](https://github.com/apache/arrow-datafusion/pull/2152) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Add IF NOT EXISTS to `CREATE TABLE` and `CREATE EXTERNAL TABLE` [\#2143](https://github.com/apache/arrow-datafusion/pull/2143) ([matthewmturner](https://github.com/matthewmturner))
-- Update quarterly roadmap for Q2 [\#2133](https://github.com/apache/arrow-datafusion/pull/2133) ([matthewmturner](https://github.com/matthewmturner))
-- \[Ballista\] Add ballista plugin manager and UDF plugin [\#2131](https://github.com/apache/arrow-datafusion/pull/2131) ([gaojun2048](https://github.com/gaojun2048))
-- Serialize scalar UDFs in physical plan [\#2130](https://github.com/apache/arrow-datafusion/pull/2130) ([thinkharderdev](https://github.com/thinkharderdev))
-- doc: update release schedule [\#2110](https://github.com/apache/arrow-datafusion/pull/2110) ([jychen7](https://github.com/jychen7))
-- Reduce repetition in Decimal binary kernels, upgrade to arrow 11.1 [\#2107](https://github.com/apache/arrow-datafusion/pull/2107) ([alamb](https://github.com/alamb))
-- update zlib version to 1.2.12 [\#2106](https://github.com/apache/arrow-datafusion/pull/2106) ([waitingkuo](https://github.com/waitingkuo))
-- Add CREATE DATABASE command to SQL [\#2094](https://github.com/apache/arrow-datafusion/pull/2094) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
-- Refactor SessionContext, BallistaContext to support multi-tenancy configurations - Part 3 [\#2091](https://github.com/apache/arrow-datafusion/pull/2091) ([mingmwang](https://github.com/mingmwang))
-- Remove dependency of common for the storage crate [\#2076](https://github.com/apache/arrow-datafusion/pull/2076) ([yahoNanJing](https://github.com/yahoNanJing))
-- [MINOR] fix doc in `EXTRACT\(field FROM source\) [\#2074](https://github.com/apache/arrow-datafusion/pull/2074) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- \[Bug\]\[Datafusion\] fix TaskContext session\_config bug [\#2070](https://github.com/apache/arrow-datafusion/pull/2070) ([gaojun2048](https://github.com/gaojun2048))
-- Short-circuit evaluation for `CaseWhen` [\#2068](https://github.com/apache/arrow-datafusion/pull/2068) ([yjshen](https://github.com/yjshen))
-- split datafusion-object-store module [\#2065](https://github.com/apache/arrow-datafusion/pull/2065) ([yahoNanJing](https://github.com/yahoNanJing))
-- Change log level for noisy logs [\#2060](https://github.com/apache/arrow-datafusion/pull/2060) ([thinkharderdev](https://github.com/thinkharderdev))
-- Update to arrow/parquet 11.0 [\#2048](https://github.com/apache/arrow-datafusion/pull/2048) ([alamb](https://github.com/alamb))
-- minor: format comments \(`//` to `// `\) [\#2047](https://github.com/apache/arrow-datafusion/pull/2047) ([jackwener](https://github.com/jackwener))
-- use cargo-tomlfmt to check Cargo.toml formatting in CI [\#2033](https://github.com/apache/arrow-datafusion/pull/2033) ([WinkerDu](https://github.com/WinkerDu))
-- Refactor SessionContext, SessionState and SessionConfig to support multi-tenancy configurations - Part 2 [\#2029](https://github.com/apache/arrow-datafusion/pull/2029) ([mingmwang](https://github.com/mingmwang))
-- Simplify prerequisites for running examples [\#2028](https://github.com/apache/arrow-datafusion/pull/2028) ([doki23](https://github.com/doki23))
-- Use SessionContext to parse Expr protobuf [\#2024](https://github.com/apache/arrow-datafusion/pull/2024) ([thinkharderdev](https://github.com/thinkharderdev))
-- Fix stuck issue for the load testing of Push-based task scheduling [\#2006](https://github.com/apache/arrow-datafusion/pull/2006) ([yahoNanJing](https://github.com/yahoNanJing))
-- Fixing a typo in documentation [\#1997](https://github.com/apache/arrow-datafusion/pull/1997) ([psvri](https://github.com/psvri))
-- Fix minor clippy issue [\#1995](https://github.com/apache/arrow-datafusion/pull/1995) ([alamb](https://github.com/alamb))
-- Make it possible to only scan part of a parquet file in a partition [\#1990](https://github.com/apache/arrow-datafusion/pull/1990) ([yjshen](https://github.com/yjshen))
-- Update Dockerfile to fix integration tests [\#1982](https://github.com/apache/arrow-datafusion/pull/1982) ([andygrove](https://github.com/andygrove))
-- Update sqlparser requirement from 0.14 to 0.15 [\#1966](https://github.com/apache/arrow-datafusion/pull/1966) ([dependabot[bot]](https://github.com/apps/dependabot))
-- fix logical conflict with protobuf [\#1958](https://github.com/apache/arrow-datafusion/pull/1958) ([alamb](https://github.com/alamb))
-- Update to arrow 10.0.0, pyo3 0.16 [\#1957](https://github.com/apache/arrow-datafusion/pull/1957) ([alamb](https://github.com/alamb))
-- update jit-related dependencies [\#1953](https://github.com/apache/arrow-datafusion/pull/1953) ([xudong963](https://github.com/xudong963))
-- Allow different types of query variables \(`@@var`\) rather than just string [\#1943](https://github.com/apache/arrow-datafusion/pull/1943) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([maxburke](https://github.com/maxburke))
-- Pruning serialization [\#1941](https://github.com/apache/arrow-datafusion/pull/1941) ([thinkharderdev](https://github.com/thinkharderdev))
-- Fix select from EmptyExec always return 0 row after optimizer passes [\#1938](https://github.com/apache/arrow-datafusion/pull/1938) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Introduce Ballista query stage scheduler [\#1935](https://github.com/apache/arrow-datafusion/pull/1935) ([yahoNanJing](https://github.com/yahoNanJing))
-- Add db benchmark script [\#1928](https://github.com/apache/arrow-datafusion/pull/1928) ([matthewmturner](https://github.com/matthewmturner))
-- fix a typo [\#1919](https://github.com/apache/arrow-datafusion/pull/1919) ([vchag](https://github.com/vchag))
-- \[MINOR\] Update copyright year in Docs [\#1918](https://github.com/apache/arrow-datafusion/pull/1918) ([alamb](https://github.com/alamb))
-- add metadata to DFSchema, close \#1806. [\#1914](https://github.com/apache/arrow-datafusion/pull/1914) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([jiacai2050](https://github.com/jiacai2050))
-- Refactor scheduler state mod [\#1913](https://github.com/apache/arrow-datafusion/pull/1913) ([yahoNanJing](https://github.com/yahoNanJing))
-- Refactor the event channel [\#1912](https://github.com/apache/arrow-datafusion/pull/1912) ([yahoNanJing](https://github.com/yahoNanJing))
-- Refactor scheduler server [\#1911](https://github.com/apache/arrow-datafusion/pull/1911) ([yahoNanJing](https://github.com/yahoNanJing))
-- Clippy fix on nightly [\#1907](https://github.com/apache/arrow-datafusion/pull/1907) ([yjshen](https://github.com/yjshen))
-- Updated Rust version to 1.59 in all the files [\#1903](https://github.com/apache/arrow-datafusion/pull/1903) ([NaincyKumariKnoldus](https://github.com/NaincyKumariKnoldus))
-- Remove uneeded Mutex in Ballista Client [\#1898](https://github.com/apache/arrow-datafusion/pull/1898) ([alamb](https://github.com/alamb))
-- Create a `datafusion-proto` crate for datafusion protobuf serialization [\#1887](https://github.com/apache/arrow-datafusion/pull/1887) ([carols10cents](https://github.com/carols10cents))
-- Fix clippy lints [\#1885](https://github.com/apache/arrow-datafusion/pull/1885) ([HaoYang670](https://github.com/HaoYang670))
-- Separate cpu-bound \(query-execution\) and IO-bound\(heartbeat\) to  … [\#1883](https://github.com/apache/arrow-datafusion/pull/1883) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- \[Minor\] Clean up DecimalArray API Usage [\#1869](https://github.com/apache/arrow-datafusion/pull/1869) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- Changes after went through "Datafusion as a library section" [\#1868](https://github.com/apache/arrow-datafusion/pull/1868) ([nonontb](https://github.com/nonontb))
-- Remove allow unused imports from ballista-core, then fix all warnings [\#1853](https://github.com/apache/arrow-datafusion/pull/1853) ([carols10cents](https://github.com/carols10cents))
-- Update to arrow 9.1.0 [\#1851](https://github.com/apache/arrow-datafusion/pull/1851) ([alamb](https://github.com/alamb))
-- move some tests out of context and into sql [\#1846](https://github.com/apache/arrow-datafusion/pull/1846) ([alamb](https://github.com/alamb))
-- Fix compiling ballista  in standalone mode, add build to CI [\#1839](https://github.com/apache/arrow-datafusion/pull/1839) ([alamb](https://github.com/alamb))
-- Update documentation example for change in API [\#1812](https://github.com/apache/arrow-datafusion/pull/1812) ([alamb](https://github.com/alamb))
-- Refactor scheduler state with different management policy for volatile and stable states [\#1810](https://github.com/apache/arrow-datafusion/pull/1810) ([yahoNanJing](https://github.com/yahoNanJing))
-- DataFusion + Conbench Integration [\#1791](https://github.com/apache/arrow-datafusion/pull/1791) ([dianaclarke](https://github.com/dianaclarke))
-- Enable periodic cleanup of work\_dir directories in ballista executor [\#1783](https://github.com/apache/arrow-datafusion/pull/1783) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Use`eq_dyn`, `neq_dyn`, `lt_dyn`, `lt_eq_dyn`, `gt_dyn`, `gt_eq_dyn` kernels from arrow [\#1475](https://github.com/apache/arrow-datafusion/pull/1475) ([alamb](https://github.com/alamb))
-
-## [7.1.0-rc1](https://github.com/apache/arrow-datafusion/tree/7.1.0-rc1) (2022-04-10)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.0.0-rc2...7.1.0-rc1)
-
-**Implemented enhancements:**
-
-- Support substring with three arguments: \(str, from, for\) for DataFrame API and Ballista [\#2092](https://github.com/apache/arrow-datafusion/issues/2092)
-- UnionAll support for Ballista [\#2032](https://github.com/apache/arrow-datafusion/issues/2032)
-- Separate cpu-bound and IO-bound work in ballista-executor by using diff tokio runtime. [\#1770](https://github.com/apache/arrow-datafusion/issues/1770)
-- \[Ballista\] Introduce DAGScheduler for better managing the stage-based task scheduling [\#1704](https://github.com/apache/arrow-datafusion/issues/1704)
-- \[Ballista\] Support to better manage cluster state, like alive executors, executor available task slots, etc [\#1703](https://github.com/apache/arrow-datafusion/issues/1703)
-
-**Closed issues:**
-
-- Optimize memory usage pattern to avoid "double memory" behavior [\#2149](https://github.com/apache/arrow-datafusion/issues/2149)
-- Document approx\_percentile\_cont\_with\_weight in users guide [\#2078](https://github.com/apache/arrow-datafusion/issues/2078)
-- \[follow up\]cleaning up statements.remove\(0\) [\#1986](https://github.com/apache/arrow-datafusion/issues/1986)
-- Formatting error on documentation for Python [\#1873](https://github.com/apache/arrow-datafusion/issues/1873)
-- Remove duplicate tests from `test_const_evaluator_scalar_functions` [\#1727](https://github.com/apache/arrow-datafusion/issues/1727)
-- Question: Is the Ballista project providing value to the overall DataFusion project? [\#1273](https://github.com/apache/arrow-datafusion/issues/1273)
-
-## [7.0.0-rc2](https://github.com/apache/arrow-datafusion/tree/7.0.0-rc2) (2022-02-14)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/7.0.0...7.0.0-rc2)
-
-## [7.0.0](https://github.com/apache/arrow-datafusion/tree/7.0.0) (2022-02-14)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/6.0.0-rc0...7.0.0)
-
-**Breaking changes:**
-
-- Update `ExecutionPlan` to know about sortedness and repartitioning optimizer pass respect the invariants [\#1776](https://github.com/apache/arrow-datafusion/pull/1776) ([alamb](https://github.com/alamb))
-- Update to `arrow 8.0.0` [\#1673](https://github.com/apache/arrow-datafusion/pull/1673) ([alamb](https://github.com/alamb))
-
-**Implemented enhancements:**
-
-- Task assignment between Scheduler and Executors [\#1221](https://github.com/apache/arrow-datafusion/issues/1221)
-- Add `approx_median()` aggregate function [\#1729](https://github.com/apache/arrow-datafusion/pull/1729) ([realno](https://github.com/realno))
-- \[Ballista\] Add Decimal128, Date64, TimestampSecond, TimestampMillisecond, Interv… [\#1659](https://github.com/apache/arrow-datafusion/pull/1659) ([gaojun2048](https://github.com/gaojun2048))
-- Add `corr` aggregate function [\#1561](https://github.com/apache/arrow-datafusion/pull/1561) ([realno](https://github.com/realno))
-- Add `covar`, `covar_pop` and `covar_samp` aggregate functions [\#1551](https://github.com/apache/arrow-datafusion/pull/1551) ([realno](https://github.com/realno))
-- Add `approx_quantile()` aggregation function [\#1539](https://github.com/apache/arrow-datafusion/pull/1539) ([domodwyer](https://github.com/domodwyer))
-- Initial MemoryManager and DiskManager APIs  for query execution + External Sort implementation [\#1526](https://github.com/apache/arrow-datafusion/pull/1526) ([yjshen](https://github.com/yjshen))
-- Add `stddev` and `variance` [\#1525](https://github.com/apache/arrow-datafusion/pull/1525) ([realno](https://github.com/realno))
-- Add `rem` operation for Expr [\#1467](https://github.com/apache/arrow-datafusion/pull/1467) ([liukun4515](https://github.com/liukun4515))
-- Implement `array_agg` aggregate function [\#1300](https://github.com/apache/arrow-datafusion/pull/1300) ([viirya](https://github.com/viirya))
-
-**Fixed bugs:**
-
-- Ballista context::tests::test\_standalone\_mode test fails [\#1020](https://github.com/apache/arrow-datafusion/issues/1020)
-- \[Ballista\] Fix scheduler state mod bug [\#1655](https://github.com/apache/arrow-datafusion/pull/1655) ([gaojun2048](https://github.com/gaojun2048))
-- Pass local address host so we do not get mismatch between IPv4 and IP… [\#1466](https://github.com/apache/arrow-datafusion/pull/1466) ([thinkharderdev](https://github.com/thinkharderdev))
-- Add Timezone to Scalar::Time\* types,   and better timezone awareness to Datafusion's time types [\#1455](https://github.com/apache/arrow-datafusion/pull/1455) ([maxburke](https://github.com/maxburke))
-
-**Documentation updates:**
-
-- Add dependencies to ballista example documentation [\#1346](https://github.com/apache/arrow-datafusion/pull/1346) ([jgoday](https://github.com/jgoday))
-- \[MINOR\] Fix some typos. [\#1310](https://github.com/apache/arrow-datafusion/pull/1310) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- fix some clippy warnings from nightly channel [\#1277](https://github.com/apache/arrow-datafusion/pull/1277) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Jimexist](https://github.com/Jimexist))
-
-**Performance improvements:**
-
-- Introduce push-based task scheduling for Ballista [\#1560](https://github.com/apache/arrow-datafusion/pull/1560) ([yahoNanJing](https://github.com/yahoNanJing))
-
-**Closed issues:**
-
-- Track memory usage in Non Limited Operators [\#1569](https://github.com/apache/arrow-datafusion/issues/1569)
-- \[Question\] Why does ballista store tables in the client instead of in the SchedulerServer [\#1473](https://github.com/apache/arrow-datafusion/issues/1473)
-- Why use the expr types before coercion to get the result type? [\#1358](https://github.com/apache/arrow-datafusion/issues/1358)
-- A problem about the projection\_push\_down optimizer gathers valid columns  [\#1312](https://github.com/apache/arrow-datafusion/issues/1312)
-- apply constant folding to `LogicalPlan::Values` [\#1170](https://github.com/apache/arrow-datafusion/issues/1170)
-- reduce usage of `IntoIterator<Item = Expr>` in logical plan builder window fn [\#372](https://github.com/apache/arrow-datafusion/issues/372)
-
-**Merged pull requests:**
-
-- Fix verification scripts for 7.0.0 release [\#1830](https://github.com/apache/arrow-datafusion/pull/1830) ([alamb](https://github.com/alamb))
-- update README for ballista [\#1817](https://github.com/apache/arrow-datafusion/pull/1817) ([liukun4515](https://github.com/liukun4515))
-- Fix logical conflict [\#1801](https://github.com/apache/arrow-datafusion/pull/1801) ([alamb](https://github.com/alamb))
-- Improve the error message and UX of tpch benchmark program [\#1800](https://github.com/apache/arrow-datafusion/pull/1800) ([alamb](https://github.com/alamb))
-- Update to sqlparser 0.14 [\#1796](https://github.com/apache/arrow-datafusion/pull/1796) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- Update datafusion versions [\#1793](https://github.com/apache/arrow-datafusion/pull/1793) ([matthewmturner](https://github.com/matthewmturner))
-- Update datafusion to use arrow 9.0.0 [\#1775](https://github.com/apache/arrow-datafusion/pull/1775) ([alamb](https://github.com/alamb))
-- Update parking\_lot requirement from 0.11 to 0.12 [\#1735](https://github.com/apache/arrow-datafusion/pull/1735) ([dependabot[bot]](https://github.com/apps/dependabot))
-- substitute `parking_lot::Mutex` for `std::sync::Mutex` [\#1720](https://github.com/apache/arrow-datafusion/pull/1720) ([xudong963](https://github.com/xudong963))
-- Create ListingTableConfig which includes file format and schema inference [\#1715](https://github.com/apache/arrow-datafusion/pull/1715) ([matthewmturner](https://github.com/matthewmturner))
-- Support `create_physical_expr` and `ExecutionContextState` or `DefaultPhysicalPlanner` for faster speed [\#1700](https://github.com/apache/arrow-datafusion/pull/1700) ([alamb](https://github.com/alamb))
-- Use NamedTempFile rather than `String` in DiskManager [\#1680](https://github.com/apache/arrow-datafusion/pull/1680) ([alamb](https://github.com/alamb))
-- Abstract over logical and physical plan representations in Ballista [\#1677](https://github.com/apache/arrow-datafusion/pull/1677) ([thinkharderdev](https://github.com/thinkharderdev))
-- upgrade clap to version 3 [\#1672](https://github.com/apache/arrow-datafusion/pull/1672) ([Jimexist](https://github.com/Jimexist))
-- Improve configuration and resource use of `MemoryManager` and `DiskManager` [\#1668](https://github.com/apache/arrow-datafusion/pull/1668) ([alamb](https://github.com/alamb))
-- Make `MemoryManager` and `MemoryStream` public [\#1664](https://github.com/apache/arrow-datafusion/pull/1664) ([yjshen](https://github.com/yjshen))
-- Consolidate Schema and RecordBatch projection [\#1638](https://github.com/apache/arrow-datafusion/pull/1638) ([alamb](https://github.com/alamb))
-- Update hashbrown requirement from 0.11 to 0.12 [\#1631](https://github.com/apache/arrow-datafusion/pull/1631) ([dependabot[bot]](https://github.com/apps/dependabot))
-- Update etcd-client requirement from 0.7 to 0.8 [\#1626](https://github.com/apache/arrow-datafusion/pull/1626) ([dependabot[bot]](https://github.com/apps/dependabot))
-- update nightly version [\#1597](https://github.com/apache/arrow-datafusion/pull/1597) ([Jimexist](https://github.com/Jimexist))
-- Add support show tables and show columns for ballista [\#1593](https://github.com/apache/arrow-datafusion/pull/1593) ([gaojun2048](https://github.com/gaojun2048))
-- minor: improve the benchmark readme [\#1567](https://github.com/apache/arrow-datafusion/pull/1567) ([xudong963](https://github.com/xudong963))
-- Consolidate `batch_size` configuration in `ExecutionConfig`, `RuntimeConfig` and `PhysicalPlanConfig` [\#1562](https://github.com/apache/arrow-datafusion/pull/1562) ([yjshen](https://github.com/yjshen))
-- Update to rust 1.58 [\#1557](https://github.com/apache/arrow-datafusion/pull/1557) ([xudong963](https://github.com/xudong963))
-- support mathematics operation for decimal data type [\#1554](https://github.com/apache/arrow-datafusion/pull/1554) ([liukun4515](https://github.com/liukun4515))
-- Make call SchedulerServer::new once in ballista-scheduler process [\#1537](https://github.com/apache/arrow-datafusion/pull/1537) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Add load test command in tpch.rs. [\#1530](https://github.com/apache/arrow-datafusion/pull/1530) ([Ted-Jiang](https://github.com/Ted-Jiang))
-- Remove one copy of ballista datatype serialization code [\#1524](https://github.com/apache/arrow-datafusion/pull/1524) ([alamb](https://github.com/alamb))
-- Update to arrow-7.0.0 [\#1523](https://github.com/apache/arrow-datafusion/pull/1523) ([alamb](https://github.com/alamb))
-- Workaround build failure: Pin quote to 1.0.10 [\#1499](https://github.com/apache/arrow-datafusion/pull/1499) ([alamb](https://github.com/alamb))
-- add rfcs for datafusion [\#1490](https://github.com/apache/arrow-datafusion/pull/1490) ([xudong963](https://github.com/xudong963))
-- support comparison for decimal data type and refactor the binary coercion rule [\#1483](https://github.com/apache/arrow-datafusion/pull/1483) ([liukun4515](https://github.com/liukun4515))
-- Update arrow-rs to 6.4.0 and replace boolean comparison in datafusion with arrow compute kernel [\#1446](https://github.com/apache/arrow-datafusion/pull/1446) ([xudong963](https://github.com/xudong963))
-- support cast/try\_cast for decimal: signed numeric to decimal [\#1442](https://github.com/apache/arrow-datafusion/pull/1442) ([liukun4515](https://github.com/liukun4515))
-- use 0.13 sql parser [\#1435](https://github.com/apache/arrow-datafusion/pull/1435) ([Jimexist](https://github.com/Jimexist))
-- Clarify communication on bi-weekly sync [\#1427](https://github.com/apache/arrow-datafusion/pull/1427) ([alamb](https://github.com/alamb))
-- Minimize features [\#1399](https://github.com/apache/arrow-datafusion/pull/1399) ([carols10cents](https://github.com/carols10cents))
-- Update rust vesion to 1.57 [\#1395](https://github.com/apache/arrow-datafusion/pull/1395) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
-- Add coercion rules for AggregateFunctions [\#1387](https://github.com/apache/arrow-datafusion/pull/1387) ([liukun4515](https://github.com/liukun4515))
-- upgrade the arrow-rs version [\#1385](https://github.com/apache/arrow-datafusion/pull/1385) ([liukun4515](https://github.com/liukun4515))
-- Extract logical plan: rename the plan name \(follow up\) [\#1354](https://github.com/apache/arrow-datafusion/pull/1354) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([liukun4515](https://github.com/liukun4515))
-- upgrade arrow-rs to 6.2.0 [\#1334](https://github.com/apache/arrow-datafusion/pull/1334) ([liukun4515](https://github.com/liukun4515))
-- Update release instructions [\#1331](https://github.com/apache/arrow-datafusion/pull/1331) ([alamb](https://github.com/alamb))
-- Extract Aggregate, Sort, and Join to struct from AggregatePlan [\#1326](https://github.com/apache/arrow-datafusion/pull/1326) ([matthewmturner](https://github.com/matthewmturner))
-- Extract `EmptyRelation`, `Limit`, `Values` from `LogicalPlan` [\#1325](https://github.com/apache/arrow-datafusion/pull/1325) ([liukun4515](https://github.com/liukun4515))
-- Extract CrossJoin, Repartition, Union in LogicalPlan [\#1322](https://github.com/apache/arrow-datafusion/pull/1322) ([liukun4515](https://github.com/liukun4515))
-- Extract Explain, Analyze, Extension in LogicalPlan as independent struct [\#1317](https://github.com/apache/arrow-datafusion/pull/1317) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
-- Extract CreateMemoryTable, DropTable, CreateExternalTable in LogicalPlan as independent struct [\#1311](https://github.com/apache/arrow-datafusion/pull/1311) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([liukun4515](https://github.com/liukun4515))
-- Extract Projection, Filter, Window in LogicalPlan as independent struct [\#1309](https://github.com/apache/arrow-datafusion/pull/1309) ([ic4y](https://github.com/ic4y))
-- Add PSQL comparison tests for except, intersect [\#1292](https://github.com/apache/arrow-datafusion/pull/1292) ([mrob95](https://github.com/mrob95))
-- Extract logical plans in LogicalPlan as independent struct: TableScan [\#1290](https://github.com/apache/arrow-datafusion/pull/1290) ([xudong963](https://github.com/xudong963))
-
-## [6.0.0-rc0](https://github.com/apache/arrow-datafusion/tree/6.0.0-rc0) (2021-11-14)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/6.0.0...6.0.0-rc0)
-
-## [6.0.0](https://github.com/apache/arrow-datafusion/tree/6.0.0) (2021-11-14)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/ballista-0.6.0...6.0.0)
-
-
-## [ballista-0.6.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.6.0) (2021-11-13)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/ballista-0.5.0...ballista-0.6.0)
-
-**Breaking changes:**
-
-- File partitioning for ListingTable [\#1141](https://github.com/apache/arrow-datafusion/pull/1141) ([rdettai](https://github.com/rdettai))
-- Register tables in BallistaContext using TableProviders instead of Dataframe [\#1028](https://github.com/apache/arrow-datafusion/pull/1028) ([rdettai](https://github.com/rdettai))
-- Make TableProvider.scan\(\) and PhysicalPlanner::create\_physical\_plan\(\) async [\#1013](https://github.com/apache/arrow-datafusion/pull/1013) ([rdettai](https://github.com/rdettai))
-- Reorganize table providers by table format [\#1010](https://github.com/apache/arrow-datafusion/pull/1010) ([rdettai](https://github.com/rdettai))
-- Move CBOs and Statistics to physical plan [\#965](https://github.com/apache/arrow-datafusion/pull/965) ([rdettai](https://github.com/rdettai))
-- Update to sqlparser v 0.10.0 [\#934](https://github.com/apache/arrow-datafusion/pull/934) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- FilePartition and PartitionedFile for scanning flexibility [\#932](https://github.com/apache/arrow-datafusion/pull/932) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([yjshen](https://github.com/yjshen))
-- Improve SQLMetric APIs, port existing metrics [\#908](https://github.com/apache/arrow-datafusion/pull/908) ([alamb](https://github.com/alamb))
-- Add support for EXPLAIN ANALYZE [\#858](https://github.com/apache/arrow-datafusion/pull/858) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- Rename concurrency to target\_partitions [\#706](https://github.com/apache/arrow-datafusion/pull/706) ([andygrove](https://github.com/andygrove))
-
-**Implemented enhancements:**
-
-- Update datafusion-cli to support Ballista, or implement new ballista-cli [\#886](https://github.com/apache/arrow-datafusion/issues/886)
-- Prepare Ballista crates for publishing [\#509](https://github.com/apache/arrow-datafusion/issues/509)
-- Add drop table support [\#1266](https://github.com/apache/arrow-datafusion/pull/1266) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([viirya](https://github.com/viirya))
-- use arrow 6.1.0 [\#1255](https://github.com/apache/arrow-datafusion/pull/1255) ([Jimexist](https://github.com/Jimexist))
-- Add support for `create table as` via MemTable [\#1243](https://github.com/apache/arrow-datafusion/pull/1243) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan))
-- add values list expression [\#1165](https://github.com/apache/arrow-datafusion/pull/1165) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Jimexist](https://github.com/Jimexist))
-- Multiple files per partitions for CSV Avro Json [\#1138](https://github.com/apache/arrow-datafusion/pull/1138) ([rdettai](https://github.com/rdettai))
-- Implement INTERSECT & INTERSECT DISTINCT [\#1135](https://github.com/apache/arrow-datafusion/pull/1135) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
-- Simplify file struct abstractions [\#1120](https://github.com/apache/arrow-datafusion/pull/1120) ([rdettai](https://github.com/rdettai))
-- Implement `is [not] distinct from` [\#1117](https://github.com/apache/arrow-datafusion/pull/1117) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Dandandan](https://github.com/Dandandan))
-- add digest\(utf8, method\) function and refactor all current hash digest functions [\#1090](https://github.com/apache/arrow-datafusion/pull/1090) ([Jimexist](https://github.com/Jimexist))
-- \[crypto\] add `blake3` algorithm to `digest` function [\#1086](https://github.com/apache/arrow-datafusion/pull/1086) ([Jimexist](https://github.com/Jimexist))
-- \[crypto\] add blake2b and blake2s functions [\#1081](https://github.com/apache/arrow-datafusion/pull/1081) ([Jimexist](https://github.com/Jimexist))
--  Update sqlparser-rs to 0.11 [\#1052](https://github.com/apache/arrow-datafusion/pull/1052) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([alamb](https://github.com/alamb))
-- remove hard coded partition count in ballista logicalplan deserialization [\#1044](https://github.com/apache/arrow-datafusion/pull/1044) ([xudong963](https://github.com/xudong963))
-- Indexed field access for List [\#1006](https://github.com/apache/arrow-datafusion/pull/1006) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki))
-- Update DataFusion to arrow 6.0 [\#984](https://github.com/apache/arrow-datafusion/pull/984) ([alamb](https://github.com/alamb))
-- Implement Display for Expr, improve operator display [\#971](https://github.com/apache/arrow-datafusion/pull/971) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([matthewmturner](https://github.com/matthewmturner))
-- ObjectStore API to read from remote storage systems [\#950](https://github.com/apache/arrow-datafusion/pull/950) ([yjshen](https://github.com/yjshen))
-- fixes \#933 replace placeholder fmt\_as fr ExecutionPlan impls [\#939](https://github.com/apache/arrow-datafusion/pull/939) ([tiphaineruy](https://github.com/tiphaineruy))
-- Support `NotLike` in Ballista [\#916](https://github.com/apache/arrow-datafusion/pull/916) ([Dandandan](https://github.com/Dandandan))
-- Avro Table Provider [\#910](https://github.com/apache/arrow-datafusion/pull/910) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([Igosuki](https://github.com/Igosuki))
-- Add BaselineMetrics, Timestamp metrics, add for `CoalescePartitionsExec`, rename output\_time -\> elapsed\_compute [\#909](https://github.com/apache/arrow-datafusion/pull/909) ([alamb](https://github.com/alamb))
-- \[Ballista\] Add executor last seen info to the ui [\#895](https://github.com/apache/arrow-datafusion/pull/895) ([msathis](https://github.com/msathis))
-- add cross join support to ballista [\#891](https://github.com/apache/arrow-datafusion/pull/891) ([houqp](https://github.com/houqp))
-- Add Ballista support to DataFusion CLI [\#889](https://github.com/apache/arrow-datafusion/pull/889) ([andygrove](https://github.com/andygrove))
-- Add support for PostgreSQL regex match [\#870](https://github.com/apache/arrow-datafusion/pull/870) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([b41sh](https://github.com/b41sh))
-
-**Fixed bugs:**
-
-- Test execution\_plans::shuffle\_writer::tests::test Fail [\#1040](https://github.com/apache/arrow-datafusion/issues/1040)
-- Integration test fails to build docker images [\#918](https://github.com/apache/arrow-datafusion/issues/918)
-- Ballista: Remove hard-coded concurrency from logical plan serde code [\#708](https://github.com/apache/arrow-datafusion/issues/708)
-- How can I make ballista distributed compute work? [\#327](https://github.com/apache/arrow-datafusion/issues/327)
-- fix subquery alias [\#1067](https://github.com/apache/arrow-datafusion/pull/1067) [[sql](https://github.com/apache/arrow-datafusion/labels/sql)] ([xudong963](https://github.com/xudong963))
-- Fix compilation for ballista in stand-alone mode [\#1008](https://github.com/apache/arrow-datafusion/pull/1008) ([Igosuki](https://github.com/Igosuki))
-
-**Documentation updates:**
-
-- Add Ballista roadmap [\#1166](https://github.com/apache/arrow-datafusion/pull/1166) ([andygrove](https://github.com/andygrove))
-- Adds note on compatible rust version [\#1097](https://github.com/apache/arrow-datafusion/pull/1097) ([1nF0rmed](https://github.com/1nF0rmed))
-- implement `approx_distinct` function using HyperLogLog [\#1087](https://github.com/apache/arrow-datafusion/pull/1087) ([Jimexist](https://github.com/Jimexist))
-- Improve User Guide [\#954](https://github.com/apache/arrow-datafusion/pull/954) ([andygrove](https://github.com/andygrove))
-- Update plan\_query\_stages doc [\#951](https://github.com/apache/arrow-datafusion/pull/951) ([rdettai](https://github.com/rdettai))
-- \[DataFusion\] -  Add show and show\_limit function for DataFrame [\#923](https://github.com/apache/arrow-datafusion/pull/923) ([francis-du](https://github.com/francis-du))
-- update docs related to protoc and optional syntax [\#902](https://github.com/apache/arrow-datafusion/pull/902) ([Jimexist](https://github.com/Jimexist))
-- Improve Ballista crate README content [\#878](https://github.com/apache/arrow-datafusion/pull/878) ([andygrove](https://github.com/andygrove))
-
-**Performance improvements:**
-
-- optimize build profile for datafusion python binding, cli and ballista [\#1137](https://github.com/apache/arrow-datafusion/pull/1137) ([houqp](https://github.com/houqp))
-
-**Closed issues:**
-
-- InList expr with NULL literals do not work [\#1190](https://github.com/apache/arrow-datafusion/issues/1190)
-- update the homepage README to include values, `approx_distinct`, etc. [\#1171](https://github.com/apache/arrow-datafusion/issues/1171)
-- \[Python\]: Inconsistencies with Python package name  [\#1011](https://github.com/apache/arrow-datafusion/issues/1011)
-- Wanting to contribute to project where to start? [\#983](https://github.com/apache/arrow-datafusion/issues/983)
-- delete redundant code [\#973](https://github.com/apache/arrow-datafusion/issues/973)
-- How to build DataFusion python wheel  [\#853](https://github.com/apache/arrow-datafusion/issues/853)
-- Produce a design for a metrics framework [\#21](https://github.com/apache/arrow-datafusion/issues/21)
-
-**Merged pull requests:**
-
-- \[nit\] simplify ballista executor `CollectExec` impl codes [\#1140](https://github.com/apache/arrow-datafusion/pull/1140) ([panarch](https://github.com/panarch))
-
-
-For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)
-
-## [ballista-0.5.0](https://github.com/apache/arrow-datafusion/tree/ballista-0.5.0) (2021-08-10)
-
-[Full Changelog](https://github.com/apache/arrow-datafusion/compare/4.0.0...ballista-0.5.0)
-
-**Breaking changes:**
-
-- \[ballista\] support date\_part and date\_turnc ser/de, pass tpch 7 [\#840](https://github.com/apache/arrow-datafusion/pull/840) ([houqp](https://github.com/houqp))
-- Box ScalarValue:Lists, reduce size by half size [\#788](https://github.com/apache/arrow-datafusion/pull/788) ([alamb](https://github.com/alamb))
-- Support DataFrame.collect for Ballista DataFrames [\#785](https://github.com/apache/arrow-datafusion/pull/785) ([andygrove](https://github.com/andygrove))
-- JOIN conditions are order dependent [\#778](https://github.com/apache/arrow-datafusion/pull/778) ([seddonm1](https://github.com/seddonm1))
-- UnresolvedShuffleExec should represent a single shuffle [\#727](https://github.com/apache/arrow-datafusion/pull/727) ([andygrove](https://github.com/andygrove))
-- Ballista: Make shuffle partitions configurable in benchmarks [\#702](https://github.com/apache/arrow-datafusion/pull/702) ([andygrove](https://github.com/andygrove))
-- Rename MergeExec to CoalescePartitionsExec [\#635](https://github.com/apache/arrow-datafusion/pull/635) ([andygrove](https://github.com/andygrove))
-- Ballista: Rename QueryStageExec to ShuffleWriterExec [\#633](https://github.com/apache/arrow-datafusion/pull/633) ([andygrove](https://github.com/andygrove))
-- fix 593, reduce cloning by taking ownership in logical planner's `from` fn [\#610](https://github.com/apache/arrow-datafusion/pull/610) ([Jimexist](https://github.com/Jimexist))
-- fix join column handling logic for `On` and `Using` constraints [\#605](https://github.com/apache/arrow-datafusion/pull/605) ([houqp](https://github.com/houqp))
-- Move ballista standalone mode to client [\#589](https://github.com/apache/arrow-datafusion/pull/589) ([edrevo](https://github.com/edrevo))
-- Ballista: Implement map-side shuffle [\#543](https://github.com/apache/arrow-datafusion/pull/543) ([andygrove](https://github.com/andygrove))
-- ShuffleReaderExec now supports multiple locations per partition [\#541](https://github.com/apache/arrow-datafusion/pull/541) ([andygrove](https://github.com/andygrove))
-- Make external hostname in executor optional [\#232](https://github.com/apache/arrow-datafusion/pull/232) ([edrevo](https://github.com/edrevo))
-- Remove namespace from executors [\#75](https://github.com/apache/arrow-datafusion/pull/75) ([edrevo](https://github.com/edrevo))
-- Support qualified columns in queries [\#55](https://github.com/apache/arrow-datafusion/pull/55) ([houqp](https://github.com/houqp))
-- Read CSV format text from stdin or memory [\#54](https://github.com/apache/arrow-datafusion/pull/54) ([heymind](https://github.com/heymind))
-- Remove Ballista DataFrame [\#48](https://github.com/apache/arrow-datafusion/pull/48) ([andygrove](https://github.com/andygrove))
-- Use atomics for SQLMetric implementation, remove unused name field [\#25](https://github.com/apache/arrow-datafusion/pull/25) ([returnString](https://github.com/returnString))
-
-**Implemented enhancements:**
-
-- Add crate documentation for Ballista crates [\#830](https://github.com/apache/arrow-datafusion/issues/830)
-- Support DataFrame.collect for Ballista DataFrames [\#787](https://github.com/apache/arrow-datafusion/issues/787)
-- Ballista: Prep for supporting shuffle correctly, part one [\#736](https://github.com/apache/arrow-datafusion/issues/736)
-- Ballista: Implement physical plan serde for ShuffleWriterExec [\#710](https://github.com/apache/arrow-datafusion/issues/710)
-- Ballista: Finish implementing shuffle mechanism [\#707](https://github.com/apache/arrow-datafusion/issues/707)
-- Rename QueryStageExec to ShuffleWriterExec [\#542](https://github.com/apache/arrow-datafusion/issues/542)
-- Ballista ShuffleReaderExec should be able to read from multiple locations per partition [\#540](https://github.com/apache/arrow-datafusion/issues/540)
-- \[Ballista\] Use deployments in k8s user guide [\#473](https://github.com/apache/arrow-datafusion/issues/473)
-- Ballista refactor QueryStageExec in preparation for map-side shuffle [\#458](https://github.com/apache/arrow-datafusion/issues/458)
-- Ballista: Implement map-side of shuffle [\#456](https://github.com/apache/arrow-datafusion/issues/456)
-- Refactor Ballista to separate Flight logic from execution logic [\#449](https://github.com/apache/arrow-datafusion/issues/449)
-- Use published versions of arrow rather than github shas [\#393](https://github.com/apache/arrow-datafusion/issues/393)
-- BallistaContext::collect\(\) logging is too noisy [\#352](https://github.com/apache/arrow-datafusion/issues/352)
-- Update Ballista to use new physical plan formatter utility [\#343](https://github.com/apache/arrow-datafusion/issues/343)
-- Add Ballista Getting Started documentation [\#329](https://github.com/apache/arrow-datafusion/issues/329)
-- Remove references to ballistacompute Docker Hub repo [\#325](https://github.com/apache/arrow-datafusion/issues/325)
-- Implement scalable distributed joins [\#63](https://github.com/apache/arrow-datafusion/issues/63)
-- Remove hard-coded Ballista version from scripts [\#32](https://github.com/apache/arrow-datafusion/issues/32)
-- Implement streaming versions of Dataframe.collect methods [\#789](https://github.com/apache/arrow-datafusion/pull/789) ([andygrove](https://github.com/andygrove))
-- Ballista shuffle is finally working as intended, providing scalable distributed joins [\#750](https://github.com/apache/arrow-datafusion/pull/750) ([andygrove](https://github.com/andygrove))
-- Update to use arrow 5.0 [\#721](https://github.com/apache/arrow-datafusion/pull/721) ([alamb](https://github.com/alamb))
-- Implement serde for ShuffleWriterExec [\#712](https://github.com/apache/arrow-datafusion/pull/712) ([andygrove](https://github.com/andygrove))
-- dedup using join column in wildcard expansion [\#678](https://github.com/apache/arrow-datafusion/pull/678) ([houqp](https://github.com/houqp))
-- Implement metrics for shuffle read and write [\#676](https://github.com/apache/arrow-datafusion/pull/676) ([andygrove](https://github.com/andygrove))
-- Remove hard-coded PartitionMode from Ballista serde [\#637](https://github.com/apache/arrow-datafusion/pull/637) ([andygrove](https://github.com/andygrove))
-- Ballista: Implement scalable distributed joins [\#634](https://github.com/apache/arrow-datafusion/pull/634) ([andygrove](https://github.com/andygrove))
-- Add Keda autoscaling for ballista in k8s [\#586](https://github.com/apache/arrow-datafusion/pull/586) ([edrevo](https://github.com/edrevo))
-- Add some resiliency to lost executors [\#568](https://github.com/apache/arrow-datafusion/pull/568) ([edrevo](https://github.com/edrevo))
-- Add `partition by` constructs in window functions and modify logical planning [\#501](https://github.com/apache/arrow-datafusion/pull/501) ([Jimexist](https://github.com/Jimexist))
-- Support anti join [\#482](https://github.com/apache/arrow-datafusion/pull/482) ([Dandandan](https://github.com/Dandandan))
-- add `order by` construct in window function and logical plans [\#463](https://github.com/apache/arrow-datafusion/pull/463) ([Jimexist](https://github.com/Jimexist))
-- Refactor Ballista executor so that FlightService delegates to an Executor struct [\#450](https://github.com/apache/arrow-datafusion/pull/450) ([andygrove](https://github.com/andygrove))
-- implement lead and lag built-in window function [\#429](https://github.com/apache/arrow-datafusion/pull/429) ([Jimexist](https://github.com/Jimexist))
-- Implement fmt\_as for ShuffleReaderExec [\#400](https://github.com/apache/arrow-datafusion/pull/400) ([andygrove](https://github.com/andygrove))
-- Add window expression part 1 - logical and physical planning, structure, to/from proto, and explain, for empty over clause only [\#334](https://github.com/apache/arrow-datafusion/pull/334) ([Jimexist](https://github.com/Jimexist))
-- \[breaking change\] fix 265, log should be log10, and add ln [\#271](https://github.com/apache/arrow-datafusion/pull/271) ([Jimexist](https://github.com/Jimexist))
-- Allow table providers to indicate their type for catalog metadata [\#205](https://github.com/apache/arrow-datafusion/pull/205) ([returnString](https://github.com/returnString))
-- Add query 19 to TPC-H regression tests [\#59](https://github.com/apache/arrow-datafusion/pull/59) ([Dandandan](https://github.com/Dandandan))
-- Use arrow eq kernels in CaseWhen expression evaluation [\#52](https://github.com/apache/arrow-datafusion/pull/52) ([Dandandan](https://github.com/Dandandan))
-- Add option param for standalone mode [\#42](https://github.com/apache/arrow-datafusion/pull/42) ([djKooks](https://github.com/djKooks))
-- \[DataFusion\] Optimize hash join inner workings, null handling fix [\#24](https://github.com/apache/arrow-datafusion/pull/24) ([Dandandan](https://github.com/Dandandan))
-- \[Ballista\] Docker files for ui [\#22](https://github.com/apache/arrow-datafusion/pull/22) ([msathis](https://github.com/msathis))
-
-**Fixed bugs:**
-
-- Ballista: TPC-H q3 @ SF=1000 never completes [\#835](https://github.com/apache/arrow-datafusion/issues/835)
-- Ballista does not support MIN/MAX aggregate functions [\#832](https://github.com/apache/arrow-datafusion/issues/832)
-- Ballista docker images fail to build [\#828](https://github.com/apache/arrow-datafusion/issues/828)
-- Ballista: UnresolvedShuffleExec should only have a single stage\_id [\#726](https://github.com/apache/arrow-datafusion/issues/726)
-- Ballista integration tests are failing [\#623](https://github.com/apache/arrow-datafusion/issues/623)
-- Integration test build failure due to arrow-rs using unstable feature [\#596](https://github.com/apache/arrow-datafusion/issues/596)
-- `cargo build` cannot build the project [\#531](https://github.com/apache/arrow-datafusion/issues/531)
-- ShuffleReaderExec does not get formatted correctly in displayable physical plan [\#399](https://github.com/apache/arrow-datafusion/issues/399)
-- Implement serde for MIN and MAX [\#833](https://github.com/apache/arrow-datafusion/pull/833) ([andygrove](https://github.com/andygrove))
-- Ballista: Prep for fixing shuffle mechansim, part 1 [\#738](https://github.com/apache/arrow-datafusion/pull/738) ([andygrove](https://github.com/andygrove))
-- Ballista: Shuffle write bug fix [\#714](https://github.com/apache/arrow-datafusion/pull/714) ([andygrove](https://github.com/andygrove))
-- honor table name for csv/parquet scan in ballista plan serde [\#629](https://github.com/apache/arrow-datafusion/pull/629) ([houqp](https://github.com/houqp))
-- MINOR: Fix integration tests by adding datafusion-cli module to docker image [\#322](https://github.com/apache/arrow-datafusion/pull/322) ([andygrove](https://github.com/andygrove))
-
-**Documentation updates:**
-
-- Add minimal crate documentation for Ballista crates [\#831](https://github.com/apache/arrow-datafusion/pull/831) ([andygrove](https://github.com/andygrove))
-- Add Ballista examples [\#775](https://github.com/apache/arrow-datafusion/pull/775) ([andygrove](https://github.com/andygrove))
-- Update ballista.proto link in architecture doc [\#502](https://github.com/apache/arrow-datafusion/pull/502) ([terrycorley](https://github.com/terrycorley))
-- Update k8s user guide to use deployments [\#474](https://github.com/apache/arrow-datafusion/pull/474) ([edrevo](https://github.com/edrevo))
-- use prettier to format md files [\#367](https://github.com/apache/arrow-datafusion/pull/367) ([Jimexist](https://github.com/Jimexist))
-- Make it easier for developers to find Ballista documentation [\#330](https://github.com/apache/arrow-datafusion/pull/330) ([andygrove](https://github.com/andygrove))
-- Instructions for cross-compiling Ballista to the Raspberry Pi [\#263](https://github.com/apache/arrow-datafusion/pull/263) ([andygrove](https://github.com/andygrove))
-- Add install guide in README [\#236](https://github.com/apache/arrow-datafusion/pull/236) ([djKooks](https://github.com/djKooks))
-
-**Performance improvements:**
-
-- Ballista: Avoid sleeping between polling for tasks [\#698](https://github.com/apache/arrow-datafusion/pull/698) ([Dandandan](https://github.com/Dandandan))
-- Make BallistaContext::collect streaming [\#535](https://github.com/apache/arrow-datafusion/pull/535) ([edrevo](https://github.com/edrevo))
-
-**Closed issues:**
-
-- Confirm git tagging strategy for releases [\#770](https://github.com/apache/arrow-datafusion/issues/770)
-- arrow::util::pretty::pretty\_format\_batches missing [\#769](https://github.com/apache/arrow-datafusion/issues/769)
-- move the `assert_batches_eq!` macros to a non part of datafusion [\#745](https://github.com/apache/arrow-datafusion/issues/745)
-- fix an issue where aliases are not respected in generating downstream schemas in window expr [\#592](https://github.com/apache/arrow-datafusion/issues/592)
-- make the planner to print more succinct and useful information in window function explain clause [\#526](https://github.com/apache/arrow-datafusion/issues/526)
-- move window frame module to be in `logical_plan` [\#517](https://github.com/apache/arrow-datafusion/issues/517)
-- use a more rust idiomatic way of handling nth\_value [\#448](https://github.com/apache/arrow-datafusion/issues/448)
-- Make Ballista not depend on arrow directly [\#446](https://github.com/apache/arrow-datafusion/issues/446)
-- create a test with more than one partition for window functions [\#435](https://github.com/apache/arrow-datafusion/issues/435)
-- Implement hash-partitioned hash aggregate [\#27](https://github.com/apache/arrow-datafusion/issues/27)
-- Consider using GitHub pages for DataFusion/Ballista documentation [\#18](https://github.com/apache/arrow-datafusion/issues/18)
-- Add Ballista to default cargo workspace [\#17](https://github.com/apache/arrow-datafusion/issues/17)
-- Update "repository" in Cargo.toml [\#16](https://github.com/apache/arrow-datafusion/issues/16)
-- Consolidate TPC-H benchmarks [\#6](https://github.com/apache/arrow-datafusion/issues/6)
-- \[Ballista\] Fix integration test script [\#4](https://github.com/apache/arrow-datafusion/issues/4)
-- Ballista should not have separate DataFrame implementation [\#2](https://github.com/apache/arrow-datafusion/issues/2)
-
-**Merged pull requests:**
-
-- Change datatype of tpch keys from Int32 to UInt64 to support sf=1000 [\#836](https://github.com/apache/arrow-datafusion/pull/836) ([andygrove](https://github.com/andygrove))
-- Add ballista-examples to docker build [\#829](https://github.com/apache/arrow-datafusion/pull/829) ([andygrove](https://github.com/andygrove))
-- Update dependencies: prost to 0.8 and tonic to 0.5 [\#818](https://github.com/apache/arrow-datafusion/pull/818) ([alamb](https://github.com/alamb))
-- Move `hash_array` into hash\_utils.rs [\#807](https://github.com/apache/arrow-datafusion/pull/807) ([alamb](https://github.com/alamb))
-- Fix: Update clippy lints for Rust 1.54 [\#794](https://github.com/apache/arrow-datafusion/pull/794) ([alamb](https://github.com/alamb))
-- MINOR: Remove unused Ballista query execution code path [\#732](https://github.com/apache/arrow-datafusion/pull/732) ([andygrove](https://github.com/andygrove))
-- \[fix\] benchmark run with compose [\#666](https://github.com/apache/arrow-datafusion/pull/666) ([rdettai](https://github.com/rdettai))
-- bring back dev scripts for ballista [\#648](https://github.com/apache/arrow-datafusion/pull/648) ([Jimexist](https://github.com/Jimexist))
-- Remove unnecessary mutex [\#639](https://github.com/apache/arrow-datafusion/pull/639) ([edrevo](https://github.com/edrevo))
-- round trip TPCH queries in tests [\#630](https://github.com/apache/arrow-datafusion/pull/630) ([houqp](https://github.com/houqp))
-- Fix build [\#627](https://github.com/apache/arrow-datafusion/pull/627) ([andygrove](https://github.com/andygrove))
-- in ballista also check for UI prettier changes [\#578](https://github.com/apache/arrow-datafusion/pull/578) ([Jimexist](https://github.com/Jimexist))
-- turn on clippy rule for needless borrow [\#545](https://github.com/apache/arrow-datafusion/pull/545) ([Jimexist](https://github.com/Jimexist))
-- reuse datafusion physical planner in ballista building from protobuf [\#532](https://github.com/apache/arrow-datafusion/pull/532) ([Jimexist](https://github.com/Jimexist))
-- update cargo.toml in python crate and fix unit test due to hash joins [\#483](https://github.com/apache/arrow-datafusion/pull/483) ([Jimexist](https://github.com/Jimexist))
-- make `VOLUME` declaration in tpch datagen docker absolute [\#466](https://github.com/apache/arrow-datafusion/pull/466) ([crepererum](https://github.com/crepererum))
-- Refactor QueryStageExec in preparation for implementing map-side shuffle [\#459](https://github.com/apache/arrow-datafusion/pull/459) ([andygrove](https://github.com/andygrove))
-- Simplified usage of `use arrow` in ballista. [\#447](https://github.com/apache/arrow-datafusion/pull/447) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- Benchmark subcommand to distinguish between DataFusion and Ballista [\#402](https://github.com/apache/arrow-datafusion/pull/402) ([jgoday](https://github.com/jgoday))
-- \#352: BallistaContext::collect\(\) logging is too noisy [\#394](https://github.com/apache/arrow-datafusion/pull/394) ([jgoday](https://github.com/jgoday))
-- cleanup function return type fn [\#350](https://github.com/apache/arrow-datafusion/pull/350) ([Jimexist](https://github.com/Jimexist))
-- Update Ballista to use new physical plan formatter utility [\#344](https://github.com/apache/arrow-datafusion/pull/344) ([andygrove](https://github.com/andygrove))
-- Update arrow dependencies again [\#341](https://github.com/apache/arrow-datafusion/pull/341) ([alamb](https://github.com/alamb))
-- Remove references to Ballista Docker images published to ballistacompute Docker Hub repo [\#326](https://github.com/apache/arrow-datafusion/pull/326) ([andygrove](https://github.com/andygrove))
-- Update arrow-rs deps [\#317](https://github.com/apache/arrow-datafusion/pull/317) ([alamb](https://github.com/alamb))
-- Update arrow deps [\#269](https://github.com/apache/arrow-datafusion/pull/269) ([alamb](https://github.com/alamb))
-- Enable redundant\_field\_names clippy lint [\#261](https://github.com/apache/arrow-datafusion/pull/261) ([Dandandan](https://github.com/Dandandan))
-- Update arrow-rs deps \(to fix build due to flatbuffers update\) [\#224](https://github.com/apache/arrow-datafusion/pull/224) ([alamb](https://github.com/alamb))
-- update arrow-rs deps to latest master [\#216](https://github.com/apache/arrow-datafusion/pull/216) ([alamb](https://github.com/alamb))
-
-
-
-\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
diff --git a/ballista/README.md b/ballista/README.md
deleted file mode 100644
index 1fc3bdbf0..000000000
--- a/ballista/README.md
+++ /dev/null
@@ -1,71 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Ballista: Distributed Compute with Apache Arrow and DataFusion
-
-Ballista is a distributed compute platform primarily implemented in Rust, and powered by Apache Arrow and
-DataFusion. It is built on an architecture that allows other programming languages (such as Python, C++, and
-Java) to be supported as first-class citizens without paying a penalty for serialization costs.
-
-The foundational technologies in Ballista are:
-
-- [Apache Arrow](https://arrow.apache.org/) memory model and compute kernels for efficient processing of data.
-- [Apache Arrow Flight Protocol](https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/) for efficient
-  data transfer between processes.
-- [Google Protocol Buffers](https://developers.google.com/protocol-buffers) for serializing query plans.
-- [Docker](https://www.docker.com/) for packaging up executors along with user-defined code.
-
-Ballista can be deployed as a standalone cluster and also supports [Kubernetes](https://kubernetes.io/). In either
-case, the scheduler can be configured to use [etcd](https://etcd.io/) as a backing store to (eventually) provide
-redundancy in the case of a scheduler failing.
-
-# Getting Started
-
-Refer to the core [Ballista crate README](rust/client/README.md) for the Getting Started guide.
-
-## Distributed Scheduler Overview
-
-Ballista uses the DataFusion query execution framework to create a physical plan and then transforms it into a
-distributed physical plan by breaking the query down into stages whenever the partitioning scheme changes.
-
-Specifically, any `RepartitionExec` operator is replaced with an `UnresolvedShuffleExec` and the child operator
-of the repartition operator is wrapped in a `ShuffleWriterExec` operator and scheduled for execution.
-
-Each executor polls the scheduler for the next task to run. Tasks are currently always `ShuffleWriterExec` operators
-and each task represents one _input_ partition that will be executed. The resulting batches are repartitioned
-according to the shuffle partitioning scheme and each _output_ partition is streamed to disk in Arrow IPC format.
-
-The scheduler will replace `UnresolvedShuffleExec` operators with `ShuffleReaderExec` operators once all shuffle
-tasks have completed. The `ShuffleReaderExec` operator connects to other executors as required using the Flight
-interface, and streams the shuffle IPC files.
-
-# How does this compare to Apache Spark?
-
-Ballista implements a similar design to Apache Spark, but there are some key differences.
-
-- The choice of Rust as the main execution language means that memory usage is deterministic and avoids the overhead of
-  GC pauses.
-- Ballista is designed from the ground up to use columnar data, enabling a number of efficiencies such as vectorized
-  processing (SIMD and GPU) and efficient compression. Although Spark does have some columnar support, it is still
-  largely row-based today.
-- The combination of Rust and Arrow provides excellent memory efficiency and memory usage can be 5x - 10x lower than
-  Apache Spark in some cases, which means that more processing can fit on a single node, reducing the overhead of
-  distributed compute.
-- The use of Apache Arrow as the memory model and network protocol means that data can be exchanged between executors
-  in any programming language with minimal serialization overhead.
diff --git a/ballista/docs/README.md b/ballista/docs/README.md
deleted file mode 100644
index 38d3db5df..000000000
--- a/ballista/docs/README.md
+++ /dev/null
@@ -1,34 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Ballista Developer Documentation
-
-This directory contains documentation for developers that are contributing to Ballista. If you are looking for
-end-user documentation for a published release, please start with the
-[DataFusion User Guide](../../docs/user-guide) instead.
-
-## Architecture & Design
-
-- Read the [Architecture Overview](architecture.md) to get an understanding of the scheduler and executor
-  processes and how distributed query execution works.
-
-## Build, Test, Release
-
-- Setting up a [development environment](dev-env.md).
-- [Integration Testing](integration-testing.md)
diff --git a/ballista/docs/architecture.md b/ballista/docs/architecture.md
deleted file mode 100644
index 2868d52b9..000000000
--- a/ballista/docs/architecture.md
+++ /dev/null
@@ -1,75 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Ballista Architecture
-
-## Overview
-
-Ballista allows queries to be executed in a distributed cluster. A cluster consists of one or
-more scheduler processes and one or more executor processes. See the following sections in this document for more
-details about these components.
-
-The scheduler accepts logical query plans and translates them into physical query plans using DataFusion and then
-runs a secondary planning/optimization process to translate the physical query plan into a distributed physical
-query plan.
-
-This process breaks a query down into a number of query stages that can be executed independently. There are
-dependencies between query stages and these dependencies form a directionally-acyclic graph (DAG) because a query
-stage cannot start until its child query stages have completed.
-
-Each query stage has one or more partitions that can be processed in parallel by the available
-executors in the cluster. This is the basic unit of scalability in Ballista.
-
-The following diagram shows the flow of requests and responses between the client, scheduler, and executor
-processes.
-
-![Query Execution Flow](images/query-execution.png)
-
-## Scheduler Process
-
-The scheduler process implements a gRPC interface (defined in
-[ballista.proto](../rust/core/proto/ballista.proto)). The interface provides the following methods:
-
-| Method               | Description                                                          |
-| -------------------- | -------------------------------------------------------------------- |
-| ExecuteQuery         | Submit a logical query plan or SQL query for execution               |
-| GetExecutorsMetadata | Retrieves a list of executors that have registered with a scheduler  |
-| GetFileMetadata      | Retrieve metadata about files available in the cluster file system   |
-| GetJobStatus         | Get the status of a submitted query                                  |
-| RegisterExecutor     | Executors call this method to register themselves with the scheduler |
-
-The scheduler can run in standalone mode, or can be run in clustered mode using etcd as backing store for state.
-
-## Executor Process
-
-The executor process implements the Apache Arrow Flight gRPC interface and is responsible for:
-
-- Executing query stages and persisting the results to disk in Apache Arrow IPC Format
-- Making query stage results available as Flights so that they can be retrieved by other executors as well as by
-  clients
-
-## Rust Client
-
-The Rust client provides a DataFrame API that is a thin wrapper around the DataFusion DataFrame and provides
-the means for a client to build a query plan for execution.
-
-The client executes the query plan by submitting an `ExecuteLogicalPlan` request to the scheduler and then calls
-`GetJobStatus` to check for completion. On completion, the client receives a list of locations for the Flights
-containing the results for the query and will then connect to the appropriate executor processes to retrieve
-those results.
diff --git a/ballista/docs/dev-env.md b/ballista/docs/dev-env.md
deleted file mode 100644
index 1c8605a06..000000000
--- a/ballista/docs/dev-env.md
+++ /dev/null
@@ -1,51 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Setting up a Rust development environment
-
-You will need a standard Rust development environment. The easiest way to achieve this is by using rustup: https://rustup.rs/
-
-## Install OpenSSL
-
-Follow instructions for [setting up OpenSSL](https://docs.rs/openssl/latest/openssl/). For Ubuntu users, the following
-command works.
-
-```bash
-sudo apt-get install pkg-config libssl-dev
-```
-
-For MacOS users, the following command works.
-
-```bash
-brew install openssl@1.1
-```
-
-## Install CMake
-
-You'll need cmake in order to compile some of ballista's dependencies. Ubuntu users can use the following command:
-
-```bash
-sudo apt-get install cmake
-```
-
-MacOS users can use the following command:
-
-```bash
-brew install cmake
-```
diff --git a/ballista/docs/images/query-execution.png b/ballista/docs/images/query-execution.png
deleted file mode 100644
index b35240282..000000000
Binary files a/ballista/docs/images/query-execution.png and /dev/null differ
diff --git a/ballista/docs/integration-testing.md b/ballista/docs/integration-testing.md
deleted file mode 100644
index 5c5b9ee9b..000000000
--- a/ballista/docs/integration-testing.md
+++ /dev/null
@@ -1,29 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Integration Testing
-
-We use the [DataFusion Benchmarks](https://github.com/apache/arrow-datafusion/tree/master/benchmarks) for integration
-testing.
-
-The integration tests can be executed by running the following command from the root of the DataFusion repository.
-
-```bash
-./dev/integration-tests.sh
-```
diff --git a/ballista/rust/.dockerignore b/ballista/rust/.dockerignore
deleted file mode 100644
index 96f99a522..000000000
--- a/ballista/rust/.dockerignore
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Turn .dockerignore to .dockerallow by excluding everything and explicitly
-# allowing specific files and directories. This enables us to quickly add
-# dependency files to the docker content without scanning the whole directory.
-# This setup requires to all of our docker containers have arrow's source
-# as a mounted directory.
-target
\ No newline at end of file
diff --git a/ballista/rust/.gitignore b/ballista/rust/.gitignore
deleted file mode 100644
index 97eec1640..000000000
--- a/ballista/rust/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-target
-temp
\ No newline at end of file
diff --git a/ballista/rust/client/Cargo.toml b/ballista/rust/client/Cargo.toml
deleted file mode 100644
index 828044450..000000000
--- a/ballista/rust/client/Cargo.toml
+++ /dev/null
@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "ballista"
-description = "Ballista Distributed Compute"
-license = "Apache-2.0"
-version = "0.7.0"
-homepage = "https://github.com/apache/arrow-datafusion"
-repository = "https://github.com/apache/arrow-datafusion"
-readme = "README.md"
-authors = ["Apache Arrow <de...@arrow.apache.org>"]
-edition = "2021"
-rust-version = "1.59"
-
-[dependencies]
-ballista-core = { path = "../core", version = "0.7.0" }
-ballista-executor = { path = "../executor", version = "0.7.0", optional = true }
-ballista-scheduler = { path = "../scheduler", version = "0.7.0", optional = true }
-
-datafusion = { path = "../../../datafusion/core", version = "8.0.0" }
-futures = "0.3"
-log = "0.4"
-parking_lot = "0.12"
-sqlparser = "0.17"
-tempfile = "3"
-tokio = "1.0"
-
-[features]
-default = []
-standalone = ["ballista-executor", "ballista-scheduler"]
diff --git a/ballista/rust/client/README.md b/ballista/rust/client/README.md
deleted file mode 100644
index ecf364f4f..000000000
--- a/ballista/rust/client/README.md
+++ /dev/null
@@ -1,133 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Ballista: Distributed Scheduler for Apache Arrow DataFusion
-
-Ballista is a distributed compute platform primarily implemented in Rust, and powered by Apache Arrow and
-DataFusion. It is built on an architecture that allows other programming languages (such as Python, C++, and
-Java) to be supported as first-class citizens without paying a penalty for serialization costs.
-
-The foundational technologies in Ballista are:
-
-- [Apache Arrow](https://arrow.apache.org/) memory model and compute kernels for efficient processing of data.
-- [Apache Arrow Flight Protocol](https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/) for efficient
-  data transfer between processes.
-- [Google Protocol Buffers](https://developers.google.com/protocol-buffers) for serializing query plans.
-- [Docker](https://www.docker.com/) for packaging up executors along with user-defined code.
-
-Ballista can be deployed as a standalone cluster and also supports [Kubernetes](https://kubernetes.io/). In either
-case, the scheduler can be configured to use [etcd](https://etcd.io/) as a backing store to (eventually) provide
-redundancy in the case of a scheduler failing.
-
-## Rust Version Compatbility
-
-This crate is tested with the latest stable version of Rust. We do not currrently test against other, older versions of the Rust compiler.
-
-## Starting a cluster
-
-There are numerous ways to start a Ballista cluster, including support for Docker and
-Kubernetes. For full documentation, refer to the
-[DataFusion User Guide](https://arrow.apache.org/datafusion/user-guide/introduction.html)
-
-A simple way to start a local cluster for testing purposes is to use cargo to install
-the scheduler and executor crates.
-
-```bash
-cargo install --locked ballista-scheduler
-cargo install --locked ballista-executor
-```
-
-With these crates installed, it is now possible to start a scheduler process.
-
-```bash
-RUST_LOG=info ballista-scheduler
-```
-
-The scheduler will bind to port 50050 by default.
-
-Next, start an executor processes in a new terminal session with the specified concurrency
-level.
-
-```bash
-RUST_LOG=info ballista-executor -c 4
-```
-
-The executor will bind to port 50051 by default. Additional executors can be started by
-manually specifying a bind port. For example:
-
-```bash
-RUST_LOG=info ballista-executor --bind-port 50052 -c 4
-```
-
-## Executing a query
-
-Ballista provides a `BallistaContext` as a starting point for creating queries. DataFrames can be created
-by invoking the `read_csv`, `read_parquet`, and `sql` methods.
-
-To build a simple ballista example, add the following dependencies to your `Cargo.toml` file:
-
-```toml
-[dependencies]
-ballista = "0.6"
-datafusion = "7.0"
-tokio = "1.0"
-```
-
-The following example runs a simple aggregate SQL query against a CSV file from the
-[New York Taxi and Limousine Commission](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page)
-data set.
-
-```rust,no_run
-use ballista::prelude::*;
-use datafusion::arrow::util::pretty;
-use datafusion::prelude::CsvReadOptions;
-
-#[tokio::main]
-async fn main() -> Result<()> {
-   // create configuration
-   let config = BallistaConfig::builder()
-       .set("ballista.shuffle.partitions", "4")
-       .build()?;
-
-   // connect to Ballista scheduler
-   let ctx = BallistaContext::remote("localhost", 50050, &config).await?;
-
-   // register csv file with the execution context
-   ctx.register_csv(
-       "tripdata",
-       "/path/to/yellow_tripdata_2020-01.csv",
-       CsvReadOptions::new(),
-   ).await?;
-
-   // execute the query
-   let df = ctx.sql(
-       "SELECT passenger_count, MIN(fare_amount), MAX(fare_amount), AVG(fare_amount), SUM(fare_amount)
-       FROM tripdata
-       GROUP BY passenger_count
-       ORDER BY passenger_count",
-   ).await?;
-
-   // collect the results and print them to stdout
-   let results = df.collect().await?;
-   pretty::print_batches(&results)?;
-   Ok(())
-}
-```
-
-More [examples](https://github.com/apache/arrow-datafusion/tree/master/ballista-examples) can be found in the arrow-datafusion repository.
diff --git a/ballista/rust/client/src/columnar_batch.rs b/ballista/rust/client/src/columnar_batch.rs
deleted file mode 100644
index 3431f5612..000000000
--- a/ballista/rust/client/src/columnar_batch.rs
+++ /dev/null
@@ -1,163 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{collections::HashMap, sync::Arc};
-
-use ballista_core::error::{ballista_error, Result};
-
-use datafusion::arrow::{
-    array::ArrayRef,
-    datatypes::{DataType, Schema},
-    record_batch::RecordBatch,
-};
-use datafusion::scalar::ScalarValue;
-
-pub type MaybeColumnarBatch = Result<Option<ColumnarBatch>>;
-
-/// Batch of columnar data.
-#[derive(Debug, Clone)]
-pub struct ColumnarBatch {
-    schema: Arc<Schema>,
-    columns: HashMap<String, ColumnarValue>,
-}
-
-impl ColumnarBatch {
-    pub fn from_arrow(batch: &RecordBatch) -> Self {
-        let columns = batch
-            .columns()
-            .iter()
-            .enumerate()
-            .map(|(i, array)| {
-                (
-                    batch.schema().field(i).name().clone(),
-                    ColumnarValue::Columnar(array.clone()),
-                )
-            })
-            .collect();
-
-        Self {
-            schema: batch.schema(),
-            columns,
-        }
-    }
-
-    pub fn from_values(values: &[ColumnarValue], schema: &Schema) -> Self {
-        let columns = schema
-            .fields()
-            .iter()
-            .enumerate()
-            .map(|(i, f)| (f.name().clone(), values[i].clone()))
-            .collect();
-
-        Self {
-            schema: Arc::new(schema.clone()),
-            columns,
-        }
-    }
-
-    pub fn to_arrow(&self) -> Result<RecordBatch> {
-        let arrays = self
-            .schema
-            .fields()
-            .iter()
-            .map(|c| {
-                match self.column(c.name())? {
-                    ColumnarValue::Columnar(array) => Ok(array.clone()),
-                    ColumnarValue::Scalar(_, _) => {
-                        // note that this can be implemented easily if needed
-                        Err(ballista_error("Cannot convert scalar value to Arrow array"))
-                    }
-                }
-            })
-            .collect::<Result<Vec<_>>>()?;
-
-        Ok(RecordBatch::try_new(self.schema.clone(), arrays)?)
-    }
-
-    pub fn schema(&self) -> Arc<Schema> {
-        self.schema.clone()
-    }
-
-    pub fn num_columns(&self) -> usize {
-        self.columns.len()
-    }
-
-    pub fn num_rows(&self) -> usize {
-        self.columns[self.schema.field(0).name()].len()
-    }
-
-    pub fn column(&self, name: &str) -> Result<&ColumnarValue> {
-        Ok(&self.columns[name])
-    }
-
-    pub fn memory_size(&self) -> usize {
-        self.columns.values().map(|c| c.memory_size()).sum()
-    }
-}
-
-/// A columnar value can either be a scalar value or an Arrow array.
-#[derive(Debug, Clone)]
-pub enum ColumnarValue {
-    Scalar(ScalarValue, usize),
-    Columnar(ArrayRef),
-}
-
-impl ColumnarValue {
-    pub fn len(&self) -> usize {
-        match self {
-            ColumnarValue::Scalar(_, n) => *n,
-            ColumnarValue::Columnar(array) => array.len(),
-        }
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    pub fn data_type(&self) -> &DataType {
-        match self {
-            ColumnarValue::Columnar(array) => array.data_type(),
-            ColumnarValue::Scalar(value, _) => match value {
-                ScalarValue::UInt8(_) => &DataType::UInt8,
-                ScalarValue::UInt16(_) => &DataType::UInt16,
-                ScalarValue::UInt32(_) => &DataType::UInt32,
-                ScalarValue::UInt64(_) => &DataType::UInt64,
-                ScalarValue::Int8(_) => &DataType::Int8,
-                ScalarValue::Int16(_) => &DataType::Int16,
-                ScalarValue::Int32(_) => &DataType::Int32,
-                ScalarValue::Int64(_) => &DataType::Int64,
-                ScalarValue::Float32(_) => &DataType::Float32,
-                ScalarValue::Float64(_) => &DataType::Float64,
-                _ => unimplemented!(),
-            },
-        }
-    }
-
-    pub fn to_arrow(&self) -> ArrayRef {
-        match self {
-            ColumnarValue::Columnar(array) => array.clone(),
-            ColumnarValue::Scalar(value, n) => value.to_array_of_size(*n),
-        }
-    }
-
-    pub fn memory_size(&self) -> usize {
-        match self {
-            ColumnarValue::Columnar(array) => array.get_array_memory_size(),
-            _ => 0,
-        }
-    }
-}
diff --git a/ballista/rust/client/src/context.rs b/ballista/rust/client/src/context.rs
deleted file mode 100644
index a413c5289..000000000
--- a/ballista/rust/client/src/context.rs
+++ /dev/null
@@ -1,947 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Distributed execution context.
-
-use log::info;
-use parking_lot::Mutex;
-use sqlparser::ast::Statement;
-use std::collections::HashMap;
-use std::fs;
-use std::path::PathBuf;
-use std::sync::Arc;
-
-use ballista_core::config::BallistaConfig;
-use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
-use ballista_core::serde::protobuf::{ExecuteQueryParams, KeyValuePair, LogicalPlanNode};
-use ballista_core::utils::create_df_ctx_with_ballista_query_planner;
-
-use datafusion::catalog::TableReference;
-use datafusion::dataframe::DataFrame;
-use datafusion::datasource::TableProvider;
-use datafusion::error::{DataFusionError, Result};
-use datafusion::logical_plan::{
-    source_as_provider, CreateExternalTable, FileType, LogicalPlan, TableScan,
-};
-use datafusion::prelude::{
-    AvroReadOptions, CsvReadOptions, ParquetReadOptions, SessionConfig, SessionContext,
-};
-use datafusion::sql::parser::{DFParser, Statement as DFStatement};
-
-struct BallistaContextState {
-    /// Ballista configuration
-    config: BallistaConfig,
-    /// Scheduler host
-    scheduler_host: String,
-    /// Scheduler port
-    scheduler_port: u16,
-    /// Tables that have been registered with this context
-    tables: HashMap<String, Arc<dyn TableProvider>>,
-}
-
-impl BallistaContextState {
-    pub fn new(
-        scheduler_host: String,
-        scheduler_port: u16,
-        config: &BallistaConfig,
-    ) -> Self {
-        Self {
-            config: config.clone(),
-            scheduler_host,
-            scheduler_port,
-            tables: HashMap::new(),
-        }
-    }
-
-    pub fn config(&self) -> &BallistaConfig {
-        &self.config
-    }
-}
-
-pub struct BallistaContext {
-    state: Arc<Mutex<BallistaContextState>>,
-    context: Arc<SessionContext>,
-}
-
-impl BallistaContext {
-    /// Create a context for executing queries against a remote Ballista scheduler instance
-    pub async fn remote(
-        host: &str,
-        port: u16,
-        config: &BallistaConfig,
-    ) -> ballista_core::error::Result<Self> {
-        let state = BallistaContextState::new(host.to_owned(), port, config);
-
-        let scheduler_url =
-            format!("http://{}:{}", &state.scheduler_host, state.scheduler_port);
-        info!(
-            "Connecting to Ballista scheduler at {}",
-            scheduler_url.clone()
-        );
-        let mut scheduler = SchedulerGrpcClient::connect(scheduler_url.clone())
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-
-        let remote_session_id = scheduler
-            .execute_query(ExecuteQueryParams {
-                query: None,
-                settings: config
-                    .settings()
-                    .iter()
-                    .map(|(k, v)| KeyValuePair {
-                        key: k.to_owned(),
-                        value: v.to_owned(),
-                    })
-                    .collect::<Vec<_>>(),
-                optional_session_id: None,
-            })
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?
-            .into_inner()
-            .session_id;
-
-        info!(
-            "Server side SessionContext created with session id: {}",
-            remote_session_id
-        );
-
-        let ctx = {
-            create_df_ctx_with_ballista_query_planner::<LogicalPlanNode>(
-                scheduler_url,
-                remote_session_id,
-                state.config(),
-            )
-        };
-
-        Ok(Self {
-            state: Arc::new(Mutex::new(state)),
-            context: Arc::new(ctx),
-        })
-    }
-
-    #[cfg(feature = "standalone")]
-    pub async fn standalone(
-        config: &BallistaConfig,
-        concurrent_tasks: usize,
-    ) -> ballista_core::error::Result<Self> {
-        use ballista_core::serde::protobuf::PhysicalPlanNode;
-        use ballista_core::serde::BallistaCodec;
-
-        log::info!("Running in local mode. Scheduler will be run in-proc");
-
-        let addr = ballista_scheduler::standalone::new_standalone_scheduler().await?;
-        let scheduler_url = format!("http://localhost:{}", addr.port());
-        let mut scheduler = loop {
-            match SchedulerGrpcClient::connect(scheduler_url.clone()).await {
-                Err(_) => {
-                    tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
-                    log::info!("Attempting to connect to in-proc scheduler...");
-                }
-                Ok(scheduler) => break scheduler,
-            }
-        };
-
-        let remote_session_id = scheduler
-            .execute_query(ExecuteQueryParams {
-                query: None,
-                settings: config
-                    .settings()
-                    .iter()
-                    .map(|(k, v)| KeyValuePair {
-                        key: k.to_owned(),
-                        value: v.to_owned(),
-                    })
-                    .collect::<Vec<_>>(),
-                optional_session_id: None,
-            })
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?
-            .into_inner()
-            .session_id;
-
-        info!(
-            "Server side SessionContext created with session id: {}",
-            remote_session_id
-        );
-
-        let ctx = {
-            create_df_ctx_with_ballista_query_planner::<LogicalPlanNode>(
-                scheduler_url,
-                remote_session_id,
-                config,
-            )
-        };
-
-        let default_codec: BallistaCodec<LogicalPlanNode, PhysicalPlanNode> =
-            BallistaCodec::default();
-
-        ballista_executor::new_standalone_executor(
-            scheduler,
-            concurrent_tasks,
-            default_codec,
-        )
-        .await?;
-
-        let state =
-            BallistaContextState::new("localhost".to_string(), addr.port(), config);
-
-        Ok(Self {
-            state: Arc::new(Mutex::new(state)),
-            context: Arc::new(ctx),
-        })
-    }
-
-    /// Create a DataFrame representing an Avro table scan
-    /// TODO fetch schema from scheduler instead of resolving locally
-    pub async fn read_avro(
-        &self,
-        path: &str,
-        options: AvroReadOptions<'_>,
-    ) -> Result<Arc<DataFrame>> {
-        // convert to absolute path because the executor likely has a different working directory
-        let path = PathBuf::from(path);
-        let path = fs::canonicalize(&path)?;
-
-        let ctx = self.context.clone();
-        let df = ctx.read_avro(path.to_str().unwrap(), options).await?;
-        Ok(df)
-    }
-
-    /// Create a DataFrame representing a Parquet table scan
-    /// TODO fetch schema from scheduler instead of resolving locally
-    pub async fn read_parquet(
-        &self,
-        path: &str,
-        options: ParquetReadOptions<'_>,
-    ) -> Result<Arc<DataFrame>> {
-        // convert to absolute path because the executor likely has a different working directory
-        let path = PathBuf::from(path);
-        let path = fs::canonicalize(&path)?;
-
-        let ctx = self.context.clone();
-        let df = ctx.read_parquet(path.to_str().unwrap(), options).await?;
-        Ok(df)
-    }
-
-    /// Create a DataFrame representing a CSV table scan
-    /// TODO fetch schema from scheduler instead of resolving locally
-    pub async fn read_csv(
-        &self,
-        path: &str,
-        options: CsvReadOptions<'_>,
-    ) -> Result<Arc<DataFrame>> {
-        // convert to absolute path because the executor likely has a different working directory
-        let path = PathBuf::from(path);
-        let path = fs::canonicalize(&path)?;
-
-        let ctx = self.context.clone();
-        let df = ctx.read_csv(path.to_str().unwrap(), options).await?;
-        Ok(df)
-    }
-
-    /// Register a DataFrame as a table that can be referenced from a SQL query
-    pub fn register_table(
-        &self,
-        name: &str,
-        table: Arc<dyn TableProvider>,
-    ) -> Result<()> {
-        let mut state = self.state.lock();
-        state.tables.insert(name.to_owned(), table);
-        Ok(())
-    }
-
-    pub async fn register_csv(
-        &self,
-        name: &str,
-        path: &str,
-        options: CsvReadOptions<'_>,
-    ) -> Result<()> {
-        match self.read_csv(path, options).await?.to_logical_plan()? {
-            LogicalPlan::TableScan(TableScan { source, .. }) => {
-                self.register_table(name, source_as_provider(&source)?)
-            }
-            _ => Err(DataFusionError::Internal("Expected tables scan".to_owned())),
-        }
-    }
-
-    pub async fn register_parquet(
-        &self,
-        name: &str,
-        path: &str,
-        options: ParquetReadOptions<'_>,
-    ) -> Result<()> {
-        match self.read_parquet(path, options).await?.to_logical_plan()? {
-            LogicalPlan::TableScan(TableScan { source, .. }) => {
-                self.register_table(name, source_as_provider(&source)?)
-            }
-            _ => Err(DataFusionError::Internal("Expected tables scan".to_owned())),
-        }
-    }
-
-    pub async fn register_avro(
-        &self,
-        name: &str,
-        path: &str,
-        options: AvroReadOptions<'_>,
-    ) -> Result<()> {
-        match self.read_avro(path, options).await?.to_logical_plan()? {
-            LogicalPlan::TableScan(TableScan { source, .. }) => {
-                self.register_table(name, source_as_provider(&source)?)
-            }
-            _ => Err(DataFusionError::Internal("Expected tables scan".to_owned())),
-        }
-    }
-
-    /// is a 'show *' sql
-    pub async fn is_show_statement(&self, sql: &str) -> Result<bool> {
-        let mut is_show_variable: bool = false;
-        let statements = DFParser::parse_sql(sql)?;
-
-        if statements.len() != 1 {
-            return Err(DataFusionError::NotImplemented(
-                "The context currently only supports a single SQL statement".to_string(),
-            ));
-        }
-
-        if let DFStatement::Statement(s) = &statements[0] {
-            let st: &Statement = s;
-            match st {
-                Statement::ShowVariable { .. } => {
-                    is_show_variable = true;
-                }
-                Statement::ShowColumns { .. } => {
-                    is_show_variable = true;
-                }
-                _ => {
-                    is_show_variable = false;
-                }
-            }
-        };
-
-        Ok(is_show_variable)
-    }
-
-    /// Create a DataFrame from a SQL statement.
-    ///
-    /// This method is `async` because queries of type `CREATE EXTERNAL TABLE`
-    /// might require the schema to be inferred.
-    pub async fn sql(&self, sql: &str) -> Result<Arc<DataFrame>> {
-        let mut ctx = self.context.clone();
-
-        let is_show = self.is_show_statement(sql).await?;
-        // the show tables、 show columns sql can not run at scheduler because the tables is store at client
-        if is_show {
-            let state = self.state.lock();
-            ctx = Arc::new(SessionContext::with_config(
-                SessionConfig::new().with_information_schema(
-                    state.config.default_with_information_schema(),
-                ),
-            ));
-        }
-
-        // register tables with DataFusion context
-        {
-            let state = self.state.lock();
-            for (name, prov) in &state.tables {
-                // ctx is shared between queries, check table exists or not before register
-                let table_ref = TableReference::Bare { table: name };
-                if !ctx.table_exist(table_ref)? {
-                    ctx.register_table(
-                        TableReference::Bare { table: name },
-                        Arc::clone(prov),
-                    )?;
-                }
-            }
-        }
-
-        let plan = ctx.create_logical_plan(sql)?;
-
-        match plan {
-            LogicalPlan::CreateExternalTable(CreateExternalTable {
-                ref schema,
-                ref name,
-                ref location,
-                ref file_type,
-                ref has_header,
-                ref delimiter,
-                ref table_partition_cols,
-                ref if_not_exists,
-            }) => {
-                let table_exists = ctx.table_exist(name.as_str())?;
-
-                match (if_not_exists, table_exists) {
-                    (_, false) => match file_type {
-                        FileType::CSV => {
-                            self.register_csv(
-                                name,
-                                location,
-                                CsvReadOptions::new()
-                                    .schema(&schema.as_ref().to_owned().into())
-                                    .has_header(*has_header)
-                                    .delimiter(*delimiter as u8)
-                                    .table_partition_cols(table_partition_cols.to_vec()),
-                            )
-                            .await?;
-                            Ok(Arc::new(DataFrame::new(ctx.state.clone(), &plan)))
-                        }
-                        FileType::Parquet => {
-                            self.register_parquet(
-                                name,
-                                location,
-                                ParquetReadOptions::default()
-                                    .table_partition_cols(table_partition_cols.to_vec()),
-                            )
-                            .await?;
-                            Ok(Arc::new(DataFrame::new(ctx.state.clone(), &plan)))
-                        }
-                        FileType::Avro => {
-                            self.register_avro(
-                                name,
-                                location,
-                                AvroReadOptions::default()
-                                    .table_partition_cols(table_partition_cols.to_vec()),
-                            )
-                            .await?;
-                            Ok(Arc::new(DataFrame::new(ctx.state.clone(), &plan)))
-                        }
-                        _ => Err(DataFusionError::NotImplemented(format!(
-                            "Unsupported file type {:?}.",
-                            file_type
-                        ))),
-                    },
-                    (true, true) => {
-                        Ok(Arc::new(DataFrame::new(ctx.state.clone(), &plan)))
-                    }
-                    (false, true) => Err(DataFusionError::Execution(format!(
-                        "Table '{:?}' already exists",
-                        name
-                    ))),
-                }
-            }
-            _ => ctx.sql(sql).await,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    #[tokio::test]
-    #[cfg(feature = "standalone")]
-    async fn test_standalone_mode() {
-        use super::*;
-        let context = BallistaContext::standalone(&BallistaConfig::new().unwrap(), 1)
-            .await
-            .unwrap();
-        let df = context.sql("SELECT 1;").await.unwrap();
-        df.collect().await.unwrap();
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "standalone")]
-    async fn test_ballista_show_tables() {
-        use super::*;
-        use std::fs::File;
-        use std::io::Write;
-        use tempfile::TempDir;
-        let context = BallistaContext::standalone(&BallistaConfig::new().unwrap(), 1)
-            .await
-            .unwrap();
-
-        let data = "Jorge,2018-12-13T12:12:10.011Z\n\
-                    Andrew,2018-11-13T17:11:10.011Z";
-
-        let tmp_dir = TempDir::new().unwrap();
-        let file_path = tmp_dir.path().join("timestamps.csv");
-
-        // scope to ensure the file is closed and written
-        {
-            File::create(&file_path)
-                .expect("creating temp file")
-                .write_all(data.as_bytes())
-                .expect("writing data");
-        }
-
-        let sql = format!(
-            "CREATE EXTERNAL TABLE csv_with_timestamps (
-                  name VARCHAR,
-                  ts TIMESTAMP
-              )
-              STORED AS CSV
-              LOCATION '{}'
-              ",
-            file_path.to_str().expect("path is utf8")
-        );
-
-        context.sql(sql.as_str()).await.unwrap();
-
-        let df = context.sql("show columns from csv_with_timestamps;").await;
-
-        assert!(df.is_err());
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "standalone")]
-    async fn test_show_tables_not_with_information_schema() {
-        use super::*;
-        use ballista_core::config::{
-            BallistaConfigBuilder, BALLISTA_WITH_INFORMATION_SCHEMA,
-        };
-        use std::fs::File;
-        use std::io::Write;
-        use tempfile::TempDir;
-        let config = BallistaConfigBuilder::default()
-            .set(BALLISTA_WITH_INFORMATION_SCHEMA, "true")
-            .build()
-            .unwrap();
-        let context = BallistaContext::standalone(&config, 1).await.unwrap();
-
-        let data = "Jorge,2018-12-13T12:12:10.011Z\n\
-                    Andrew,2018-11-13T17:11:10.011Z";
-
-        let tmp_dir = TempDir::new().unwrap();
-        let file_path = tmp_dir.path().join("timestamps.csv");
-
-        // scope to ensure the file is closed and written
-        {
-            File::create(&file_path)
-                .expect("creating temp file")
-                .write_all(data.as_bytes())
-                .expect("writing data");
-        }
-
-        let sql = format!(
-            "CREATE EXTERNAL TABLE csv_with_timestamps (
-                  name VARCHAR,
-                  ts TIMESTAMP
-              )
-              STORED AS CSV
-              LOCATION '{}'
-              ",
-            file_path.to_str().expect("path is utf8")
-        );
-
-        context.sql(sql.as_str()).await.unwrap();
-        let df = context.sql("show tables;").await;
-        assert!(df.is_ok());
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "standalone")]
-    #[ignore]
-    // Tracking: https://github.com/apache/arrow-datafusion/issues/1840
-    async fn test_task_stuck_when_referenced_task_failed() {
-        use super::*;
-        use datafusion::arrow::datatypes::Schema;
-        use datafusion::arrow::util::pretty;
-        use datafusion::datasource::file_format::csv::CsvFormat;
-        use datafusion::datasource::listing::{
-            ListingOptions, ListingTable, ListingTableConfig,
-        };
-
-        use ballista_core::config::{
-            BallistaConfigBuilder, BALLISTA_WITH_INFORMATION_SCHEMA,
-        };
-        let config = BallistaConfigBuilder::default()
-            .set(BALLISTA_WITH_INFORMATION_SCHEMA, "true")
-            .build()
-            .unwrap();
-        let context = BallistaContext::standalone(&config, 1).await.unwrap();
-
-        let testdata = datafusion::test_util::parquet_test_data();
-        context
-            .register_parquet(
-                "single_nan",
-                &format!("{}/single_nan.parquet", testdata),
-                ParquetReadOptions::default(),
-            )
-            .await
-            .unwrap();
-
-        {
-            let mut guard = context.state.lock();
-            let csv_table = guard.tables.get("single_nan");
-
-            if let Some(table_provide) = csv_table {
-                if let Some(listing_table) = table_provide
-                    .clone()
-                    .as_any()
-                    .downcast_ref::<ListingTable>()
-                {
-                    let x = listing_table.options();
-                    let error_options = ListingOptions {
-                        file_extension: x.file_extension.clone(),
-                        format: Arc::new(CsvFormat::default()),
-                        table_partition_cols: x.table_partition_cols.clone(),
-                        collect_stat: x.collect_stat,
-                        target_partitions: x.target_partitions,
-                    };
-
-                    let config = ListingTableConfig::new(
-                        listing_table.object_store().clone(),
-                        listing_table.table_path().to_string(),
-                    )
-                    .with_schema(Arc::new(Schema::new(vec![])))
-                    .with_listing_options(error_options);
-
-                    let error_table = ListingTable::try_new(config).unwrap();
-
-                    // change the table to an error table
-                    guard
-                        .tables
-                        .insert("single_nan".to_string(), Arc::new(error_table));
-                }
-            }
-        }
-
-        let df = context
-            .sql("select count(1) from single_nan;")
-            .await
-            .unwrap();
-        let results = df.collect().await.unwrap();
-        pretty::print_batches(&results);
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "standalone")]
-    async fn test_empty_exec_with_one_row() {
-        use crate::context::BallistaContext;
-        use ballista_core::config::{
-            BallistaConfigBuilder, BALLISTA_WITH_INFORMATION_SCHEMA,
-        };
-
-        let config = BallistaConfigBuilder::default()
-            .set(BALLISTA_WITH_INFORMATION_SCHEMA, "true")
-            .build()
-            .unwrap();
-        let context = BallistaContext::standalone(&config, 1).await.unwrap();
-
-        let sql = "select EXTRACT(year FROM to_timestamp('2020-09-08T12:13:14+00:00'));";
-
-        let df = context.sql(sql).await.unwrap();
-        assert!(!df.collect().await.unwrap().is_empty());
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "standalone")]
-    async fn test_union_and_union_all() {
-        use super::*;
-        use ballista_core::config::{
-            BallistaConfigBuilder, BALLISTA_WITH_INFORMATION_SCHEMA,
-        };
-        use datafusion::arrow::util::pretty::pretty_format_batches;
-        let config = BallistaConfigBuilder::default()
-            .set(BALLISTA_WITH_INFORMATION_SCHEMA, "true")
-            .build()
-            .unwrap();
-        let context = BallistaContext::standalone(&config, 1).await.unwrap();
-
-        let df = context
-            .sql("SELECT 1 as NUMBER union SELECT 1 as NUMBER;")
-            .await
-            .unwrap();
-        let res1 = df.collect().await.unwrap();
-        let expected1 = vec![
-            "+--------+",
-            "| number |",
-            "+--------+",
-            "| 1      |",
-            "+--------+",
-        ];
-        assert_eq!(
-            expected1,
-            pretty_format_batches(&*res1)
-                .unwrap()
-                .to_string()
-                .trim()
-                .lines()
-                .collect::<Vec<&str>>()
-        );
-        let expected2 = vec![
-            "+--------+",
-            "| number |",
-            "+--------+",
-            "| 1      |",
-            "| 1      |",
-            "+--------+",
-        ];
-        let df = context
-            .sql("SELECT 1 as NUMBER union all SELECT 1 as NUMBER;")
-            .await
-            .unwrap();
-        let res2 = df.collect().await.unwrap();
-        assert_eq!(
-            expected2,
-            pretty_format_batches(&*res2)
-                .unwrap()
-                .to_string()
-                .trim()
-                .lines()
-                .collect::<Vec<&str>>()
-        );
-    }
-
-    #[tokio::test]
-    #[cfg(feature = "standalone")]
-    async fn test_aggregate_func() {
-        use crate::context::BallistaContext;
-        use ballista_core::config::{
-            BallistaConfigBuilder, BALLISTA_WITH_INFORMATION_SCHEMA,
-        };
-        use datafusion::arrow;
-        use datafusion::arrow::util::pretty::pretty_format_batches;
-        use datafusion::prelude::ParquetReadOptions;
-
-        let config = BallistaConfigBuilder::default()
-            .set(BALLISTA_WITH_INFORMATION_SCHEMA, "true")
-            .build()
-            .unwrap();
-        let context = BallistaContext::standalone(&config, 1).await.unwrap();
-
-        let testdata = datafusion::test_util::parquet_test_data();
-        context
-            .register_parquet(
-                "test",
-                &format!("{}/alltypes_plain.parquet", testdata),
-                ParquetReadOptions::default(),
-            )
-            .await
-            .unwrap();
-
-        let df = context.sql("select min(\"id\") from test").await.unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+--------------+",
-            "| MIN(test.id) |",
-            "+--------------+",
-            "| 0            |",
-            "+--------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context.sql("select max(\"id\") from test").await.unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+--------------+",
-            "| MAX(test.id) |",
-            "+--------------+",
-            "| 7            |",
-            "+--------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context.sql("select SUM(\"id\") from test").await.unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+--------------+",
-            "| SUM(test.id) |",
-            "+--------------+",
-            "| 28           |",
-            "+--------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context.sql("select AVG(\"id\") from test").await.unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+--------------+",
-            "| AVG(test.id) |",
-            "+--------------+",
-            "| 3.5          |",
-            "+--------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context.sql("select COUNT(\"id\") from test").await.unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+----------------+",
-            "| COUNT(test.id) |",
-            "+----------------+",
-            "| 8              |",
-            "+----------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select approx_distinct(\"id\") from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+-------------------------+",
-            "| APPROXDISTINCT(test.id) |",
-            "+-------------------------+",
-            "| 8                       |",
-            "+-------------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select ARRAY_AGG(\"id\") from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+--------------------------+",
-            "| ARRAYAGG(test.id)        |",
-            "+--------------------------+",
-            "| [4, 5, 6, 7, 2, 3, 0, 1] |",
-            "+--------------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context.sql("select VAR(\"id\") from test").await.unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+-------------------+",
-            "| VARIANCE(test.id) |",
-            "+-------------------+",
-            "| 6.000000000000001 |",
-            "+-------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select VAR_POP(\"id\") from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+----------------------+",
-            "| VARIANCEPOP(test.id) |",
-            "+----------------------+",
-            "| 5.250000000000001    |",
-            "+----------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select VAR_SAMP(\"id\") from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+-------------------+",
-            "| VARIANCE(test.id) |",
-            "+-------------------+",
-            "| 6.000000000000001 |",
-            "+-------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select STDDEV(\"id\") from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+--------------------+",
-            "| STDDEV(test.id)    |",
-            "+--------------------+",
-            "| 2.4494897427831783 |",
-            "+--------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select STDDEV_SAMP(\"id\") from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+--------------------+",
-            "| STDDEV(test.id)    |",
-            "+--------------------+",
-            "| 2.4494897427831783 |",
-            "+--------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select COVAR(id, tinyint_col) from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+--------------------------------------+",
-            "| COVARIANCE(test.id,test.tinyint_col) |",
-            "+--------------------------------------+",
-            "| 0.28571428571428586                  |",
-            "+--------------------------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select CORR(id, tinyint_col) from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+---------------------------------------+",
-            "| CORRELATION(test.id,test.tinyint_col) |",
-            "+---------------------------------------+",
-            "| 0.21821789023599245                   |",
-            "+---------------------------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select approx_percentile_cont_with_weight(\"id\", 2, 0.5) from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+---------------------------------------------------------------+",
-            "| APPROXPERCENTILECONTWITHWEIGHT(test.id,Int64(2),Float64(0.5)) |",
-            "+---------------------------------------------------------------+",
-            "| 1                                                             |",
-            "+---------------------------------------------------------------+",
-        ];
-        assert_result_eq(expected, &*res);
-
-        let df = context
-            .sql("select approx_percentile_cont(\"double_col\", 0.5) from test")
-            .await
-            .unwrap();
-        let res = df.collect().await.unwrap();
-        let expected = vec![
-            "+----------------------------------------------------+",
-            "| APPROXPERCENTILECONT(test.double_col,Float64(0.5)) |",
-            "+----------------------------------------------------+",
-            "| 7.574999999999999                                  |",
-            "+----------------------------------------------------+",
-        ];
-
-        assert_result_eq(expected, &*res);
-
-        fn assert_result_eq(
-            expected: Vec<&str>,
-            results: &[arrow::record_batch::RecordBatch],
-        ) {
-            assert_eq!(
-                expected,
-                pretty_format_batches(results)
-                    .unwrap()
-                    .to_string()
-                    .trim()
-                    .lines()
-                    .collect::<Vec<&str>>()
-            );
-        }
-    }
-}
diff --git a/ballista/rust/client/src/lib.rs b/ballista/rust/client/src/lib.rs
deleted file mode 100644
index 125278dcc..000000000
--- a/ballista/rust/client/src/lib.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#![doc = include_str!("../README.md")]
-
-pub mod columnar_batch;
-pub mod context;
-pub mod prelude;
diff --git a/ballista/rust/client/src/prelude.rs b/ballista/rust/client/src/prelude.rs
deleted file mode 100644
index 15558a357..000000000
--- a/ballista/rust/client/src/prelude.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Ballista Prelude (common imports)
-
-pub use crate::context::BallistaContext;
-pub use ballista_core::config::BallistaConfig;
-pub use ballista_core::config::BALLISTA_DEFAULT_BATCH_SIZE;
-pub use ballista_core::config::BALLISTA_DEFAULT_SHUFFLE_PARTITIONS;
-pub use ballista_core::error::{BallistaError, Result};
-
-pub use futures::StreamExt;
diff --git a/ballista/rust/core/Cargo.toml b/ballista/rust/core/Cargo.toml
deleted file mode 100644
index 3d43aceb4..000000000
--- a/ballista/rust/core/Cargo.toml
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "ballista-core"
-description = "Ballista Distributed Compute"
-license = "Apache-2.0"
-version = "0.7.0"
-homepage = "https://github.com/apache/arrow-datafusion"
-repository = "https://github.com/apache/arrow-datafusion"
-readme = "README.md"
-authors = ["Apache Arrow <de...@arrow.apache.org>"]
-edition = "2018"
-build = "build.rs"
-
-[features]
-# Used for testing ONLY: causes all values to hash to the same value (test for collisions)
-force_hash_collisions = ["datafusion/force_hash_collisions"]
-simd = ["datafusion/simd"]
-
-[dependencies]
-ahash = { version = "0.7", default-features = false }
-
-arrow-flight = { version = "14.0.0" }
-async-trait = "0.1.41"
-chrono = { version = "0.4", default-features = false }
-clap = { version = "3", features = ["derive", "cargo"] }
-datafusion = { path = "../../../datafusion/core", version = "8.0.0" }
-datafusion-proto = { path = "../../../datafusion/proto", version = "8.0.0" }
-futures = "0.3"
-hashbrown = "0.12"
-
-libloading = "0.7.3"
-log = "0.4"
-once_cell = "1.9.0"
-
-parking_lot = "0.12"
-parse_arg = "0.1.3"
-prost = "0.10"
-prost-types = "0.10"
-serde = { version = "1", features = ["derive"] }
-sqlparser = "0.17"
-tokio = "1.0"
-tonic = "0.7"
-uuid = { version = "1.0", features = ["v4"] }
-walkdir = "2.3.2"
-
-[dev-dependencies]
-tempfile = "3"
-
-[build-dependencies]
-rustc_version = "0.4.0"
-tonic-build = { version = "0.7" }
diff --git a/ballista/rust/core/README.md b/ballista/rust/core/README.md
deleted file mode 100644
index 2b4c9fbfd..000000000
--- a/ballista/rust/core/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
-# Ballista Core Library
-
-This crate contains the Ballista core library which is used as a dependency by the `ballista-client`,
-`ballista-scheduler`, and `ballista-executor` crates.
diff --git a/ballista/rust/core/build.rs b/ballista/rust/core/build.rs
deleted file mode 100644
index c2acde108..000000000
--- a/ballista/rust/core/build.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-fn main() -> Result<(), String> {
-    // for use in docker build where file changes can be wonky
-    println!("cargo:rerun-if-env-changed=FORCE_REBUILD");
-
-    println!("cargo:rerun-if-changed=proto/ballista.proto");
-    let version = rustc_version::version().unwrap();
-    println!("cargo:rustc-env=RUSTC_VERSION={}", version);
-    println!("cargo:rerun-if-changed=proto/datafusion.proto");
-    tonic_build::configure()
-        .extern_path(".datafusion", "::datafusion_proto::protobuf")
-        .compile(&["proto/ballista.proto"], &["proto"])
-        .map_err(|e| format!("protobuf compilation failed: {}", e))
-}
diff --git a/ballista/rust/core/proto/ballista.proto b/ballista/rust/core/proto/ballista.proto
deleted file mode 100644
index 1ceb412f1..000000000
--- a/ballista/rust/core/proto/ballista.proto
+++ /dev/null
@@ -1,918 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-package ballista.protobuf;
-
-option java_multiple_files = true;
-option java_package = "org.ballistacompute.protobuf";
-option java_outer_classname = "BallistaProto";
-
-import "datafusion.proto";
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Ballista Logical Plan
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-// LogicalPlan is a nested type
-message LogicalPlanNode {
-  oneof LogicalPlanType {
-    ListingTableScanNode listing_scan = 1;
-    ProjectionNode projection = 3;
-    SelectionNode selection = 4;
-    LimitNode limit = 5;
-    AggregateNode aggregate = 6;
-    JoinNode join = 7;
-    SortNode sort = 8;
-    RepartitionNode repartition = 9;
-    EmptyRelationNode empty_relation = 10;
-    CreateExternalTableNode create_external_table = 11;
-    ExplainNode explain = 12;
-    WindowNode window = 13;
-    AnalyzeNode analyze = 14;
-    CrossJoinNode cross_join = 15;
-    ValuesNode values = 16;
-    LogicalExtensionNode extension = 17;
-    CreateCatalogSchemaNode create_catalog_schema = 18;
-    UnionNode union = 19;
-    CreateCatalogNode create_catalog = 20;
-    SubqueryAliasNode subquery_alias = 21;
-    CreateViewNode create_view = 22;
-    OffsetNode offset = 23;
-  }
-}
-
-message LogicalExtensionNode {
-  bytes node = 1;
-  repeated LogicalPlanNode inputs = 2;
-}
-
-message ProjectionColumns {
-  repeated string columns = 1;
-}
-
-message Statistics {
-  int64 num_rows = 1;
-  int64 total_byte_size = 2;
-  repeated ColumnStats column_stats = 3;
-  bool is_exact = 4;
-}
-
-message FileRange {
-  int64 start = 1;
-  int64 end = 2;
-}
-
-message PartitionedFile {
-  string path = 1;
-  uint64 size = 2;
-  uint64 last_modified_ns = 3;
-  repeated datafusion.ScalarValue partition_values = 4;
-  FileRange range = 5;
-}
-
-message CsvFormat {
-  bool has_header = 1;
-  string delimiter = 2;
-}
-
-message ParquetFormat {
-  bool enable_pruning = 1;
-}
-
-message AvroFormat {}
-
-message ListingTableScanNode {
-  string table_name = 1;
-  string path = 2;
-  string file_extension = 3;
-  ProjectionColumns projection = 4;
-  datafusion.Schema schema = 5;
-  repeated datafusion.LogicalExprNode filters = 6;
-  repeated string table_partition_cols = 7;
-  bool collect_stat = 8;
-  uint32 target_partitions = 9;
-  oneof FileFormatType {
-    CsvFormat csv = 10;
-    ParquetFormat parquet = 11;
-    AvroFormat avro = 12;
-  }
-}
-
-message ProjectionNode {
-  LogicalPlanNode input = 1;
-  repeated datafusion.LogicalExprNode expr = 2;
-  oneof optional_alias {
-    string alias = 3;
-  }
-}
-
-message SelectionNode {
-  LogicalPlanNode input = 1;
-  datafusion.LogicalExprNode expr = 2;
-}
-
-message SortNode {
-  LogicalPlanNode input = 1;
-  repeated datafusion.LogicalExprNode expr = 2;
-}
-
-message RepartitionNode {
-  LogicalPlanNode input = 1;
-  oneof partition_method {
-    uint64 round_robin = 2;
-    HashRepartition hash = 3;
-  }
-}
-
-message HashRepartition {
-  repeated datafusion.LogicalExprNode hash_expr = 1;
-  uint64 partition_count = 2;
-}
-
-message EmptyRelationNode {
-  bool produce_one_row = 1;
-}
-
-message CreateExternalTableNode {
-  string name = 1;
-  string location = 2;
-  FileType file_type = 3;
-  bool has_header = 4;
-  datafusion.DfSchema schema = 5;
-  repeated string table_partition_cols = 6;
-  bool if_not_exists = 7;
-  string delimiter = 8;
-}
-
-message CreateCatalogSchemaNode {
-  string schema_name = 1;
-  bool if_not_exists = 2;
-  datafusion.DfSchema schema = 3;
-}
-
-message CreateCatalogNode {
-  string catalog_name = 1;
-  bool if_not_exists = 2;
-  datafusion.DfSchema schema = 3;
-}
-
-message CreateViewNode {
-  string name = 1;
-  LogicalPlanNode input = 2;
-  bool or_replace = 3;
-}
-
-// a node containing data for defining values list. unlike in SQL where it's two dimensional, here
-// the list is flattened, and with the field n_cols it can be parsed and partitioned into rows
-message ValuesNode {
-  uint64 n_cols = 1;
-  repeated datafusion.LogicalExprNode values_list = 2;
-}
-
-enum FileType {
-  NdJson = 0;
-  Parquet = 1;
-  CSV = 2;
-  Avro = 3;
-}
-
-message AnalyzeNode {
-  LogicalPlanNode input = 1;
-  bool verbose = 2;
-}
-
-message ExplainNode {
-  LogicalPlanNode input = 1;
-  bool verbose = 2;
-}
-
-message AggregateNode {
-  LogicalPlanNode input = 1;
-  repeated datafusion.LogicalExprNode group_expr = 2;
-  repeated datafusion.LogicalExprNode aggr_expr = 3;
-}
-
-message WindowNode {
-  LogicalPlanNode input = 1;
-  repeated datafusion.LogicalExprNode window_expr = 2;
-}
-
-enum JoinType {
-  INNER = 0;
-  LEFT = 1;
-  RIGHT = 2;
-  FULL = 3;
-  SEMI = 4;
-  ANTI = 5;
-}
-
-enum JoinConstraint {
-  ON = 0;
-  USING = 1;
-}
-
-message JoinNode {
-  LogicalPlanNode left = 1;
-  LogicalPlanNode right = 2;
-  JoinType join_type = 3;
-  JoinConstraint join_constraint = 4;
-  repeated datafusion.Column left_join_column = 5;
-  repeated datafusion.Column right_join_column = 6;
-  bool null_equals_null = 7;
-}
-
-message UnionNode {
-  repeated LogicalPlanNode inputs = 1;
-}
-
-message CrossJoinNode {
-  LogicalPlanNode left = 1;
-  LogicalPlanNode right = 2;
-}
-
-message LimitNode {
-  LogicalPlanNode input = 1;
-  uint32 limit = 2;
-}
-
-message OffsetNode {
-  LogicalPlanNode input = 1;
-  uint32 offset = 2;
-}
-
-message SelectionExecNode {
-  datafusion.LogicalExprNode expr = 1;
-}
-
-message SubqueryAliasNode {
-  LogicalPlanNode input = 1;
-  string alias = 2;
-}
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Ballista Physical Plan
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-// PhysicalPlanNode is a nested type
-message PhysicalPlanNode {
-  oneof PhysicalPlanType {
-    ParquetScanExecNode parquet_scan = 1;
-    CsvScanExecNode csv_scan = 2;
-    EmptyExecNode empty = 3;
-    ProjectionExecNode projection = 4;
-    GlobalLimitExecNode global_limit = 6;
-    LocalLimitExecNode local_limit = 7;
-    AggregateExecNode aggregate = 8;
-    HashJoinExecNode hash_join = 9;
-    ShuffleReaderExecNode shuffle_reader = 10;
-    SortExecNode sort = 11;
-    CoalesceBatchesExecNode coalesce_batches = 12;
-    FilterExecNode filter = 13;
-    CoalescePartitionsExecNode merge = 14;
-    UnresolvedShuffleExecNode unresolved = 15;
-    RepartitionExecNode repartition = 16;
-    WindowAggExecNode window = 17;
-    ShuffleWriterExecNode shuffle_writer = 18;
-    CrossJoinExecNode cross_join = 19;
-    AvroScanExecNode avro_scan = 20;
-    PhysicalExtensionNode extension = 21;
-    UnionExecNode union = 22;
-    ExplainExecNode explain = 23;
-  }
-}
-
-message PhysicalExtensionNode {
-  bytes node = 1;
-  repeated PhysicalPlanNode inputs = 2;
-}
-
-// physical expressions
-message PhysicalExprNode {
-  oneof ExprType {
-    // column references
-    PhysicalColumn column = 1;
-
-    datafusion.ScalarValue literal = 2;
-
-    // binary expressions
-    PhysicalBinaryExprNode binary_expr = 3;
-
-    // aggregate expressions
-    PhysicalAggregateExprNode aggregate_expr = 4;
-
-    // null checks
-    PhysicalIsNull is_null_expr = 5;
-    PhysicalIsNotNull is_not_null_expr = 6;
-    PhysicalNot not_expr = 7;
-
-    PhysicalCaseNode case_ = 8;
-    PhysicalCastNode cast = 9;
-    PhysicalSortExprNode sort = 10;
-    PhysicalNegativeNode negative = 11;
-    PhysicalInListNode in_list = 12;
-    PhysicalScalarFunctionNode scalar_function = 13;
-    PhysicalTryCastNode try_cast = 14;
-
-    // window expressions
-    PhysicalWindowExprNode window_expr = 15;
-
-    PhysicalScalarUdfNode scalar_udf = 16;
-  }
-}
-
-message PhysicalScalarUdfNode {
-  string name = 1;
-  repeated PhysicalExprNode args = 2;
-  datafusion.ArrowType return_type = 4;
-}
-
-message PhysicalAggregateExprNode {
-  datafusion.AggregateFunction aggr_function = 1;
-  repeated PhysicalExprNode expr = 2;
-}
-
-message PhysicalWindowExprNode {
-  oneof window_function {
-    datafusion.AggregateFunction aggr_function = 1;
-    datafusion.BuiltInWindowFunction built_in_function = 2;
-    // udaf = 3
-  }
-  PhysicalExprNode expr = 4;
-}
-
-message PhysicalIsNull {
-  PhysicalExprNode expr = 1;
-}
-
-message PhysicalIsNotNull {
-  PhysicalExprNode expr = 1;
-}
-
-message PhysicalNot {
-  PhysicalExprNode expr = 1;
-}
-
-message PhysicalAliasNode {
-  PhysicalExprNode expr = 1;
-  string alias = 2;
-}
-
-message PhysicalBinaryExprNode {
-  PhysicalExprNode l = 1;
-  PhysicalExprNode r = 2;
-  string op = 3;
-}
-
-message PhysicalSortExprNode {
-  PhysicalExprNode expr = 1;
-  bool asc = 2;
-  bool nulls_first = 3;
-}
-
-message PhysicalWhenThen {
-  PhysicalExprNode when_expr = 1;
-  PhysicalExprNode then_expr = 2;
-}
-
-message PhysicalInListNode {
-  PhysicalExprNode expr = 1;
-  repeated PhysicalExprNode list = 2;
-  bool negated = 3;
-}
-
-message PhysicalCaseNode {
-  PhysicalExprNode expr = 1;
-  repeated PhysicalWhenThen when_then_expr = 2;
-  PhysicalExprNode else_expr = 3;
-}
-
-message PhysicalScalarFunctionNode {
-  string name = 1;
-  datafusion.ScalarFunction fun = 2;
-  repeated PhysicalExprNode args = 3;
-  datafusion.ArrowType return_type = 4;
-}
-
-message PhysicalTryCastNode {
-  PhysicalExprNode expr = 1;
-  datafusion.ArrowType arrow_type = 2;
-}
-
-message PhysicalCastNode {
-  PhysicalExprNode expr = 1;
-  datafusion.ArrowType arrow_type = 2;
-}
-
-message PhysicalNegativeNode {
-  PhysicalExprNode expr = 1;
-}
-
-message UnresolvedShuffleExecNode {
-  uint32 stage_id = 1;
-  datafusion.Schema schema = 2;
-  uint32 input_partition_count = 3;
-  uint32 output_partition_count = 4;
-}
-
-message FilterExecNode {
-  PhysicalPlanNode input = 1;
-  PhysicalExprNode expr = 2;
-}
-
-message FileGroup {
-  repeated PartitionedFile files = 1;
-}
-
-message ScanLimit {
-  // wrap into a message to make it optional
-  uint32 limit = 1;
-}
-
-message FileScanExecConf {
-  repeated FileGroup file_groups = 1;
-  datafusion.Schema schema = 2;
-  repeated uint32 projection = 4;
-  ScanLimit limit = 5;
-  Statistics statistics = 6;
-  repeated string table_partition_cols = 7;
-}
-
-message ParquetScanExecNode {
-  FileScanExecConf base_conf = 1;
-  datafusion.LogicalExprNode pruning_predicate = 2;
-}
-
-message CsvScanExecNode {
-  FileScanExecConf base_conf = 1;
-  bool has_header = 2;
-  string delimiter = 3;
-}
-
-message AvroScanExecNode {
-  FileScanExecConf base_conf = 1;
-}
-
-enum PartitionMode {
-  COLLECT_LEFT = 0;
-  PARTITIONED = 1;
-}
-
-message HashJoinExecNode {
-  PhysicalPlanNode left = 1;
-  PhysicalPlanNode right = 2;
-  repeated JoinOn on = 3;
-  JoinType join_type = 4;
-  PartitionMode partition_mode = 6;
-  bool null_equals_null = 7;
-}
-
-message UnionExecNode {
-  repeated PhysicalPlanNode inputs = 1;
-}
-
-message ExplainExecNode {
-  datafusion.Schema schema = 1;
-  repeated datafusion.StringifiedPlan stringified_plans = 2;
-  bool verbose = 3;
-}
-
-message CrossJoinExecNode {
-  PhysicalPlanNode left = 1;
-  PhysicalPlanNode right = 2;
-}
-
-message PhysicalColumn {
-  string name = 1;
-  uint32 index = 2;
-}
-
-message JoinOn {
-  PhysicalColumn left = 1;
-  PhysicalColumn right = 2;
-}
-
-message EmptyExecNode {
-  bool produce_one_row = 1;
-  datafusion.Schema schema = 2;
-}
-
-message ProjectionExecNode {
-  PhysicalPlanNode input = 1;
-  repeated PhysicalExprNode expr = 2;
-  repeated string expr_name = 3;
-}
-
-enum AggregateMode {
-  PARTIAL = 0;
-  FINAL = 1;
-  FINAL_PARTITIONED = 2;
-}
-
-message WindowAggExecNode {
-  PhysicalPlanNode input = 1;
-  repeated PhysicalExprNode window_expr = 2;
-  repeated string window_expr_name = 3;
-  datafusion.Schema input_schema = 4;
-}
-
-message AggregateExecNode {
-  repeated PhysicalExprNode group_expr = 1;
-  repeated PhysicalExprNode aggr_expr = 2;
-  AggregateMode mode = 3;
-  PhysicalPlanNode input = 4;
-  repeated string group_expr_name = 5;
-  repeated string aggr_expr_name = 6;
-  // we need the input schema to the partial aggregate to pass to the final aggregate
-  datafusion.Schema input_schema = 7;
-}
-
-message ShuffleWriterExecNode {
-  //TODO it seems redundant to provide job and stage id here since we also have them
-  // in the TaskDefinition that wraps this plan
-  string job_id = 1;
-  uint32 stage_id = 2;
-  PhysicalPlanNode input = 3;
-  PhysicalHashRepartition output_partitioning = 4;
-}
-
-message ShuffleReaderExecNode {
-  repeated ShuffleReaderPartition partition = 1;
-  datafusion.Schema schema = 2;
-}
-
-message ShuffleReaderPartition {
-  // each partition of a shuffle read can read data from multiple locations
-  repeated PartitionLocation location = 1;
-}
-
-message GlobalLimitExecNode {
-  PhysicalPlanNode input = 1;
-  uint32 limit = 2;
-}
-
-message LocalLimitExecNode {
-  PhysicalPlanNode input = 1;
-  uint32 limit = 2;
-}
-
-message SortExecNode {
-  PhysicalPlanNode input = 1;
-  repeated PhysicalExprNode expr = 2;
-}
-
-message CoalesceBatchesExecNode {
-  PhysicalPlanNode input = 1;
-  uint32 target_batch_size = 2;
-}
-
-message CoalescePartitionsExecNode {
-  PhysicalPlanNode input = 1;
-}
-
-message PhysicalHashRepartition {
-  repeated PhysicalExprNode hash_expr = 1;
-  uint64 partition_count = 2;
-}
-
-message RepartitionExecNode{
-  PhysicalPlanNode input = 1;
-  oneof partition_method {
-    uint64 round_robin = 2;
-    PhysicalHashRepartition hash = 3;
-    uint64 unknown = 4;
-  }
-}
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Ballista Scheduling
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-message KeyValuePair {
-  string key = 1;
-  string value = 2;
-}
-
-message Action {
-
-  oneof ActionType {
-    // Fetch a partition from an executor
-    FetchPartition fetch_partition = 3;
-  }
-
-  // configuration settings
-  repeated KeyValuePair settings = 100;
-}
-
-message ExecutePartition {
-  string job_id = 1;
-  uint32 stage_id = 2;
-  repeated uint32 partition_id = 3;
-  PhysicalPlanNode plan = 4;
-  // The task could need to read partitions from other executors
-  repeated PartitionLocation partition_location = 5;
-  // Output partition for shuffle writer
-  PhysicalHashRepartition output_partitioning = 6;
-}
-
-message FetchPartition {
-  string job_id = 1;
-  uint32 stage_id = 2;
-  uint32 partition_id = 3;
-  string path = 4;
-}
-
-// Mapping from partition id to executor id
-message PartitionLocation {
-  PartitionId partition_id = 1;
-  ExecutorMetadata executor_meta = 2;
-  PartitionStats partition_stats = 3;
-  string path = 4;
-}
-
-// Unique identifier for a materialized partition of data
-message PartitionId {
-  string job_id = 1;
-  uint32 stage_id = 2;
-  uint32 partition_id = 4;
-}
-
-message PartitionStats {
-  int64 num_rows = 1;
-  int64 num_batches = 2;
-  int64 num_bytes = 3;
-  repeated ColumnStats column_stats = 4;
-}
-
-message ColumnStats {
-  datafusion.ScalarValue min_value = 1;
-  datafusion.ScalarValue max_value = 2;
-  uint32 null_count = 3;
-  uint32 distinct_count = 4;
-}
-
-// Used by scheduler
-message ExecutorMetadata {
-  string id = 1;
-  string host = 2;
-  uint32 port = 3;
-  uint32 grpc_port = 4;
-  ExecutorSpecification specification = 5;
-}
-
-// Used by grpc
-message ExecutorRegistration {
-  string id = 1;
-  // "optional" keyword is stable in protoc 3.15 but prost is still on 3.14 (see https://github.com/tokio-rs/prost/issues/430 and https://github.com/tokio-rs/prost/pull/455)
-  // this syntax is ugly but is binary compatible with the "optional" keyword (see https://stackoverflow.com/questions/42622015/how-to-define-an-optional-field-in-protobuf-3)
-  oneof optional_host {
-    string host = 2;
-  }
-  uint32 port = 3;
-  uint32 grpc_port = 4;
-  ExecutorSpecification specification = 5;
-}
-
-message ExecutorHeartbeat {
-  string executor_id = 1;
-  // Unix epoch-based timestamp in seconds
-  uint64 timestamp = 2;
-  ExecutorState state = 3;
-}
-
-message ExecutorState {
-  repeated ExecutorMetric metrics = 1;
-}
-
-message ExecutorMetric {
-  // TODO add more metrics
-  oneof metric {
-    uint64 available_memory = 1;
-  }
-}
-
-message ExecutorSpecification {
-  repeated ExecutorResource resources = 1;
-}
-
-message ExecutorResource {
-  // TODO add more resources
-  oneof resource {
-    uint32 task_slots = 1;
-  }
-}
-
-message ExecutorData {
-  string executor_id = 1;
-  repeated ExecutorResourcePair resources = 2;
-}
-
-message ExecutorResourcePair {
-  ExecutorResource total = 1;
-  ExecutorResource available = 2;
-}
-
-message RunningTask {
-  string executor_id = 1;
-}
-
-message FailedTask {
-  string error = 1;
-}
-
-message CompletedTask {
-  string executor_id = 1;
-  // TODO tasks are currently always shuffle writes but this will not always be the case
-  // so we might want to think about some refactoring of the task definitions
-  repeated ShuffleWritePartition partitions = 2;
-}
-
-message ShuffleWritePartition {
-  uint64 partition_id = 1;
-  string path = 2;
-  uint64 num_batches = 3;
-  uint64 num_rows = 4;
-  uint64 num_bytes = 5;
-}
-
-message TaskStatus {
-  PartitionId task_id = 1;
-  oneof status {
-    RunningTask running = 2;
-    FailedTask failed = 3;
-    CompletedTask completed = 4;
-  }
-}
-
-message PollWorkParams {
-  ExecutorRegistration metadata = 1;
-  bool can_accept_task = 2;
-  // All tasks must be reported until they reach the failed or completed state
-  repeated TaskStatus task_status = 3;
-}
-
-message TaskDefinition {
-  PartitionId task_id = 1;
-  bytes plan = 2;
-  // Output partition for shuffle writer
-  PhysicalHashRepartition output_partitioning = 3;
-  string session_id = 4;
-  repeated KeyValuePair props = 5;
-}
-
-message JobSessionConfig {
-  string session_id = 1;
-  repeated KeyValuePair configs = 2;
-}
-
-message PollWorkResult {
-  TaskDefinition task = 1;
-}
-
-message RegisterExecutorParams {
-  ExecutorRegistration metadata = 1;
-}
-
-message RegisterExecutorResult {
-  bool success = 1;
-}
-
-message HeartBeatParams {
-  string executor_id = 1;
-  ExecutorState state = 2;
-}
-
-message HeartBeatResult {
-  // TODO it's from Spark for BlockManager
-  bool reregister = 1;
-}
-
-message StopExecutorParams {
-}
-
-message StopExecutorResult {
-}
-
-message UpdateTaskStatusParams {
-  string executor_id = 1;
-  // All tasks must be reported until they reach the failed or completed state
-  repeated TaskStatus task_status = 2;
-}
-
-message UpdateTaskStatusResult {
-  bool success = 1;
-}
-
-message ExecuteQueryParams {
-  oneof query {
-    bytes logical_plan = 1;
-    string sql = 2;
-  }
-  oneof optional_session_id {
-    string session_id = 3;
-  }
-  repeated KeyValuePair settings = 4;
-}
-
-message ExecuteSqlParams {
-  string sql = 1;
-}
-
-message ExecuteQueryResult {
-  string job_id = 1;
-  string session_id = 2;
-}
-
-message GetJobStatusParams {
-  string job_id = 1;
-}
-
-message CompletedJob {
-  repeated PartitionLocation partition_location = 1;
-}
-
-message QueuedJob {}
-
-// TODO: add progress report
-message RunningJob {}
-
-message FailedJob {
-  string error = 1;
-}
-
-message JobStatus {
-  oneof status {
-    QueuedJob queued = 1;
-    RunningJob running = 2;
-    FailedJob failed = 3;
-    CompletedJob completed = 4;
-  }
-}
-
-message GetJobStatusResult {
-  JobStatus status = 1;
-}
-
-message GetFileMetadataParams {
-  string path = 1;
-  FileType file_type = 2;
-}
-
-message GetFileMetadataResult {
-  datafusion.Schema schema = 1;
-}
-
-message FilePartitionMetadata {
-  repeated string filename = 1;
-}
-
-message LaunchTaskParams {
-  // Allow to launch a task set to an executor at once
-  repeated TaskDefinition task = 1;
-}
-
-message LaunchTaskResult {
-  bool success = 1;
-  // TODO when part of the task set are scheduled successfully
-}
-
-service SchedulerGrpc {
-  // Executors must poll the scheduler for heartbeat and to receive tasks
-  rpc PollWork (PollWorkParams) returns (PollWorkResult) {}
-
-  rpc RegisterExecutor(RegisterExecutorParams) returns (RegisterExecutorResult) {}
-
-  // Push-based task scheduler will only leverage this interface
-  // rather than the PollWork interface to report executor states
-  rpc HeartBeatFromExecutor (HeartBeatParams) returns (HeartBeatResult) {}
-
-  rpc UpdateTaskStatus (UpdateTaskStatusParams) returns (UpdateTaskStatusResult) {}
-
-  rpc GetFileMetadata (GetFileMetadataParams) returns (GetFileMetadataResult) {}
-
-  rpc ExecuteQuery (ExecuteQueryParams) returns (ExecuteQueryResult) {}
-
-  rpc GetJobStatus (GetJobStatusParams) returns (GetJobStatusResult) {}
-}
-
-service ExecutorGrpc {
-  rpc LaunchTask (LaunchTaskParams) returns (LaunchTaskResult) {}
-
-  rpc StopExecutor (StopExecutorParams) returns (StopExecutorResult) {}
-}
diff --git a/ballista/rust/core/proto/datafusion.proto b/ballista/rust/core/proto/datafusion.proto
deleted file mode 100644
index 9999abbf2..000000000
--- a/ballista/rust/core/proto/datafusion.proto
+++ /dev/null
@@ -1,542 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-package datafusion;
-
-option java_multiple_files = true;
-option java_package = "org.datafusioncompute.protobuf";
-option java_outer_classname = "DatafusionProto";
-
-message ColumnRelation {
-  string relation = 1;
-}
-
-message Column {
-  string name = 1;
-  ColumnRelation relation = 2;
-}
-
-message DfField{
-  Field field = 1;
-  ColumnRelation qualifier = 2;
-}
-
-message DfSchema {
-  repeated DfField columns = 1;
-}
-
-// logical expressions
-message LogicalExprNode {
-  oneof ExprType {
-    // column references
-    Column column = 1;
-
-    // alias
-    AliasNode alias = 2;
-
-    ScalarValue literal = 3;
-
-    // binary expressions
-    BinaryExprNode binary_expr = 4;
-
-    // aggregate expressions
-    AggregateExprNode aggregate_expr = 5;
-
-    // null checks
-    IsNull is_null_expr = 6;
-    IsNotNull is_not_null_expr = 7;
-    Not not_expr = 8;
-
-    BetweenNode between = 9;
-    CaseNode case_ = 10;
-    CastNode cast = 11;
-    SortExprNode sort = 12;
-    NegativeNode negative = 13;
-    InListNode in_list = 14;
-    bool wildcard = 15;
-    ScalarFunctionNode scalar_function = 16;
-    TryCastNode try_cast = 17;
-
-    // window expressions
-    WindowExprNode window_expr = 18;
-  }
-}
-
-message IsNull {
-  LogicalExprNode expr = 1;
-}
-
-message IsNotNull {
-  LogicalExprNode expr = 1;
-}
-
-message Not {
-  LogicalExprNode expr = 1;
-}
-
-message AliasNode {
-  LogicalExprNode expr = 1;
-  string alias = 2;
-}
-
-message BinaryExprNode {
-  LogicalExprNode l = 1;
-  LogicalExprNode r = 2;
-  string op = 3;
-}
-
-message NegativeNode {
-  LogicalExprNode expr = 1;
-}
-
-message InListNode {
-  LogicalExprNode expr = 1;
-  repeated LogicalExprNode list = 2;
-  bool negated = 3;
-}
-
-enum ScalarFunction {
-  Abs=0;
-  Acos=1;
-  Asin=2;
-  Atan=3;
-  Ascii=4;
-  Ceil=5;
-  Cos=6;
-  Digest=7;
-  Exp=8;
-  Floor=9;
-  Ln=10;
-  Log=11;
-  Log10=12;
-  Log2=13;
-  Round=14;
-  Signum=15;
-  Sin=16;
-  Sqrt=17;
-  Tan=18;
-  Trunc=19;
-  Array=20;
-  RegexpMatch=21;
-  BitLength=22;
-  Btrim=23;
-  CharacterLength=24;
-  Chr=25;
-  Concat=26;
-  ConcatWithSeparator=27;
-  DatePart=28;
-  DateTrunc=29;
-  InitCap=30;
-  Left=31;
-  Lpad=32;
-  Lower=33;
-  Ltrim=34;
-  MD5=35;
-  NullIf=36;
-  OctetLength=37;
-  Random=38;
-  RegexpReplace=39;
-  Repeat=40;
-  Replace=41;
-  Reverse=42;
-  Right=43;
-  Rpad=44;
-  Rtrim=45;
-  SHA224=46;
-  SHA256=47;
-  SHA384=48;
-  SHA512=49;
-  SplitPart=50;
-  StartsWith=51;
-  Strpos=52;
-  Substr=53;
-  ToHex=54;
-  ToTimestamp=55;
-  ToTimestampMillis=56;
-  ToTimestampMicros=57;
-  ToTimestampSeconds=58;
-  Now=59;
-  Translate=60;
-  Trim=61;
-  Upper=62;
-  Coalesce=63;
-}
-
-message ScalarFunctionNode {
-  ScalarFunction fun = 1;
-  repeated LogicalExprNode args = 2;
-}
-
-enum AggregateFunction {
-  MIN = 0;
-  MAX = 1;
-  SUM = 2;
-  AVG = 3;
-  COUNT = 4;
-  APPROX_DISTINCT = 5;
-  ARRAY_AGG = 6;
-  VARIANCE=7;
-  VARIANCE_POP=8;
-  COVARIANCE=9;
-  COVARIANCE_POP=10;
-  STDDEV=11;
-  STDDEV_POP=12;
-  CORRELATION=13;
-  APPROX_PERCENTILE_CONT = 14;
-  APPROX_MEDIAN=15;
-  APPROX_PERCENTILE_CONT_WITH_WEIGHT = 16;
-}
-
-message AggregateExprNode {
-  AggregateFunction aggr_function = 1;
-  repeated LogicalExprNode expr = 2;
-}
-
-enum BuiltInWindowFunction {
-  ROW_NUMBER = 0;
-  RANK = 1;
-  DENSE_RANK = 2;
-  PERCENT_RANK = 3;
-  CUME_DIST = 4;
-  NTILE = 5;
-  LAG = 6;
-  LEAD = 7;
-  FIRST_VALUE = 8;
-  LAST_VALUE = 9;
-  NTH_VALUE = 10;
-}
-
-message WindowExprNode {
-  oneof window_function {
-    AggregateFunction aggr_function = 1;
-    BuiltInWindowFunction built_in_function = 2;
-    // udaf = 3
-  }
-  LogicalExprNode expr = 4;
-  repeated LogicalExprNode partition_by = 5;
-  repeated LogicalExprNode order_by = 6;
-  // repeated LogicalExprNode filter = 7;
-  oneof window_frame {
-    WindowFrame frame = 8;
-  }
-}
-
-message BetweenNode {
-  LogicalExprNode expr = 1;
-  bool negated = 2;
-  LogicalExprNode low = 3;
-  LogicalExprNode high = 4;
-}
-
-message CaseNode {
-  LogicalExprNode expr = 1;
-  repeated WhenThen when_then_expr = 2;
-  LogicalExprNode else_expr = 3;
-}
-
-message WhenThen {
-  LogicalExprNode when_expr = 1;
-  LogicalExprNode then_expr = 2;
-}
-
-message CastNode {
-  LogicalExprNode expr = 1;
-  ArrowType arrow_type = 2;
-}
-
-message TryCastNode {
-  LogicalExprNode expr = 1;
-  ArrowType arrow_type = 2;
-}
-
-message SortExprNode {
-  LogicalExprNode expr = 1;
-  bool asc = 2;
-  bool nulls_first = 3;
-}
-
-enum WindowFrameUnits {
-  ROWS = 0;
-  RANGE = 1;
-  GROUPS = 2;
-}
-
-message WindowFrame {
-  WindowFrameUnits window_frame_units = 1;
-  WindowFrameBound start_bound = 2;
-  // "optional" keyword is stable in protoc 3.15 but prost is still on 3.14 (see https://github.com/tokio-rs/prost/issues/430 and https://github.com/tokio-rs/prost/pull/455)
-  // this syntax is ugly but is binary compatible with the "optional" keyword (see https://stackoverflow.com/questions/42622015/how-to-define-an-optional-field-in-protobuf-3)
-  oneof end_bound {
-    WindowFrameBound bound = 3;
-  }
-}
-
-enum WindowFrameBoundType {
-  CURRENT_ROW = 0;
-  PRECEDING = 1;
-  FOLLOWING = 2;
-}
-
-message WindowFrameBound {
-  WindowFrameBoundType window_frame_bound_type = 1;
-  // "optional" keyword is stable in protoc 3.15 but prost is still on 3.14 (see https://github.com/tokio-rs/prost/issues/430 and https://github.com/tokio-rs/prost/pull/455)
-  // this syntax is ugly but is binary compatible with the "optional" keyword (see https://stackoverflow.com/questions/42622015/how-to-define-an-optional-field-in-protobuf-3)
-  oneof bound_value {
-    uint64 value = 2;
-  }
-}
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Arrow Data Types
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-message Schema {
-  repeated datafusion.Field columns = 1;
-}
-
-message Field {
-  // name of the field
-  string name = 1;
-  ArrowType arrow_type = 2;
-  bool nullable = 3;
-  // for complex data types like structs, unions
-  repeated Field children = 4;
-}
-
-message FixedSizeBinary{
-    int32 length = 1;
-}
-
-message Timestamp{
-    TimeUnit time_unit = 1;
-    string timezone = 2;
-}
-
-enum DateUnit{
-    Day = 0;
-    DateMillisecond = 1;
-}
-
-enum TimeUnit{
-    Second = 0;
-    TimeMillisecond = 1;
-    Microsecond = 2;
-    Nanosecond = 3;
-}
-
-enum IntervalUnit{
-    YearMonth = 0;
-    DayTime = 1;
-    MonthDayNano = 2;
-}
-
-message Decimal{
-    uint64 whole = 1;
-    uint64 fractional = 2;
-}
-
-message List{
-    Field field_type = 1;
-}
-
-message FixedSizeList{
-    Field field_type = 1;
-    int32 list_size = 2;
-}
-
-message Dictionary{
-    ArrowType key = 1;
-    ArrowType value = 2;
-}
-
-message Struct{
-    repeated Field sub_field_types = 1;
-}
-
-enum UnionMode{
-  sparse = 0;
-  dense = 1;
-}
-
-message Union{
-    repeated Field union_types = 1;
-    UnionMode union_mode = 2;
-}
-
-message ScalarListValue{
-    ScalarType datatype = 1;
-    repeated ScalarValue values = 2;
-}
-
-message ScalarValue{
-    oneof value {
-        bool   bool_value = 1;
-        string utf8_value = 2;
-        string large_utf8_value = 3;
-        int32  int8_value = 4;
-        int32  int16_value = 5;
-        int32  int32_value = 6;
-        int64  int64_value = 7;
-        uint32 uint8_value = 8;
-        uint32 uint16_value = 9;
-        uint32 uint32_value = 10;
-        uint64 uint64_value = 11;
-        float  float32_value = 12;
-        double float64_value = 13;
-        //Literal Date32 value always has a unit of day
-        int32  date_32_value = 14;
-        int64  time_microsecond_value = 15;
-        int64  time_nanosecond_value = 16;
-        ScalarListValue list_value = 17;
-        ScalarType null_list_value = 18;
-
-        PrimitiveScalarType null_value = 19;
-        Decimal128 decimal128_value = 20;
-        int64 date_64_value = 21;
-        int64 time_second_value = 22;
-        int64 time_millisecond_value = 23;
-        int32 interval_yearmonth_value = 24;
-        int64 interval_daytime_value = 25;
-    }
-}
-
-message Decimal128{
-  bytes value = 1;
-  int64 p = 2;
-  int64 s = 3;
-}
-
-// Contains all valid datafusion scalar type except for
-// List
-enum PrimitiveScalarType{
-
-    BOOL = 0;     // arrow::Type::BOOL
-    UINT8 = 1;    // arrow::Type::UINT8
-    INT8 = 2;     // arrow::Type::INT8
-    UINT16 = 3;   // represents arrow::Type fields in src/arrow/type.h
-    INT16 = 4;
-    UINT32 = 5;
-    INT32 = 6;
-    UINT64 = 7;
-    INT64 = 8;
-    FLOAT32 = 9;
-    FLOAT64 = 10;
-    UTF8 = 11;
-    LARGE_UTF8 = 12;
-    DATE32 = 13;
-    TIME_MICROSECOND = 14;
-    TIME_NANOSECOND = 15;
-    NULL = 16;
-
-    DECIMAL128 = 17;
-    DATE64 = 20;
-    TIME_SECOND = 21;
-    TIME_MILLISECOND = 22;
-    INTERVAL_YEARMONTH = 23;
-    INTERVAL_DAYTIME = 24;
-}
-
-message ScalarType{
-    oneof datatype{
-        PrimitiveScalarType scalar = 1;
-        ScalarListType list = 2;
-    }
-}
-
-message ScalarListType{
-    repeated string field_names = 3;
-    PrimitiveScalarType deepest_type = 2;
-}
-
-// Broke out into multiple message types so that type
-// metadata did not need to be in separate message
-//All types that are of the empty message types contain no additional metadata
-// about the type
-message ArrowType{
-    oneof arrow_type_enum{
-        EmptyMessage NONE = 1;     // arrow::Type::NA
-        EmptyMessage BOOL =  2;     // arrow::Type::BOOL
-        EmptyMessage UINT8 = 3;    // arrow::Type::UINT8
-        EmptyMessage INT8 =  4;     // arrow::Type::INT8
-        EmptyMessage UINT16 =5;   // represents arrow::Type fields in src/arrow/type.h
-        EmptyMessage INT16 = 6;
-        EmptyMessage UINT32 =7;
-        EmptyMessage INT32 = 8;
-        EmptyMessage UINT64 =9;
-        EmptyMessage INT64 =10 ;
-        EmptyMessage FLOAT16 =11 ;
-        EmptyMessage FLOAT32 =12 ;
-        EmptyMessage FLOAT64 =13 ;
-        EmptyMessage UTF8 =14 ;
-        EmptyMessage LARGE_UTF8 = 32;
-        EmptyMessage BINARY =15 ;
-        int32 FIXED_SIZE_BINARY =16 ;
-        EmptyMessage LARGE_BINARY = 31;
-        EmptyMessage DATE32 =17 ;
-        EmptyMessage DATE64 =18 ;
-        TimeUnit DURATION = 19;
-        Timestamp TIMESTAMP =20 ;
-        TimeUnit TIME32 =21 ;
-        TimeUnit TIME64 =22 ;
-        IntervalUnit INTERVAL =23 ;
-        Decimal DECIMAL =24 ;
-        List LIST =25;
-        List LARGE_LIST = 26;
-        FixedSizeList FIXED_SIZE_LIST = 27;
-        Struct STRUCT =28;
-        Union UNION =29;
-        Dictionary DICTIONARY =30;
-    }
-}
-
-//Useful for representing an empty enum variant in rust
-// E.G. enum example{One, Two(i32)}
-// maps to
-// message example{
-//    oneof{
-//        EmptyMessage One = 1;
-//        i32 Two = 2;
-//   }
-//}
-message EmptyMessage{}
-
-message OptimizedLogicalPlanType {
-  string optimizer_name = 1;
-}
-
-message OptimizedPhysicalPlanType {
-  string optimizer_name = 1;
-}
-
-message PlanType {
-  oneof plan_type_enum {
-    EmptyMessage InitialLogicalPlan = 1;
-    OptimizedLogicalPlanType OptimizedLogicalPlan = 2;
-    EmptyMessage FinalLogicalPlan = 3;
-    EmptyMessage InitialPhysicalPlan = 4;
-    OptimizedPhysicalPlanType OptimizedPhysicalPlan = 5;
-    EmptyMessage FinalPhysicalPlan = 6;
-  }
-}
-
-message StringifiedPlan {
-  PlanType plan_type = 1;
-  string plan = 2;
-}
diff --git a/ballista/rust/core/src/client.rs b/ballista/rust/core/src/client.rs
deleted file mode 100644
index a5c4a062b..000000000
--- a/ballista/rust/core/src/client.rs
+++ /dev/null
@@ -1,178 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Client API for sending requests to executors.
-
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use std::{
-    convert::{TryFrom, TryInto},
-    task::{Context, Poll},
-};
-
-use crate::error::{ballista_error, BallistaError, Result};
-use crate::serde::protobuf::{self};
-use crate::serde::scheduler::Action;
-
-use arrow_flight::utils::flight_data_to_arrow_batch;
-use arrow_flight::Ticket;
-use arrow_flight::{flight_service_client::FlightServiceClient, FlightData};
-use datafusion::arrow::array::ArrayRef;
-use datafusion::arrow::{
-    datatypes::{Schema, SchemaRef},
-    error::{ArrowError, Result as ArrowResult},
-    record_batch::RecordBatch,
-};
-
-use datafusion::physical_plan::{RecordBatchStream, SendableRecordBatchStream};
-use futures::{Stream, StreamExt};
-use log::debug;
-use prost::Message;
-use tonic::Streaming;
-
-/// Client for interacting with Ballista executors.
-#[derive(Clone)]
-pub struct BallistaClient {
-    flight_client: FlightServiceClient<tonic::transport::channel::Channel>,
-}
-
-impl BallistaClient {
-    /// Create a new BallistaClient to connect to the executor listening on the specified
-    /// host and port
-    pub async fn try_new(host: &str, port: u16) -> Result<Self> {
-        let addr = format!("http://{}:{}", host, port);
-        debug!("BallistaClient connecting to {}", addr);
-        let flight_client =
-            FlightServiceClient::connect(addr.clone())
-                .await
-                .map_err(|e| {
-                    BallistaError::General(format!(
-                        "Error connecting to Ballista scheduler or executor at {}: {:?}",
-                        addr, e
-                    ))
-                })?;
-        debug!("BallistaClient connected OK");
-
-        Ok(Self { flight_client })
-    }
-
-    /// Fetch a partition from an executor
-    pub async fn fetch_partition(
-        &mut self,
-        job_id: &str,
-        stage_id: usize,
-        partition_id: usize,
-        path: &str,
-    ) -> Result<SendableRecordBatchStream> {
-        let action = Action::FetchPartition {
-            job_id: job_id.to_string(),
-            stage_id,
-            partition_id,
-            path: path.to_owned(),
-        };
-        self.execute_action(&action).await
-    }
-
-    /// Execute an action and retrieve the results
-    pub async fn execute_action(
-        &mut self,
-        action: &Action,
-    ) -> Result<SendableRecordBatchStream> {
-        let serialized_action: protobuf::Action = action.to_owned().try_into()?;
-
-        let mut buf: Vec<u8> = Vec::with_capacity(serialized_action.encoded_len());
-
-        serialized_action
-            .encode(&mut buf)
-            .map_err(|e| BallistaError::General(format!("{:?}", e)))?;
-
-        let request = tonic::Request::new(Ticket { ticket: buf });
-
-        let mut stream = self
-            .flight_client
-            .do_get(request)
-            .await
-            .map_err(|e| BallistaError::General(format!("{:?}", e)))?
-            .into_inner();
-
-        // the schema should be the first message returned, else client should error
-        match stream
-            .message()
-            .await
-            .map_err(|e| BallistaError::General(format!("{:?}", e)))?
-        {
-            Some(flight_data) => {
-                // convert FlightData to a stream
-                let schema = Arc::new(Schema::try_from(&flight_data)?);
-
-                // all the remaining stream messages should be dictionary and record batches
-                Ok(Box::pin(FlightDataStream::new(stream, schema)))
-            }
-            None => Err(ballista_error(
-                "Did not receive schema batch from flight server",
-            )),
-        }
-    }
-}
-
-struct FlightDataStream {
-    stream: Streaming<FlightData>,
-    schema: SchemaRef,
-    dictionaries_by_id: HashMap<i64, ArrayRef>,
-}
-
-impl FlightDataStream {
-    pub fn new(stream: Streaming<FlightData>, schema: SchemaRef) -> Self {
-        Self {
-            stream,
-            schema,
-            dictionaries_by_id: HashMap::new(),
-        }
-    }
-}
-
-impl Stream for FlightDataStream {
-    type Item = ArrowResult<RecordBatch>;
-
-    fn poll_next(
-        mut self: std::pin::Pin<&mut Self>,
-        cx: &mut Context<'_>,
-    ) -> Poll<Option<Self::Item>> {
-        self.stream.poll_next_unpin(cx).map(|x| match x {
-            Some(flight_data_chunk_result) => {
-                let converted_chunk = flight_data_chunk_result
-                    .map_err(|e| ArrowError::from_external_error(Box::new(e)))
-                    .and_then(|flight_data_chunk| {
-                        flight_data_to_arrow_batch(
-                            &flight_data_chunk,
-                            self.schema.clone(),
-                            &self.dictionaries_by_id,
-                        )
-                    });
-                Some(converted_chunk)
-            }
-            None => None,
-        })
-    }
-}
-
-impl RecordBatchStream for FlightDataStream {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-}
diff --git a/ballista/rust/core/src/config.rs b/ballista/rust/core/src/config.rs
deleted file mode 100644
index 1ff02c16d..000000000
--- a/ballista/rust/core/src/config.rs
+++ /dev/null
@@ -1,323 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-//! Ballista configuration
-
-use clap::ArgEnum;
-use core::fmt;
-use std::collections::HashMap;
-use std::result;
-
-use crate::error::{BallistaError, Result};
-
-use datafusion::arrow::datatypes::DataType;
-
-pub const BALLISTA_DEFAULT_SHUFFLE_PARTITIONS: &str = "ballista.shuffle.partitions";
-pub const BALLISTA_DEFAULT_BATCH_SIZE: &str = "ballista.batch.size";
-pub const BALLISTA_REPARTITION_JOINS: &str = "ballista.repartition.joins";
-pub const BALLISTA_REPARTITION_AGGREGATIONS: &str = "ballista.repartition.aggregations";
-pub const BALLISTA_REPARTITION_WINDOWS: &str = "ballista.repartition.windows";
-pub const BALLISTA_PARQUET_PRUNING: &str = "ballista.parquet.pruning";
-pub const BALLISTA_WITH_INFORMATION_SCHEMA: &str = "ballista.with_information_schema";
-/// give a plugin files dir, and then the dynamic library files in this dir will be load when scheduler state init.
-pub const BALLISTA_PLUGIN_DIR: &str = "ballista.plugin_dir";
-
-pub type ParseResult<T> = result::Result<T, String>;
-
-/// Configuration option meta-data
-#[derive(Debug, Clone)]
-pub struct ConfigEntry {
-    name: String,
-    _description: String,
-    _data_type: DataType,
-    default_value: Option<String>,
-}
-
-impl ConfigEntry {
-    fn new(
-        name: String,
-        _description: String,
-        _data_type: DataType,
-        default_value: Option<String>,
-    ) -> Self {
-        Self {
-            name,
-            _description,
-            _data_type,
-            default_value,
-        }
-    }
-}
-
-/// Ballista configuration builder
-pub struct BallistaConfigBuilder {
-    settings: HashMap<String, String>,
-}
-
-impl Default for BallistaConfigBuilder {
-    /// Create a new config builder
-    fn default() -> Self {
-        Self {
-            settings: HashMap::new(),
-        }
-    }
-}
-
-impl BallistaConfigBuilder {
-    /// Create a new config with an additional setting
-    pub fn set(&self, k: &str, v: &str) -> Self {
-        let mut settings = self.settings.clone();
-        settings.insert(k.to_owned(), v.to_owned());
-        Self { settings }
-    }
-
-    pub fn build(&self) -> Result<BallistaConfig> {
-        BallistaConfig::with_settings(self.settings.clone())
-    }
-}
-
-/// Ballista configuration
-#[derive(Debug, Clone)]
-pub struct BallistaConfig {
-    /// Settings stored in map for easy serde
-    settings: HashMap<String, String>,
-}
-
-impl BallistaConfig {
-    /// Create a default configuration
-    pub fn new() -> Result<Self> {
-        Self::with_settings(HashMap::new())
-    }
-
-    /// Create a configuration builder
-    pub fn builder() -> BallistaConfigBuilder {
-        BallistaConfigBuilder::default()
-    }
-
-    /// Create a new configuration based on key-value pairs
-    pub fn with_settings(settings: HashMap<String, String>) -> Result<Self> {
-        let supported_entries = BallistaConfig::valid_entries();
-        for (name, entry) in &supported_entries {
-            if let Some(v) = settings.get(name) {
-                // validate that we can parse the user-supplied value
-                let _ = Self::parse_value(v.as_str(), entry._data_type.clone()).map_err(|e| BallistaError::General(format!("Failed to parse user-supplied value '{}' for configuration setting '{}': {}", name, v, e)))?;
-            } else if let Some(v) = entry.default_value.clone() {
-                let _ = Self::parse_value(v.as_str(), entry._data_type.clone()).map_err(|e| BallistaError::General(format!("Failed to parse default value '{}' for configuration setting '{}': {}", name, v, e)))?;
-            } else {
-                return Err(BallistaError::General(format!(
-                    "No value specified for mandatory configuration setting '{}'",
-                    name
-                )));
-            }
-        }
-
-        Ok(Self { settings })
-    }
-
-    pub fn parse_value(val: &str, data_type: DataType) -> ParseResult<()> {
-        match data_type {
-            DataType::UInt16 => {
-                val.to_string()
-                    .parse::<usize>()
-                    .map_err(|e| format!("{:?}", e))?;
-            }
-            DataType::Boolean => {
-                val.to_string()
-                    .parse::<bool>()
-                    .map_err(|e| format!("{:?}", e))?;
-            }
-            DataType::Utf8 => {
-                val.to_string();
-            }
-            _ => {
-                return Err(format!("not support data type: {}", data_type));
-            }
-        }
-
-        Ok(())
-    }
-
-    /// All available configuration options
-    pub fn valid_entries() -> HashMap<String, ConfigEntry> {
-        let entries = vec![
-            ConfigEntry::new(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS.to_string(),
-                "Sets the default number of partitions to create when repartitioning query stages".to_string(),
-                DataType::UInt16, Some("2".to_string())),
-            ConfigEntry::new(BALLISTA_DEFAULT_BATCH_SIZE.to_string(),
-                             "Sets the default batch size".to_string(),
-                             DataType::UInt16, Some("8192".to_string())),
-            ConfigEntry::new(BALLISTA_REPARTITION_JOINS.to_string(),
-                             "Configuration for repartition joins".to_string(),
-                             DataType::Boolean, Some("true".to_string())),
-            ConfigEntry::new(BALLISTA_REPARTITION_AGGREGATIONS.to_string(),
-                             "Configuration for repartition aggregations".to_string(),
-                             DataType::Boolean,Some("true".to_string())),
-            ConfigEntry::new(BALLISTA_REPARTITION_WINDOWS.to_string(),
-                             "Configuration for repartition windows".to_string(),
-                             DataType::Boolean,Some("true".to_string())),
-            ConfigEntry::new(BALLISTA_PARQUET_PRUNING.to_string(),
-                             "Configuration for parquet prune".to_string(),
-                             DataType::Boolean,Some("true".to_string())),
-            ConfigEntry::new(BALLISTA_WITH_INFORMATION_SCHEMA.to_string(),
-                "Sets whether enable information_schema".to_string(),
-                DataType::Boolean,Some("false".to_string())),
-            ConfigEntry::new(BALLISTA_PLUGIN_DIR.to_string(),
-                             "Sets the plugin dir".to_string(),
-                             DataType::Utf8,Some("".to_string())),
-        ];
-        entries
-            .iter()
-            .map(|e| (e.name.clone(), e.clone()))
-            .collect::<HashMap<_, _>>()
-    }
-
-    pub fn settings(&self) -> &HashMap<String, String> {
-        &self.settings
-    }
-
-    pub fn default_shuffle_partitions(&self) -> usize {
-        self.get_usize_setting(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS)
-    }
-
-    pub fn default_plugin_dir(&self) -> String {
-        self.get_string_setting(BALLISTA_PLUGIN_DIR)
-    }
-
-    pub fn default_batch_size(&self) -> usize {
-        self.get_usize_setting(BALLISTA_DEFAULT_BATCH_SIZE)
-    }
-
-    pub fn repartition_joins(&self) -> bool {
-        self.get_bool_setting(BALLISTA_REPARTITION_JOINS)
-    }
-
-    pub fn repartition_aggregations(&self) -> bool {
-        self.get_bool_setting(BALLISTA_REPARTITION_AGGREGATIONS)
-    }
-
-    pub fn repartition_windows(&self) -> bool {
-        self.get_bool_setting(BALLISTA_REPARTITION_WINDOWS)
-    }
-
-    pub fn parquet_pruning(&self) -> bool {
-        self.get_bool_setting(BALLISTA_PARQUET_PRUNING)
-    }
-
-    pub fn default_with_information_schema(&self) -> bool {
-        self.get_bool_setting(BALLISTA_WITH_INFORMATION_SCHEMA)
-    }
-
-    fn get_usize_setting(&self, key: &str) -> usize {
-        if let Some(v) = self.settings.get(key) {
-            // infallible because we validate all configs in the constructor
-            v.parse().unwrap()
-        } else {
-            let entries = Self::valid_entries();
-            // infallible because we validate all configs in the constructor
-            let v = entries.get(key).unwrap().default_value.as_ref().unwrap();
-            v.parse().unwrap()
-        }
-    }
-
-    fn get_bool_setting(&self, key: &str) -> bool {
-        if let Some(v) = self.settings.get(key) {
-            // infallible because we validate all configs in the constructor
-            v.parse::<bool>().unwrap()
-        } else {
-            let entries = Self::valid_entries();
-            // infallible because we validate all configs in the constructor
-            let v = entries.get(key).unwrap().default_value.as_ref().unwrap();
-            v.parse::<bool>().unwrap()
-        }
-    }
-    fn get_string_setting(&self, key: &str) -> String {
-        if let Some(v) = self.settings.get(key) {
-            // infallible because we validate all configs in the constructor
-            v.to_string()
-        } else {
-            let entries = Self::valid_entries();
-            // infallible because we validate all configs in the constructor
-            let v = entries.get(key).unwrap().default_value.as_ref().unwrap();
-            v.to_string()
-        }
-    }
-}
-
-// an enum used to configure the scheduler policy
-// needs to be visible to code generated by configure_me
-#[derive(Clone, ArgEnum, Copy, Debug, serde::Deserialize)]
-pub enum TaskSchedulingPolicy {
-    PullStaged,
-    PushStaged,
-}
-
-impl std::str::FromStr for TaskSchedulingPolicy {
-    type Err = String;
-
-    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
-        ArgEnum::from_str(s, true)
-    }
-}
-
-impl parse_arg::ParseArgFromStr for TaskSchedulingPolicy {
-    fn describe_type<W: fmt::Write>(mut writer: W) -> fmt::Result {
-        write!(writer, "The scheduler policy for the scheduler")
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn default_config() -> Result<()> {
-        let config = BallistaConfig::new()?;
-        assert_eq!(2, config.default_shuffle_partitions());
-        assert!(!config.default_with_information_schema());
-        assert_eq!("", config.default_plugin_dir().as_str());
-        Ok(())
-    }
-
-    #[test]
-    fn custom_config() -> Result<()> {
-        let config = BallistaConfig::builder()
-            .set(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS, "123")
-            .set(BALLISTA_WITH_INFORMATION_SCHEMA, "true")
-            .build()?;
-        assert_eq!(123, config.default_shuffle_partitions());
-        assert!(config.default_with_information_schema());
-        Ok(())
-    }
-
-    #[test]
-    fn custom_config_invalid() -> Result<()> {
-        let config = BallistaConfig::builder()
-            .set(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS, "true")
-            .set(BALLISTA_PLUGIN_DIR, "test_dir")
-            .build();
-        assert!(config.is_err());
-        assert_eq!("General(\"Failed to parse user-supplied value 'ballista.shuffle.partitions' for configuration setting 'true': ParseIntError { kind: InvalidDigit }\")", format!("{:?}", config.unwrap_err()));
-
-        let config = BallistaConfig::builder()
-            .set(BALLISTA_WITH_INFORMATION_SCHEMA, "123")
-            .build();
-        assert!(config.is_err());
-        assert_eq!("General(\"Failed to parse user-supplied value 'ballista.with_information_schema' for configuration setting '123': ParseBoolError\")", format!("{:?}", config.unwrap_err()));
-        Ok(())
-    }
-}
diff --git a/ballista/rust/core/src/error.rs b/ballista/rust/core/src/error.rs
deleted file mode 100644
index ba6cdc74d..000000000
--- a/ballista/rust/core/src/error.rs
+++ /dev/null
@@ -1,185 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Ballista error types
-
-use std::{
-    error::Error,
-    fmt::{Display, Formatter},
-    io, result,
-};
-
-use datafusion::arrow::error::ArrowError;
-use datafusion::error::DataFusionError;
-use sqlparser::parser;
-
-pub type Result<T> = result::Result<T, BallistaError>;
-
-/// Ballista error
-#[derive(Debug)]
-pub enum BallistaError {
-    NotImplemented(String),
-    General(String),
-    Internal(String),
-    ArrowError(ArrowError),
-    DataFusionError(DataFusionError),
-    SqlError(parser::ParserError),
-    IoError(io::Error),
-    // ReqwestError(reqwest::Error),
-    // HttpError(http::Error),
-    // KubeAPIError(kube::error::Error),
-    // KubeAPIRequestError(k8s_openapi::RequestError),
-    // KubeAPIResponseError(k8s_openapi::ResponseError),
-    TonicError(tonic::transport::Error),
-    GrpcError(tonic::Status),
-    TokioError(tokio::task::JoinError),
-}
-
-#[allow(clippy::from_over_into)]
-impl<T> Into<Result<T>> for BallistaError {
-    fn into(self) -> Result<T> {
-        Err(self)
-    }
-}
-
-pub fn ballista_error(message: &str) -> BallistaError {
-    BallistaError::General(message.to_owned())
-}
-
-impl From<String> for BallistaError {
-    fn from(e: String) -> Self {
-        BallistaError::General(e)
-    }
-}
-
-impl From<ArrowError> for BallistaError {
-    fn from(e: ArrowError) -> Self {
-        BallistaError::ArrowError(e)
-    }
-}
-
-impl From<parser::ParserError> for BallistaError {
-    fn from(e: parser::ParserError) -> Self {
-        BallistaError::SqlError(e)
-    }
-}
-
-impl From<DataFusionError> for BallistaError {
-    fn from(e: DataFusionError) -> Self {
-        BallistaError::DataFusionError(e)
-    }
-}
-
-impl From<io::Error> for BallistaError {
-    fn from(e: io::Error) -> Self {
-        BallistaError::IoError(e)
-    }
-}
-
-// impl From<reqwest::Error> for BallistaError {
-//     fn from(e: reqwest::Error) -> Self {
-//         BallistaError::ReqwestError(e)
-//     }
-// }
-//
-// impl From<http::Error> for BallistaError {
-//     fn from(e: http::Error) -> Self {
-//         BallistaError::HttpError(e)
-//     }
-// }
-
-// impl From<kube::error::Error> for BallistaError {
-//     fn from(e: kube::error::Error) -> Self {
-//         BallistaError::KubeAPIError(e)
-//     }
-// }
-
-// impl From<k8s_openapi::RequestError> for BallistaError {
-//     fn from(e: k8s_openapi::RequestError) -> Self {
-//         BallistaError::KubeAPIRequestError(e)
-//     }
-// }
-
-// impl From<k8s_openapi::ResponseError> for BallistaError {
-//     fn from(e: k8s_openapi::ResponseError) -> Self {
-//         BallistaError::KubeAPIResponseError(e)
-//     }
-// }
-
-impl From<tonic::transport::Error> for BallistaError {
-    fn from(e: tonic::transport::Error) -> Self {
-        BallistaError::TonicError(e)
-    }
-}
-
-impl From<tonic::Status> for BallistaError {
-    fn from(e: tonic::Status) -> Self {
-        BallistaError::GrpcError(e)
-    }
-}
-
-impl From<tokio::task::JoinError> for BallistaError {
-    fn from(e: tokio::task::JoinError) -> Self {
-        BallistaError::TokioError(e)
-    }
-}
-
-impl From<datafusion_proto::from_proto::Error> for BallistaError {
-    fn from(e: datafusion_proto::from_proto::Error) -> Self {
-        BallistaError::General(e.to_string())
-    }
-}
-
-impl From<datafusion_proto::to_proto::Error> for BallistaError {
-    fn from(e: datafusion_proto::to_proto::Error) -> Self {
-        BallistaError::General(e.to_string())
-    }
-}
-
-impl Display for BallistaError {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        match self {
-            BallistaError::NotImplemented(ref desc) => {
-                write!(f, "Not implemented: {}", desc)
-            }
-            BallistaError::General(ref desc) => write!(f, "General error: {}", desc),
-            BallistaError::ArrowError(ref desc) => write!(f, "Arrow error: {}", desc),
-            BallistaError::DataFusionError(ref desc) => {
-                write!(f, "DataFusion error: {:?}", desc)
-            }
-            BallistaError::SqlError(ref desc) => write!(f, "SQL error: {:?}", desc),
-            BallistaError::IoError(ref desc) => write!(f, "IO error: {}", desc),
-            // BallistaError::ReqwestError(ref desc) => write!(f, "Reqwest error: {}", desc),
-            // BallistaError::HttpError(ref desc) => write!(f, "HTTP error: {}", desc),
-            // BallistaError::KubeAPIError(ref desc) => write!(f, "Kube API error: {}", desc),
-            // BallistaError::KubeAPIRequestError(ref desc) => {
-            //     write!(f, "KubeAPI request error: {}", desc)
-            // }
-            // BallistaError::KubeAPIResponseError(ref desc) => {
-            //     write!(f, "KubeAPI response error: {}", desc)
-            // }
-            BallistaError::TonicError(desc) => write!(f, "Tonic error: {}", desc),
-            BallistaError::GrpcError(desc) => write!(f, "Grpc error: {}", desc),
-            BallistaError::Internal(desc) => {
-                write!(f, "Internal Ballista error: {}", desc)
-            }
-            BallistaError::TokioError(desc) => write!(f, "Tokio join error: {}", desc),
-        }
-    }
-}
-
-impl Error for BallistaError {}
diff --git a/ballista/rust/core/src/event_loop.rs b/ballista/rust/core/src/event_loop.rs
deleted file mode 100644
index 217b7fc30..000000000
--- a/ballista/rust/core/src/event_loop.rs
+++ /dev/null
@@ -1,141 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use log::{error, info};
-use tokio::sync::mpsc;
-
-use crate::error::{BallistaError, Result};
-
-#[async_trait]
-pub trait EventAction<E>: Send + Sync {
-    fn on_start(&self);
-
-    fn on_stop(&self);
-
-    async fn on_receive(&self, event: E) -> Result<Option<E>>;
-
-    fn on_error(&self, error: BallistaError);
-}
-
-#[derive(Clone)]
-pub struct EventLoop<E> {
-    pub name: String,
-    pub buffer_size: usize,
-    stopped: Arc<AtomicBool>,
-    action: Arc<dyn EventAction<E>>,
-    tx_event: Option<mpsc::Sender<E>>,
-}
-
-impl<E: Send + 'static> EventLoop<E> {
-    pub fn new(
-        name: String,
-        buffer_size: usize,
-        action: Arc<dyn EventAction<E>>,
-    ) -> Self {
-        Self {
-            name,
-            buffer_size,
-            stopped: Arc::new(AtomicBool::new(false)),
-            action,
-            tx_event: None,
-        }
-    }
-
-    fn run(&self, mut rx_event: mpsc::Receiver<E>) {
-        assert!(
-            self.tx_event.is_some(),
-            "The event sender should be initialized first!"
-        );
-        let tx_event = self.tx_event.as_ref().unwrap().clone();
-        let name = self.name.clone();
-        let stopped = self.stopped.clone();
-        let action = self.action.clone();
-        tokio::spawn(async move {
-            info!("Starting the event loop {}", name);
-            while !stopped.load(Ordering::SeqCst) {
-                if let Some(event) = rx_event.recv().await {
-                    match action.on_receive(event).await {
-                        Ok(Some(event)) => {
-                            if let Err(e) = tx_event.send(event).await {
-                                let msg = format!("Fail to send event due to {}", e);
-                                error!("{}", msg);
-                                action.on_error(BallistaError::General(msg));
-                            }
-                        }
-                        Err(e) => {
-                            error!("Fail to process event due to {}", e);
-                            action.on_error(e);
-                        }
-                        _ => {}
-                    }
-                } else {
-                    info!("Event Channel closed, shutting down");
-                    break;
-                }
-            }
-            info!("The event loop {} has been stopped", name);
-        });
-    }
-
-    pub fn start(&mut self) -> Result<()> {
-        if self.stopped.load(Ordering::SeqCst) {
-            return Err(BallistaError::General(format!(
-                "{} has already been stopped",
-                self.name
-            )));
-        }
-        self.action.on_start();
-
-        let (tx_event, rx_event) = mpsc::channel::<E>(self.buffer_size);
-        self.tx_event = Some(tx_event);
-        self.run(rx_event);
-
-        Ok(())
-    }
-
-    pub fn stop(&self) {
-        if !self.stopped.swap(true, Ordering::SeqCst) {
-            self.action.on_stop();
-        } else {
-            // Keep quiet to allow calling `stop` multiple times.
-        }
-    }
-
-    pub fn get_sender(&self) -> Result<EventSender<E>> {
-        Ok(EventSender {
-            tx_event: self.tx_event.as_ref().cloned().ok_or_else(|| {
-                BallistaError::General("Event sender not exist!!!".to_string())
-            })?,
-        })
-    }
-}
-
-pub struct EventSender<E> {
-    tx_event: mpsc::Sender<E>,
-}
-
-impl<E> EventSender<E> {
-    pub async fn post_event(&self, event: E) -> Result<()> {
-        self.tx_event.send(event).await.map_err(|e| {
-            BallistaError::General(format!("Fail to send event due to {}", e))
-        })
-    }
-}
diff --git a/ballista/rust/core/src/execution_plans/distributed_query.rs b/ballista/rust/core/src/execution_plans/distributed_query.rs
deleted file mode 100644
index 0b20acb47..000000000
--- a/ballista/rust/core/src/execution_plans/distributed_query.rs
+++ /dev/null
@@ -1,329 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::any::Any;
-
-use std::fmt::Debug;
-use std::marker::PhantomData;
-
-use std::sync::Arc;
-use std::time::Duration;
-
-use crate::client::BallistaClient;
-use crate::config::BallistaConfig;
-use crate::serde::protobuf::{
-    execute_query_params::Query, job_status, scheduler_grpc_client::SchedulerGrpcClient,
-    ExecuteQueryParams, GetJobStatusParams, GetJobStatusResult, KeyValuePair,
-    PartitionLocation,
-};
-
-use datafusion::arrow::datatypes::SchemaRef;
-use datafusion::error::{DataFusionError, Result};
-use datafusion::logical_plan::LogicalPlan;
-use datafusion::physical_plan::expressions::PhysicalSortExpr;
-use datafusion::physical_plan::{
-    DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream, Statistics,
-};
-
-use crate::serde::protobuf::execute_query_params::OptionalSessionId;
-use crate::serde::{AsLogicalPlan, DefaultLogicalExtensionCodec, LogicalExtensionCodec};
-use datafusion::arrow::error::{ArrowError, Result as ArrowResult};
-use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::execution::context::TaskContext;
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt};
-use log::{error, info};
-
-/// This operator sends a logical plan to a Ballista scheduler for execution and
-/// polls the scheduler until the query is complete and then fetches the resulting
-/// batches directly from the executors that hold the results from the final
-/// query stage.
-#[derive(Debug, Clone)]
-pub struct DistributedQueryExec<T: 'static + AsLogicalPlan> {
-    /// Ballista scheduler URL
-    scheduler_url: String,
-    /// Ballista configuration
-    config: BallistaConfig,
-    /// Logical plan to execute
-    plan: LogicalPlan,
-    /// Codec for LogicalPlan extensions
-    extension_codec: Arc<dyn LogicalExtensionCodec>,
-    /// Phantom data for serializable plan message
-    plan_repr: PhantomData<T>,
-    /// Session id
-    session_id: String,
-}
-
-impl<T: 'static + AsLogicalPlan> DistributedQueryExec<T> {
-    pub fn new(
-        scheduler_url: String,
-        config: BallistaConfig,
-        plan: LogicalPlan,
-        session_id: String,
-    ) -> Self {
-        Self {
-            scheduler_url,
-            config,
-            plan,
-            extension_codec: Arc::new(DefaultLogicalExtensionCodec {}),
-            plan_repr: PhantomData,
-            session_id,
-        }
-    }
-
-    pub fn with_extension(
-        scheduler_url: String,
-        config: BallistaConfig,
-        plan: LogicalPlan,
-        extension_codec: Arc<dyn LogicalExtensionCodec>,
-        session_id: String,
-    ) -> Self {
-        Self {
-            scheduler_url,
-            config,
-            plan,
-            extension_codec,
-            plan_repr: PhantomData,
-            session_id,
-        }
-    }
-
-    pub fn with_repr(
-        scheduler_url: String,
-        config: BallistaConfig,
-        plan: LogicalPlan,
-        extension_codec: Arc<dyn LogicalExtensionCodec>,
-        plan_repr: PhantomData<T>,
-        session_id: String,
-    ) -> Self {
-        Self {
-            scheduler_url,
-            config,
-            plan,
-            extension_codec,
-            plan_repr,
-            session_id,
-        }
-    }
-}
-
-impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedQueryExec<T> {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.plan.schema().as_ref().clone().into()
-    }
-
-    fn output_partitioning(&self) -> Partitioning {
-        Partitioning::UnknownPartitioning(1)
-    }
-
-    fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> {
-        None
-    }
-
-    fn relies_on_input_order(&self) -> bool {
-        false
-    }
-
-    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
-        vec![]
-    }
-
-    fn with_new_children(
-        self: Arc<Self>,
-        _children: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> datafusion::error::Result<Arc<dyn ExecutionPlan>> {
-        Ok(Arc::new(DistributedQueryExec {
-            scheduler_url: self.scheduler_url.clone(),
-            config: self.config.clone(),
-            plan: self.plan.clone(),
-            extension_codec: self.extension_codec.clone(),
-            plan_repr: self.plan_repr,
-            session_id: self.session_id.clone(),
-        }))
-    }
-
-    fn execute(
-        &self,
-        partition: usize,
-        _context: Arc<TaskContext>,
-    ) -> Result<SendableRecordBatchStream> {
-        assert_eq!(0, partition);
-
-        let mut buf: Vec<u8> = vec![];
-        let plan_message =
-            T::try_from_logical_plan(&self.plan, self.extension_codec.as_ref()).map_err(
-                |e| {
-                    DataFusionError::Internal(format!(
-                        "failed to serialize logical plan: {:?}",
-                        e
-                    ))
-                },
-            )?;
-        plan_message.try_encode(&mut buf).map_err(|e| {
-            DataFusionError::Execution(format!("failed to encode logical plan: {:?}", e))
-        })?;
-
-        let query = ExecuteQueryParams {
-            query: Some(Query::LogicalPlan(buf)),
-            settings: self
-                .config
-                .settings()
-                .iter()
-                .map(|(k, v)| KeyValuePair {
-                    key: k.to_owned(),
-                    value: v.to_owned(),
-                })
-                .collect::<Vec<_>>(),
-            optional_session_id: Some(OptionalSessionId::SessionId(
-                self.session_id.clone(),
-            )),
-        };
-
-        let stream = futures::stream::once(
-            execute_query(self.scheduler_url.clone(), self.session_id.clone(), query)
-                .map_err(|e| ArrowError::ExternalError(Box::new(e))),
-        )
-        .try_flatten();
-
-        let schema = self.schema();
-        Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
-    }
-
-    fn fmt_as(
-        &self,
-        t: DisplayFormatType,
-        f: &mut std::fmt::Formatter,
-    ) -> std::fmt::Result {
-        match t {
-            DisplayFormatType::Default => {
-                write!(
-                    f,
-                    "DistributedQueryExec: scheduler_url={}",
-                    self.scheduler_url
-                )
-            }
-        }
-    }
-
-    fn statistics(&self) -> Statistics {
-        // This execution plan sends the logical plan to the scheduler without
-        // performing the node by node conversion to a full physical plan.
-        // This implies that we cannot infer the statistics at this stage.
-        Statistics::default()
-    }
-}
-
-async fn execute_query(
-    scheduler_url: String,
-    session_id: String,
-    query: ExecuteQueryParams,
-) -> Result<impl Stream<Item = ArrowResult<RecordBatch>> + Send> {
-    info!("Connecting to Ballista scheduler at {}", scheduler_url);
-    // TODO reuse the scheduler to avoid connecting to the Ballista scheduler again and again
-
-    let mut scheduler = SchedulerGrpcClient::connect(scheduler_url.clone())
-        .await
-        .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-
-    let query_result = scheduler
-        .execute_query(query)
-        .await
-        .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?
-        .into_inner();
-
-    assert_eq!(
-        session_id, query_result.session_id,
-        "Session id inconsistent between Client and Server side in DistributedQueryExec."
-    );
-
-    let job_id = query_result.job_id;
-    let mut prev_status: Option<job_status::Status> = None;
-
-    loop {
-        let GetJobStatusResult { status } = scheduler
-            .get_job_status(GetJobStatusParams {
-                job_id: job_id.clone(),
-            })
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?
-            .into_inner();
-        let status = status.and_then(|s| s.status).ok_or_else(|| {
-            DataFusionError::Internal("Received empty status message".to_owned())
-        })?;
-        let wait_future = tokio::time::sleep(Duration::from_millis(100));
-        let has_status_change = prev_status.map(|x| x != status).unwrap_or(true);
-        match status {
-            job_status::Status::Queued(_) => {
-                if has_status_change {
-                    info!("Job {} still queued...", job_id);
-                }
-                wait_future.await;
-                prev_status = Some(status);
-            }
-            job_status::Status::Running(_) => {
-                if has_status_change {
-                    info!("Job {} is running...", job_id);
-                }
-                wait_future.await;
-                prev_status = Some(status);
-            }
-            job_status::Status::Failed(err) => {
-                let msg = format!("Job {} failed: {}", job_id, err.error);
-                error!("{}", msg);
-                break Err(DataFusionError::Execution(msg));
-            }
-            job_status::Status::Completed(completed) => {
-                let streams = completed.partition_location.into_iter().map(|p| {
-                    let f = fetch_partition(p)
-                        .map_err(|e| ArrowError::ExternalError(Box::new(e)));
-
-                    futures::stream::once(f).try_flatten()
-                });
-
-                break Ok(futures::stream::iter(streams).flatten());
-            }
-        };
-    }
-}
-
-async fn fetch_partition(
-    location: PartitionLocation,
-) -> Result<SendableRecordBatchStream> {
-    let metadata = location.executor_meta.ok_or_else(|| {
-        DataFusionError::Internal("Received empty executor metadata".to_owned())
-    })?;
-    let partition_id = location.partition_id.ok_or_else(|| {
-        DataFusionError::Internal("Received empty partition id".to_owned())
-    })?;
-    let mut ballista_client =
-        BallistaClient::try_new(metadata.host.as_str(), metadata.port as u16)
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-    ballista_client
-        .fetch_partition(
-            &partition_id.job_id,
-            partition_id.stage_id as usize,
-            partition_id.partition_id as usize,
-            &location.path,
-        )
-        .await
-        .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))
-}
diff --git a/ballista/rust/core/src/execution_plans/mod.rs b/ballista/rust/core/src/execution_plans/mod.rs
deleted file mode 100644
index 7a5e105c6..000000000
--- a/ballista/rust/core/src/execution_plans/mod.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! This module contains execution plans that are needed to distribute DataFusion's execution plans into
-//! several Ballista executors.
-
-mod distributed_query;
-mod shuffle_reader;
-mod shuffle_writer;
-mod unresolved_shuffle;
-
-pub use distributed_query::DistributedQueryExec;
-pub use shuffle_reader::ShuffleReaderExec;
-pub use shuffle_writer::ShuffleWriterExec;
-pub use unresolved_shuffle::UnresolvedShuffleExec;
diff --git a/ballista/rust/core/src/execution_plans/shuffle_reader.rs b/ballista/rust/core/src/execution_plans/shuffle_reader.rs
deleted file mode 100644
index 3046a2276..000000000
--- a/ballista/rust/core/src/execution_plans/shuffle_reader.rs
+++ /dev/null
@@ -1,294 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::any::Any;
-use std::sync::Arc;
-
-use crate::client::BallistaClient;
-use crate::serde::scheduler::{PartitionLocation, PartitionStats};
-
-use datafusion::arrow::datatypes::SchemaRef;
-
-use datafusion::error::{DataFusionError, Result};
-use datafusion::physical_plan::expressions::PhysicalSortExpr;
-use datafusion::physical_plan::metrics::{
-    ExecutionPlanMetricsSet, MetricBuilder, MetricsSet,
-};
-use datafusion::physical_plan::{
-    DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream, Statistics,
-};
-use futures::{StreamExt, TryStreamExt};
-
-use datafusion::arrow::error::ArrowError;
-use datafusion::execution::context::TaskContext;
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use log::info;
-
-/// ShuffleReaderExec reads partitions that have already been materialized by a ShuffleWriterExec
-/// being executed by an executor
-#[derive(Debug, Clone)]
-pub struct ShuffleReaderExec {
-    /// Each partition of a shuffle can read data from multiple locations
-    pub(crate) partition: Vec<Vec<PartitionLocation>>,
-    pub(crate) schema: SchemaRef,
-    /// Execution metrics
-    metrics: ExecutionPlanMetricsSet,
-}
-
-impl ShuffleReaderExec {
-    /// Create a new ShuffleReaderExec
-    pub fn try_new(
-        partition: Vec<Vec<PartitionLocation>>,
-        schema: SchemaRef,
-    ) -> Result<Self> {
-        Ok(Self {
-            partition,
-            schema,
-            metrics: ExecutionPlanMetricsSet::new(),
-        })
-    }
-}
-
-impl ExecutionPlan for ShuffleReaderExec {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn output_partitioning(&self) -> Partitioning {
-        // TODO partitioning may be known and could be populated here
-        // see https://github.com/apache/arrow-datafusion/issues/758
-        Partitioning::UnknownPartitioning(self.partition.len())
-    }
-
-    fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> {
-        None
-    }
-
-    fn relies_on_input_order(&self) -> bool {
-        false
-    }
-
-    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
-        vec![]
-    }
-
-    fn with_new_children(
-        self: Arc<Self>,
-        _children: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
-        Err(DataFusionError::Plan(
-            "Ballista ShuffleReaderExec does not support with_new_children()".to_owned(),
-        ))
-    }
-
-    fn execute(
-        &self,
-        partition: usize,
-        _context: Arc<TaskContext>,
-    ) -> Result<SendableRecordBatchStream> {
-        info!("ShuffleReaderExec::execute({})", partition);
-
-        let fetch_time =
-            MetricBuilder::new(&self.metrics).subset_time("fetch_time", partition);
-
-        let locations = self.partition[partition].clone();
-        let stream = locations.into_iter().map(move |p| {
-            let fetch_time = fetch_time.clone();
-            futures::stream::once(async move {
-                let timer = fetch_time.timer();
-                let r = fetch_partition(&p).await;
-                timer.done();
-
-                r.map_err(|e| ArrowError::ExternalError(Box::new(e)))
-            })
-            .try_flatten()
-        });
-
-        let result = RecordBatchStreamAdapter::new(
-            Arc::new(self.schema.as_ref().clone()),
-            futures::stream::iter(stream).flatten(),
-        );
-        Ok(Box::pin(result))
-    }
-
-    fn fmt_as(
-        &self,
-        t: DisplayFormatType,
-        f: &mut std::fmt::Formatter,
-    ) -> std::fmt::Result {
-        match t {
-            DisplayFormatType::Default => {
-                let loc_str = self
-                    .partition
-                    .iter()
-                    .enumerate()
-                    .map(|(partition_id, locations)| {
-                        format!(
-                            "[partition={} paths={}]",
-                            partition_id,
-                            locations
-                                .iter()
-                                .map(|l| l.path.clone())
-                                .collect::<Vec<String>>()
-                                .join(",")
-                        )
-                    })
-                    .collect::<Vec<String>>()
-                    .join(", ");
-                write!(
-                    f,
-                    "ShuffleReaderExec: partition_locations({})={}",
-                    self.partition.len(),
-                    loc_str
-                )
-            }
-        }
-    }
-
-    fn metrics(&self) -> Option<MetricsSet> {
-        Some(self.metrics.clone_inner())
-    }
-
-    fn statistics(&self) -> Statistics {
-        stats_for_partitions(
-            self.partition
-                .iter()
-                .flatten()
-                .map(|loc| loc.partition_stats),
-        )
-    }
-}
-
-fn stats_for_partitions(
-    partition_stats: impl Iterator<Item = PartitionStats>,
-) -> Statistics {
-    // TODO stats: add column statistics to PartitionStats
-    partition_stats.fold(
-        Statistics {
-            is_exact: true,
-            num_rows: Some(0),
-            total_byte_size: Some(0),
-            column_statistics: None,
-        },
-        |mut acc, part| {
-            // if any statistic is unkown it makes the entire statistic unkown
-            acc.num_rows = acc.num_rows.zip(part.num_rows).map(|(a, b)| a + b as usize);
-            acc.total_byte_size = acc
-                .total_byte_size
-                .zip(part.num_bytes)
-                .map(|(a, b)| a + b as usize);
-            acc
-        },
-    )
-}
-
-async fn fetch_partition(
-    location: &PartitionLocation,
-) -> Result<SendableRecordBatchStream> {
-    let metadata = &location.executor_meta;
-    let partition_id = &location.partition_id;
-    let mut ballista_client =
-        BallistaClient::try_new(metadata.host.as_str(), metadata.port as u16)
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-    ballista_client
-        .fetch_partition(
-            &partition_id.job_id,
-            partition_id.stage_id as usize,
-            partition_id.partition_id as usize,
-            &location.path,
-        )
-        .await
-        .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    async fn test_stats_for_partitions_empty() {
-        let result = stats_for_partitions(std::iter::empty());
-
-        let exptected = Statistics {
-            is_exact: true,
-            num_rows: Some(0),
-            total_byte_size: Some(0),
-            column_statistics: None,
-        };
-
-        assert_eq!(result, exptected);
-    }
-
-    #[tokio::test]
-    async fn test_stats_for_partitions_full() {
-        let part_stats = vec![
-            PartitionStats {
-                num_rows: Some(10),
-                num_bytes: Some(84),
-                num_batches: Some(1),
-            },
-            PartitionStats {
-                num_rows: Some(4),
-                num_bytes: Some(65),
-                num_batches: None,
-            },
-        ];
-
-        let result = stats_for_partitions(part_stats.into_iter());
-
-        let exptected = Statistics {
-            is_exact: true,
-            num_rows: Some(14),
-            total_byte_size: Some(149),
-            column_statistics: None,
-        };
-
-        assert_eq!(result, exptected);
-    }
-
-    #[tokio::test]
-    async fn test_stats_for_partitions_missing() {
-        let part_stats = vec![
-            PartitionStats {
-                num_rows: Some(10),
-                num_bytes: Some(84),
-                num_batches: Some(1),
-            },
-            PartitionStats {
-                num_rows: None,
-                num_bytes: None,
-                num_batches: None,
-            },
-        ];
-
-        let result = stats_for_partitions(part_stats.into_iter());
-
-        let exptected = Statistics {
-            is_exact: true,
-            num_rows: None,
-            total_byte_size: None,
-            column_statistics: None,
-        };
-
-        assert_eq!(result, exptected);
-    }
-}
diff --git a/ballista/rust/core/src/execution_plans/shuffle_writer.rs b/ballista/rust/core/src/execution_plans/shuffle_writer.rs
deleted file mode 100644
index 45a102185..000000000
--- a/ballista/rust/core/src/execution_plans/shuffle_writer.rs
+++ /dev/null
@@ -1,559 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! ShuffleWriterExec represents a section of a query plan that has consistent partitioning and
-//! can be executed as one unit with each partition being executed in parallel. The output of each
-//! partition is re-partitioned and streamed to disk in Arrow IPC format. Future stages of the query
-//! will use the ShuffleReaderExec to read these results.
-
-use datafusion::physical_plan::expressions::PhysicalSortExpr;
-
-use std::any::Any;
-use std::future::Future;
-use std::iter::Iterator;
-use std::path::PathBuf;
-use std::sync::Arc;
-use std::time::Instant;
-
-use crate::utils;
-
-use crate::serde::protobuf::ShuffleWritePartition;
-use crate::serde::scheduler::PartitionStats;
-use datafusion::arrow::array::{
-    ArrayBuilder, ArrayRef, StringBuilder, StructBuilder, UInt32Builder, UInt64Builder,
-};
-use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-
-use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::error::{DataFusionError, Result};
-use datafusion::physical_plan::common::IPCWriter;
-use datafusion::physical_plan::memory::MemoryStream;
-use datafusion::physical_plan::metrics::{
-    self, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet,
-};
-
-use datafusion::physical_plan::{
-    DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream, Statistics,
-};
-use futures::{StreamExt, TryFutureExt, TryStreamExt};
-
-use datafusion::arrow::error::ArrowError;
-use datafusion::execution::context::TaskContext;
-use datafusion::physical_plan::repartition::BatchPartitioner;
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use log::{debug, info};
-
-/// ShuffleWriterExec represents a section of a query plan that has consistent partitioning and
-/// can be executed as one unit with each partition being executed in parallel. The output of each
-/// partition is re-partitioned and streamed to disk in Arrow IPC format. Future stages of the query
-/// will use the ShuffleReaderExec to read these results.
-#[derive(Debug, Clone)]
-pub struct ShuffleWriterExec {
-    /// Unique ID for the job (query) that this stage is a part of
-    job_id: String,
-    /// Unique query stage ID within the job
-    stage_id: usize,
-    /// Physical execution plan for this query stage
-    plan: Arc<dyn ExecutionPlan>,
-    /// Path to write output streams to
-    work_dir: String,
-    /// Optional shuffle output partitioning
-    shuffle_output_partitioning: Option<Partitioning>,
-    /// Execution metrics
-    metrics: ExecutionPlanMetricsSet,
-}
-
-#[derive(Debug, Clone)]
-struct ShuffleWriteMetrics {
-    /// Time spend writing batches to shuffle files
-    write_time: metrics::Time,
-    repart_time: metrics::Time,
-    input_rows: metrics::Count,
-    output_rows: metrics::Count,
-}
-
-impl ShuffleWriteMetrics {
-    fn new(partition: usize, metrics: &ExecutionPlanMetricsSet) -> Self {
-        let write_time = MetricBuilder::new(metrics).subset_time("write_time", partition);
-        let repart_time =
-            MetricBuilder::new(metrics).subset_time("repart_time", partition);
-
-        let input_rows = MetricBuilder::new(metrics).counter("input_rows", partition);
-
-        let output_rows = MetricBuilder::new(metrics).output_rows(partition);
-
-        Self {
-            write_time,
-            repart_time,
-            input_rows,
-            output_rows,
-        }
-    }
-}
-
-impl ShuffleWriterExec {
-    /// Create a new shuffle writer
-    pub fn try_new(
-        job_id: String,
-        stage_id: usize,
-        plan: Arc<dyn ExecutionPlan>,
-        work_dir: String,
-        shuffle_output_partitioning: Option<Partitioning>,
-    ) -> Result<Self> {
-        Ok(Self {
-            job_id,
-            stage_id,
-            plan,
-            work_dir,
-            shuffle_output_partitioning,
-            metrics: ExecutionPlanMetricsSet::new(),
-        })
-    }
-
-    /// Get the Job ID for this query stage
-    pub fn job_id(&self) -> &str {
-        &self.job_id
-    }
-
-    /// Get the Stage ID for this query stage
-    pub fn stage_id(&self) -> usize {
-        self.stage_id
-    }
-
-    /// Get the true output partitioning
-    pub fn shuffle_output_partitioning(&self) -> Option<&Partitioning> {
-        self.shuffle_output_partitioning.as_ref()
-    }
-
-    pub fn execute_shuffle_write(
-        &self,
-        input_partition: usize,
-        context: Arc<TaskContext>,
-    ) -> impl Future<Output = Result<Vec<ShuffleWritePartition>>> {
-        let mut path = PathBuf::from(&self.work_dir);
-        path.push(&self.job_id);
-        path.push(&format!("{}", self.stage_id));
-
-        let write_metrics = ShuffleWriteMetrics::new(input_partition, &self.metrics);
-        let output_partitioning = self.shuffle_output_partitioning.clone();
-        let plan = self.plan.clone();
-
-        async move {
-            let now = Instant::now();
-            let mut stream = plan.execute(input_partition, context)?;
-
-            match output_partitioning {
-                None => {
-                    let timer = write_metrics.write_time.timer();
-                    path.push(&format!("{}", input_partition));
-                    std::fs::create_dir_all(&path)?;
-                    path.push("data.arrow");
-                    let path = path.to_str().unwrap();
-                    info!("Writing results to {}", path);
-
-                    // stream results to disk
-                    let stats = utils::write_stream_to_disk(
-                        &mut stream,
-                        path,
-                        &write_metrics.write_time,
-                    )
-                    .await
-                    .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-
-                    write_metrics
-                        .input_rows
-                        .add(stats.num_rows.unwrap_or(0) as usize);
-                    write_metrics
-                        .output_rows
-                        .add(stats.num_rows.unwrap_or(0) as usize);
-                    timer.done();
-
-                    info!(
-                        "Executed partition {} in {} seconds. Statistics: {}",
-                        input_partition,
-                        now.elapsed().as_secs(),
-                        stats
-                    );
-
-                    Ok(vec![ShuffleWritePartition {
-                        partition_id: input_partition as u64,
-                        path: path.to_owned(),
-                        num_batches: stats.num_batches.unwrap_or(0),
-                        num_rows: stats.num_rows.unwrap_or(0),
-                        num_bytes: stats.num_bytes.unwrap_or(0),
-                    }])
-                }
-
-                Some(Partitioning::Hash(exprs, num_output_partitions)) => {
-                    // we won't necessary produce output for every possible partition, so we
-                    // create writers on demand
-                    let mut writers: Vec<Option<IPCWriter>> = vec![];
-                    for _ in 0..num_output_partitions {
-                        writers.push(None);
-                    }
-
-                    let mut partitioner = BatchPartitioner::try_new(
-                        Partitioning::Hash(exprs, num_output_partitions),
-                        write_metrics.repart_time.clone(),
-                    )?;
-
-                    while let Some(result) = stream.next().await {
-                        let input_batch = result?;
-
-                        write_metrics.input_rows.add(input_batch.num_rows());
-
-                        partitioner.partition(
-                            input_batch,
-                            |output_partition, output_batch| {
-                                // write non-empty batch out
-
-                                // TODO optimize so we don't write or fetch empty partitions
-                                // if output_batch.num_rows() > 0 {
-                                let timer = write_metrics.write_time.timer();
-                                match &mut writers[output_partition] {
-                                    Some(w) => {
-                                        w.write(&output_batch)?;
-                                    }
-                                    None => {
-                                        let mut path = path.clone();
-                                        path.push(&format!("{}", output_partition));
-                                        std::fs::create_dir_all(&path)?;
-
-                                        path.push(format!(
-                                            "data-{}.arrow",
-                                            input_partition
-                                        ));
-                                        info!("Writing results to {:?}", path);
-
-                                        let mut writer = IPCWriter::new(
-                                            &path,
-                                            stream.schema().as_ref(),
-                                        )?;
-
-                                        writer.write(&output_batch)?;
-                                        writers[output_partition] = Some(writer);
-                                    }
-                                }
-                                write_metrics.output_rows.add(output_batch.num_rows());
-                                timer.done();
-                                Ok(())
-                            },
-                        )?;
-                    }
-
-                    let mut part_locs = vec![];
-
-                    for (i, w) in writers.iter_mut().enumerate() {
-                        match w {
-                            Some(w) => {
-                                w.finish()?;
-                                info!(
-                                    "Finished writing shuffle partition {} at {:?}. Batches: {}. Rows: {}. Bytes: {}.",
-                                    i,
-                                    w.path(),
-                                    w.num_batches,
-                                    w.num_rows,
-                                    w.num_bytes
-                                );
-
-                                part_locs.push(ShuffleWritePartition {
-                                    partition_id: i as u64,
-                                    path: w.path().to_string_lossy().to_string(),
-                                    num_batches: w.num_batches,
-                                    num_rows: w.num_rows,
-                                    num_bytes: w.num_bytes,
-                                });
-                            }
-                            None => {}
-                        }
-                    }
-                    Ok(part_locs)
-                }
-
-                _ => Err(DataFusionError::Execution(
-                    "Invalid shuffle partitioning scheme".to_owned(),
-                )),
-            }
-        }
-    }
-}
-
-impl ExecutionPlan for ShuffleWriterExec {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.plan.schema()
-    }
-
-    fn output_partitioning(&self) -> Partitioning {
-        // This operator needs to be executed once for each *input* partition and there
-        // isn't really a mechanism yet in DataFusion to support this use case so we report
-        // the input partitioning as the output partitioning here. The executor reports
-        // output partition meta data back to the scheduler.
-        self.plan.output_partitioning()
-    }
-
-    fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> {
-        None
-    }
-
-    fn relies_on_input_order(&self) -> bool {
-        false
-    }
-
-    fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
-        vec![self.plan.clone()]
-    }
-
-    fn with_new_children(
-        self: Arc<Self>,
-        children: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
-        Ok(Arc::new(ShuffleWriterExec::try_new(
-            self.job_id.clone(),
-            self.stage_id,
-            children[0].clone(),
-            self.work_dir.clone(),
-            self.shuffle_output_partitioning.clone(),
-        )?))
-    }
-
-    fn execute(
-        &self,
-        partition: usize,
-        context: Arc<TaskContext>,
-    ) -> Result<SendableRecordBatchStream> {
-        let schema = result_schema();
-
-        let schema_captured = schema.clone();
-        let fut_stream = self
-            .execute_shuffle_write(partition, context)
-            .and_then(|part_loc| async move {
-                // build metadata result batch
-                let num_writers = part_loc.len();
-                let mut partition_builder = UInt32Builder::new(num_writers);
-                let mut path_builder = StringBuilder::new(num_writers);
-                let mut num_rows_builder = UInt64Builder::new(num_writers);
-                let mut num_batches_builder = UInt64Builder::new(num_writers);
-                let mut num_bytes_builder = UInt64Builder::new(num_writers);
-
-                for loc in &part_loc {
-                    path_builder.append_value(loc.path.clone())?;
-                    partition_builder.append_value(loc.partition_id as u32)?;
-                    num_rows_builder.append_value(loc.num_rows)?;
-                    num_batches_builder.append_value(loc.num_batches)?;
-                    num_bytes_builder.append_value(loc.num_bytes)?;
-                }
-
-                // build arrays
-                let partition_num: ArrayRef = Arc::new(partition_builder.finish());
-                let path: ArrayRef = Arc::new(path_builder.finish());
-                let field_builders: Vec<Box<dyn ArrayBuilder>> = vec![
-                    Box::new(num_rows_builder),
-                    Box::new(num_batches_builder),
-                    Box::new(num_bytes_builder),
-                ];
-                let mut stats_builder = StructBuilder::new(
-                    PartitionStats::default().arrow_struct_fields(),
-                    field_builders,
-                );
-                for _ in 0..num_writers {
-                    stats_builder.append(true)?;
-                }
-                let stats = Arc::new(stats_builder.finish());
-
-                // build result batch containing metadata
-                let batch = RecordBatch::try_new(
-                    schema_captured.clone(),
-                    vec![partition_num, path, stats],
-                )?;
-
-                debug!("RESULTS METADATA:\n{:?}", batch);
-
-                MemoryStream::try_new(vec![batch], schema_captured, None)
-            })
-            .map_err(|e| ArrowError::ExternalError(Box::new(e)));
-
-        Ok(Box::pin(RecordBatchStreamAdapter::new(
-            schema,
-            futures::stream::once(fut_stream).try_flatten(),
-        )))
-    }
-
-    fn metrics(&self) -> Option<MetricsSet> {
-        Some(self.metrics.clone_inner())
-    }
-
-    fn fmt_as(
-        &self,
-        t: DisplayFormatType,
-        f: &mut std::fmt::Formatter,
-    ) -> std::fmt::Result {
-        match t {
-            DisplayFormatType::Default => {
-                write!(
-                    f,
-                    "ShuffleWriterExec: {:?}",
-                    self.shuffle_output_partitioning
-                )
-            }
-        }
-    }
-
-    fn statistics(&self) -> Statistics {
-        self.plan.statistics()
-    }
-}
-
-fn result_schema() -> SchemaRef {
-    let stats = PartitionStats::default();
-    Arc::new(Schema::new(vec![
-        Field::new("partition", DataType::UInt32, false),
-        Field::new("path", DataType::Utf8, false),
-        stats.arrow_struct_repr(),
-    ]))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use datafusion::arrow::array::{StringArray, StructArray, UInt32Array, UInt64Array};
-    use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
-    use datafusion::physical_plan::expressions::Column;
-
-    use datafusion::physical_plan::memory::MemoryExec;
-    use datafusion::prelude::SessionContext;
-    use tempfile::TempDir;
-
-    #[tokio::test]
-    // number of rows in each partition is a function of the hash output, so don't test here
-    #[cfg(not(feature = "force_hash_collisions"))]
-    async fn test() -> Result<()> {
-        let session_ctx = SessionContext::new();
-        let task_ctx = session_ctx.task_ctx();
-
-        let input_plan = Arc::new(CoalescePartitionsExec::new(create_input_plan()?));
-        let work_dir = TempDir::new()?;
-        let query_stage = ShuffleWriterExec::try_new(
-            "jobOne".to_owned(),
-            1,
-            input_plan,
-            work_dir.into_path().to_str().unwrap().to_owned(),
-            Some(Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 2)),
-        )?;
-        let mut stream = query_stage.execute(0, task_ctx)?;
-        let batches = utils::collect_stream(&mut stream)
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-        assert_eq!(1, batches.len());
-        let batch = &batches[0];
-        assert_eq!(3, batch.num_columns());
-        assert_eq!(2, batch.num_rows());
-        let path = batch.columns()[1]
-            .as_any()
-            .downcast_ref::<StringArray>()
-            .unwrap();
-
-        let file0 = path.value(0);
-        assert!(
-            file0.ends_with("/jobOne/1/0/data-0.arrow")
-                || file0.ends_with("\\jobOne\\1\\0\\data-0.arrow")
-        );
-        let file1 = path.value(1);
-        assert!(
-            file1.ends_with("/jobOne/1/1/data-0.arrow")
-                || file1.ends_with("\\jobOne\\1\\1\\data-0.arrow")
-        );
-
-        let stats = batch.columns()[2]
-            .as_any()
-            .downcast_ref::<StructArray>()
-            .unwrap();
-
-        let num_rows = stats
-            .column_by_name("num_rows")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<UInt64Array>()
-            .unwrap();
-        assert_eq!(4, num_rows.value(0));
-        assert_eq!(4, num_rows.value(1));
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    // number of rows in each partition is a function of the hash output, so don't test here
-    #[cfg(not(feature = "force_hash_collisions"))]
-    async fn test_partitioned() -> Result<()> {
-        let session_ctx = SessionContext::new();
-        let task_ctx = session_ctx.task_ctx();
-
-        let input_plan = create_input_plan()?;
-        let work_dir = TempDir::new()?;
-        let query_stage = ShuffleWriterExec::try_new(
-            "jobOne".to_owned(),
-            1,
-            input_plan,
-            work_dir.into_path().to_str().unwrap().to_owned(),
-            Some(Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 2)),
-        )?;
-        let mut stream = query_stage.execute(0, task_ctx)?;
-        let batches = utils::collect_stream(&mut stream)
-            .await
-            .map_err(|e| DataFusionError::Execution(format!("{:?}", e)))?;
-        assert_eq!(1, batches.len());
-        let batch = &batches[0];
-        assert_eq!(3, batch.num_columns());
-        assert_eq!(2, batch.num_rows());
-        let stats = batch.columns()[2]
-            .as_any()
-            .downcast_ref::<StructArray>()
-            .unwrap();
-        let num_rows = stats
-            .column_by_name("num_rows")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<UInt64Array>()
-            .unwrap();
-        assert_eq!(2, num_rows.value(0));
-        assert_eq!(2, num_rows.value(1));
-
-        Ok(())
-    }
-
-    fn create_input_plan() -> Result<Arc<dyn ExecutionPlan>> {
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("a", DataType::UInt32, true),
-            Field::new("b", DataType::Utf8, true),
-        ]));
-
-        // define data.
-        let batch = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(UInt32Array::from(vec![Some(1), Some(2)])),
-                Arc::new(StringArray::from(vec![Some("hello"), Some("world")])),
-            ],
-        )?;
-        let partition = vec![batch.clone(), batch];
-        let partitions = vec![partition.clone(), partition];
-        Ok(Arc::new(MemoryExec::try_new(&partitions, schema, None)?))
-    }
-}
diff --git a/ballista/rust/core/src/execution_plans/unresolved_shuffle.rs b/ballista/rust/core/src/execution_plans/unresolved_shuffle.rs
deleted file mode 100644
index 15d403fb6..000000000
--- a/ballista/rust/core/src/execution_plans/unresolved_shuffle.rs
... 30273 lines suppressed ...