You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/09/26 21:11:06 UTC

(datafusion-ballista) branch main updated: ci: run ruff over the whole repository (#2472)

This is an automated email from the ASF dual-hosted git repository.

andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git


The following commit(s) were added to refs/heads/main by this push:
     new 96d41af9c ci: run ruff over the whole repository (#2472)
96d41af9c is described below

commit 96d41af9cd9911cb840a2b8bf1ad83e7442f7440
Author: NoahKusaba <97...@users.noreply.github.com>
AuthorDate: Sat Sep 26 17:11:01 2026 -0400

    ci: run ruff over the whole repository (#2472)
    
    Ruff ran only inside python/, leaving the dev/ scripts, benchmark scripts and
    docs/source/conf.py unchecked. Lint from the repository root; a root ruff.toml
    covers files outside python/. Reformats the newly covered files and fixes the
    real findings; E402 in benchmark scripts is deliberate and ignored per-file.
    
    Co-authored-by: Claude Opus 5.5 <no...@anthropic.com>
---
 .github/workflows/build.yml                   |  5 +-
 benchmarks/db-benchmark/groupby-datafusion.py | 93 +++++++++++++++++++-------
 benchmarks/db-benchmark/join-datafusion.py    | 84 +++++++++++++++++-------
 benchmarks/tpch.py                            | 30 +++++----
 dev/release/check-rat-report.py               | 19 +++---
 dev/release/download-python-wheels.py         | 38 +++++++----
 dev/release/generate-changelog.py             | 80 ++++++++++++++---------
 dev/update_arrow_deps.py                      | 50 +++++++-------
 dev/update_ballista_versions.py               | 82 ++++++++++++-----------
 dev/update_datafusion_proto.py                |  6 +-
 dev/update_datafusion_versions.py             | 94 ++++++++++++++-------------
 docs/source/conf.py                           | 32 ++++-----
 ruff.toml                                     | 23 +++++++
 13 files changed, 393 insertions(+), 243 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ab627634b..749c2b773 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -94,9 +94,10 @@ jobs:
       # Update output format to enable automatic inline annotations.
       - name: Run Ruff
         run: |
+          # Lint the whole repo; ruff uses the nearest pyproject.toml or ruff.toml per file.
           cd python
-          uv run --no-project ruff check --output-format=github .
-          uv run --no-project ruff format --check .
+          uv run --no-project ruff check --output-format=github ..
+          uv run --no-project ruff format --check ..
 
   generate-license:
     if: github.event_name != 'pull_request'
diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py
index 5bf6c36a0..ae2f4f8bc 100644
--- a/benchmarks/db-benchmark/groupby-datafusion.py
+++ b/benchmarks/db-benchmark/groupby-datafusion.py
@@ -29,6 +29,7 @@ from pyarrow import csv as pacsv
 
 # exec(open("./_helpers/helpers.py").read())
 
+
 def ans_shape(batches):
     rows, cols = 0, 0
     for batch in batches:
@@ -36,10 +37,11 @@ def ans_shape(batches):
         if cols == 0:
             cols = batch.num_columns
         else:
-            assert(cols == batch.num_columns)
-    
+            assert cols == batch.num_columns
+
     return rows, cols
 
+
 # ver = df.__version__
 ver = "7.0.0"
 git = ""
@@ -53,7 +55,9 @@ data_name = os.environ["SRC_DATANAME"]
 src_grp = os.path.join("data", data_name + ".csv")
 print("loading dataset %s" % src_grp, flush=True)
 
-data = pacsv.read_csv(src_grp, convert_options=pacsv.ConvertOptions(auto_dict_encode=True))
+data = pacsv.read_csv(
+    src_grp, convert_options=pacsv.ConvertOptions(auto_dict_encode=True)
+)
 print("dataset loaded")
 
 ctx = df.ExecutionContext()
@@ -67,7 +71,7 @@ in_rows = data.num_rows
 
 task_init = timeit.default_timer()
 
-question = "sum v1 by id1" # q1
+question = "sum v1 by id1"  # q1
 gc.collect()
 t_start = timeit.default_timer()
 ans = ctx.sql("SELECT id1, SUM(v1) AS v1 FROM x GROUP BY id1").collect()
@@ -84,7 +88,7 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "sum v1 by id1:id2" # q2
+question = "sum v1 by id1:id2"  # q2
 gc.collect()
 t_start = timeit.default_timer()
 ans = ctx.sql("SELECT id1, id2, SUM(v1) AS v1 FROM x GROUP BY id1, id2").collect()
@@ -101,7 +105,7 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "sum v1 mean v3 by id3" # q3
+question = "sum v1 mean v3 by id3"  # q3
 gc.collect()
 t_start = timeit.default_timer()
 ans = ctx.sql("SELECT id3, SUM(v1) AS v1, AVG(v3) AS v3 FROM x GROUP BY id3").collect()
@@ -112,16 +116,23 @@ print(f"q3: {t}")
 # m = memory_usage()
 t_start = timeit.default_timer()
 df = ctx.create_dataframe([ans])
-chk = df.aggregate([], [f.sum(col("v1")), f.sum(col("v3"))]).collect()[0].to_pandas().to_numpy()[0]
+chk = (
+    df.aggregate([], [f.sum(col("v1")), f.sum(col("v3"))])
+    .collect()[0]
+    .to_pandas()
+    .to_numpy()[0]
+)
 chkt = timeit.default_timer() - t_start
 # write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk)
 del ans
 gc.collect()
 
-question = "mean v1:v3 by id4" # q4
+question = "mean v1:v3 by id4"  # q4
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT id4, AVG(v1) AS v1, AVG(v2) AS v2, AVG(v3) AS v3 FROM x GROUP BY id4").collect()
+ans = ctx.sql(
+    "SELECT id4, AVG(v1) AS v1, AVG(v2) AS v2, AVG(v3) AS v3 FROM x GROUP BY id4"
+).collect()
 shape = ans_shape(ans)
 # print(shape, flush=True)
 t = timeit.default_timer() - t_start
@@ -129,16 +140,23 @@ print(f"q4: {t}")
 # m = memory_usage()
 t_start = timeit.default_timer()
 df = ctx.create_dataframe([ans])
-chk = df.aggregate([], [f.sum(col("v1")), f.sum(col("v2")), f.sum(col("v3"))]).collect()[0].to_pandas().to_numpy()[0]
+chk = (
+    df.aggregate([], [f.sum(col("v1")), f.sum(col("v2")), f.sum(col("v3"))])
+    .collect()[0]
+    .to_pandas()
+    .to_numpy()[0]
+)
 chkt = timeit.default_timer() - t_start
 # write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk)
 del ans
 gc.collect()
 
-question = "sum v1:v3 by id6" # q5
+question = "sum v1:v3 by id6"  # q5
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT id6, SUM(v1) AS v1, SUM(v2) AS v2, SUM(v3) AS v3 FROM x GROUP BY id6").collect()
+ans = ctx.sql(
+    "SELECT id6, SUM(v1) AS v1, SUM(v2) AS v2, SUM(v3) AS v3 FROM x GROUP BY id6"
+).collect()
 shape = ans_shape(ans)
 # print(shape, flush=True)
 t = timeit.default_timer() - t_start
@@ -146,16 +164,23 @@ print(f"q5: {t}")
 # m = memory_usage()
 t_start = timeit.default_timer()
 df = ctx.create_dataframe([ans])
-chk = df.aggregate([], [f.sum(col("v1")), f.sum(col("v2")), f.sum(col("v3"))]).collect()[0].to_pandas().to_numpy()[0]
+chk = (
+    df.aggregate([], [f.sum(col("v1")), f.sum(col("v2")), f.sum(col("v3"))])
+    .collect()[0]
+    .to_pandas()
+    .to_numpy()[0]
+)
 chkt = timeit.default_timer() - t_start
 # write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk)
 del ans
 gc.collect()
 
-question = "median v3 sd v3 by id4 id5" # q6
+question = "median v3 sd v3 by id4 id5"  # q6
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT id4, id5, approx_percentile_cont(v3, .5) AS median_v3, stddev(v3) AS stddev_v3 FROM x GROUP BY id4, id5").collect()
+ans = ctx.sql(
+    "SELECT id4, id5, approx_percentile_cont(v3, .5) AS median_v3, stddev(v3) AS stddev_v3 FROM x GROUP BY id4, id5"
+).collect()
 shape = ans_shape(ans)
 # print(shape, flush=True)
 t = timeit.default_timer() - t_start
@@ -163,16 +188,23 @@ print(f"q6: {t}")
 # m = memory_usage()
 t_start = timeit.default_timer()
 df = ctx.create_dataframe([ans])
-chk = df.aggregate([], [f.sum(col("median_v3")), f.sum(col("stddev_v3"))]).collect()[0].to_pandas().to_numpy()[0]
+chk = (
+    df.aggregate([], [f.sum(col("median_v3")), f.sum(col("stddev_v3"))])
+    .collect()[0]
+    .to_pandas()
+    .to_numpy()[0]
+)
 chkt = timeit.default_timer() - t_start
 # write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk)
 del ans
 gc.collect()
 
-question = "max v1 - min v2 by id3" # q7
+question = "max v1 - min v2 by id3"  # q7
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT id3, MAX(v1) - MIN(v2) AS range_v1_v2 FROM x GROUP BY id3").collect()
+ans = ctx.sql(
+    "SELECT id3, MAX(v1) - MIN(v2) AS range_v1_v2 FROM x GROUP BY id3"
+).collect()
 shape = ans_shape(ans)
 # print(shape, flush=True)
 t = timeit.default_timer() - t_start
@@ -186,10 +218,12 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "largest two v3 by id6" # q8
+question = "largest two v3 by id6"  # q8
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT id6, v3 from (SELECT id6, v3, row_number() OVER (PARTITION BY id6 ORDER BY v3 DESC) AS row FROM x) t WHERE row <= 2").collect()
+ans = ctx.sql(
+    "SELECT id6, v3 from (SELECT id6, v3, row_number() OVER (PARTITION BY id6 ORDER BY v3 DESC) AS row FROM x) t WHERE row <= 2"
+).collect()
 shape = ans_shape(ans)
 # print(shape, flush=True)
 t = timeit.default_timer() - t_start
@@ -203,7 +237,7 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "regression v1 v2 by id2 id4" # q9
+question = "regression v1 v2 by id2 id4"  # q9
 gc.collect()
 t_start = timeit.default_timer()
 ans = ctx.sql("SELECT corr(v1, v2) as corr FROM x GROUP BY id2, id4").collect()
@@ -220,10 +254,12 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "sum v3 count by id1:id6" # q10
+question = "sum v3 count by id1:id6"  # q10
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT id1, id2, id3, id4, id5, id6, SUM(v3) as v3, COUNT(*) AS cnt FROM x GROUP BY id1, id2, id3, id4, id5, id6").collect()
+ans = ctx.sql(
+    "SELECT id1, id2, id3, id4, id5, id6, SUM(v3) as v3, COUNT(*) AS cnt FROM x GROUP BY id1, id2, id3, id4, id5, id6"
+).collect()
 shape = ans_shape(ans)
 # print(shape, flush=True)
 t = timeit.default_timer() - t_start
@@ -231,12 +267,19 @@ print(f"q10: {t}")
 # m = memory_usage()
 t_start = timeit.default_timer()
 df = ctx.create_dataframe([ans])
-chk = df.aggregate([], [f.sum(col("v3")), f.sum(col("cnt"))]).collect()[0].to_pandas().to_numpy()[0]
+chk = (
+    df.aggregate([], [f.sum(col("v3")), f.sum(col("cnt"))])
+    .collect()[0]
+    .to_pandas()
+    .to_numpy()[0]
+)
 chkt = timeit.default_timer() - t_start
 # write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk)
 del ans
 gc.collect()
 
-print("grouping finished, took %0.fs" % (timeit.default_timer() - task_init), flush=True)
+print(
+    "grouping finished, took %0.fs" % (timeit.default_timer() - task_init), flush=True
+)
 
 exit(0)
diff --git a/benchmarks/db-benchmark/join-datafusion.py b/benchmarks/db-benchmark/join-datafusion.py
index 1a83b1f60..04b9af095 100755
--- a/benchmarks/db-benchmark/join-datafusion.py
+++ b/benchmarks/db-benchmark/join-datafusion.py
@@ -29,11 +29,16 @@ from pyarrow import csv as pacsv
 
 # exec(open("./_helpers/helpers.py").read())
 
+
 def join_to_tbls(data_name):
     x_n = int(float(data_name.split("_")[1]))
-    y_n = ["{:.0e}".format(x_n/1e6), "{:.0e}".format(x_n/1e3), "{:.0e}".format(x_n)]
-    y_n = [y_n[0].replace('+0', ''), y_n[1].replace('+0', ''), y_n[2].replace('+0', '')]
-    return [data_name.replace('NA', y_n[0]), data_name.replace('NA', y_n[1]), data_name.replace('NA', y_n[2])]
+    y_n = ["{:.0e}".format(x_n / 1e6), "{:.0e}".format(x_n / 1e3), "{:.0e}".format(x_n)]
+    y_n = [y_n[0].replace("+0", ""), y_n[1].replace("+0", ""), y_n[2].replace("+0", "")]
+    return [
+        data_name.replace("NA", y_n[0]),
+        data_name.replace("NA", y_n[1]),
+        data_name.replace("NA", y_n[2]),
+    ]
 
 
 def ans_shape(batches):
@@ -43,10 +48,11 @@ def ans_shape(batches):
         if cols == 0:
             cols = batch.num_columns
         else:
-            assert(cols == batch.num_columns)
-    
+            assert cols == batch.num_columns
+
     return rows, cols
 
+
 ver = "6.0.0"
 task = "join"
 git = ""
@@ -58,21 +64,43 @@ on_disk = "FALSE"
 data_name = os.environ["SRC_DATANAME"]
 src_jn_x = os.path.join("data", data_name + ".csv")
 y_data_name = join_to_tbls(data_name)
-src_jn_y = [os.path.join("data", y_data_name[0] + ".csv"), os.path.join("data", y_data_name[1] + ".csv"), os.path.join("data", y_data_name[2] + ".csv")]
+src_jn_y = [
+    os.path.join("data", y_data_name[0] + ".csv"),
+    os.path.join("data", y_data_name[1] + ".csv"),
+    os.path.join("data", y_data_name[2] + ".csv"),
+]
 if len(src_jn_y) != 3:
-  raise Exception("Something went wrong in preparing files used for join")
-
-print("loading datasets " + data_name + ", " + y_data_name[0] + ", " + y_data_name[2] + ", " + y_data_name[2], flush=True)
+    raise Exception("Something went wrong in preparing files used for join")
+
+print(
+    "loading datasets "
+    + data_name
+    + ", "
+    + y_data_name[0]
+    + ", "
+    + y_data_name[2]
+    + ", "
+    + y_data_name[2],
+    flush=True,
+)
 
 ctx = df.ExecutionContext()
 
-x_data = pacsv.read_csv(src_jn_x, convert_options=pacsv.ConvertOptions(auto_dict_encode=True))
+x_data = pacsv.read_csv(
+    src_jn_x, convert_options=pacsv.ConvertOptions(auto_dict_encode=True)
+)
 ctx.register_record_batches("x", [x_data.to_batches()])
-small_data = pacsv.read_csv(src_jn_y[0], convert_options=pacsv.ConvertOptions(auto_dict_encode=True))
+small_data = pacsv.read_csv(
+    src_jn_y[0], convert_options=pacsv.ConvertOptions(auto_dict_encode=True)
+)
 ctx.register_record_batches("small", [small_data.to_batches()])
-medium_data = pacsv.read_csv(src_jn_y[1], convert_options=pacsv.ConvertOptions(auto_dict_encode=True))
+medium_data = pacsv.read_csv(
+    src_jn_y[1], convert_options=pacsv.ConvertOptions(auto_dict_encode=True)
+)
 ctx.register_record_batches("medium", [medium_data.to_batches()])
-large_data = pacsv.read_csv(src_jn_y[2], convert_options=pacsv.ConvertOptions(auto_dict_encode=True))
+large_data = pacsv.read_csv(
+    src_jn_y[2], convert_options=pacsv.ConvertOptions(auto_dict_encode=True)
+)
 ctx.register_record_batches("large", [large_data.to_batches()])
 
 print(x_data.num_rows, flush=True)
@@ -83,10 +111,12 @@ print(large_data.num_rows, flush=True)
 task_init = timeit.default_timer()
 print("joining...", flush=True)
 
-question = "small inner on int" # q1
+question = "small inner on int"  # q1
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT x.id1, x.id2, x.id3, x.id4 as xid4, small.id4 as smallid4, x.id5, x.id6, x.v1, small.v2 FROM x INNER JOIN small ON x.id1 = small.id1").collect()
+ans = ctx.sql(
+    "SELECT x.id1, x.id2, x.id3, x.id4 as xid4, small.id4 as smallid4, x.id5, x.id6, x.v1, small.v2 FROM x INNER JOIN small ON x.id1 = small.id1"
+).collect()
 # ans = ctx.sql("SELECT * FROM x INNER JOIN small ON x.id1 = small.id1").collect()
 # print(set([b.schema for b in ans]))
 shape = ans_shape(ans)
@@ -102,10 +132,12 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "medium inner on int" # q2
+question = "medium inner on int"  # q2
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x INNER JOIN medium ON x.id2 = medium.id2").collect()
+ans = ctx.sql(
+    "SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x INNER JOIN medium ON x.id2 = medium.id2"
+).collect()
 shape = ans_shape(ans)
 # print(shape, flush=True)
 t = timeit.default_timer() - t_start
@@ -119,10 +151,12 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "medium outer on int" # q3
+question = "medium outer on int"  # q3
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x LEFT JOIN medium ON x.id2 = medium.id2").collect()
+ans = ctx.sql(
+    "SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x LEFT JOIN medium ON x.id2 = medium.id2"
+).collect()
 shape = ans_shape(ans)
 # print(shape, flush=True)
 t = timeit.default_timer() - t_start
@@ -136,10 +170,12 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "medium inner on factor" # q4
+question = "medium inner on factor"  # q4
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x LEFT JOIN medium ON x.id5 = medium.id5").collect()
+ans = ctx.sql(
+    "SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x LEFT JOIN medium ON x.id5 = medium.id5"
+).collect()
 shape = ans_shape(ans)
 # print(shape)
 t = timeit.default_timer() - t_start
@@ -153,10 +189,12 @@ chkt = timeit.default_timer() - t_start
 del ans
 gc.collect()
 
-question = "big inner on int" # q5
+question = "big inner on int"  # q5
 gc.collect()
 t_start = timeit.default_timer()
-ans = ctx.sql("SELECT x.id1 as xid1, large.id1 as largeid1, x.id2 as xid2, large.id2 as largeid2, x.id3, x.id4 as xid4, large.id4 as largeid4, x.id5 as xid5, large.id5 as largeid5, x.id6 as xid6, large.id6 as largeid6, x.v1, large.v2 FROM x LEFT JOIN large ON x.id3 = large.id3").collect()
+ans = ctx.sql(
+    "SELECT x.id1 as xid1, large.id1 as largeid1, x.id2 as xid2, large.id2 as largeid2, x.id3, x.id4 as xid4, large.id4 as largeid4, x.id5 as xid5, large.id5 as largeid5, x.id6 as xid6, large.id6 as largeid6, x.v1, large.v2 FROM x LEFT JOIN large ON x.id3 = large.id3"
+).collect()
 shape = ans_shape(ans)
 # print(shape)
 t = timeit.default_timer() - t_start
diff --git a/benchmarks/tpch.py b/benchmarks/tpch.py
index 3039403d0..b1fc0157d 100644
--- a/benchmarks/tpch.py
+++ b/benchmarks/tpch.py
@@ -15,14 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import sys
 import time
 import argparse
 
-parser = argparse.ArgumentParser(description='Run SQL benchmarks.')
-parser.add_argument('--query', help='query to run, such as q1')
-parser.add_argument('--path', help='path to data files')
-parser.add_argument('--ext', default='parquet', help='optional file extension, such as parquet')
+parser = argparse.ArgumentParser(description="Run SQL benchmarks.")
+parser.add_argument("--query", help="query to run, such as q1")
+parser.add_argument("--path", help="path to data files")
+parser.add_argument(
+    "--ext", default="parquet", help="optional file extension, such as parquet"
+)
 
 args = parser.parse_args()
 
@@ -32,10 +33,19 @@ table_ext = args.ext
 
 from ballista import BallistaSessionContext
 from datafusion.context import SessionContext
-    
+
 ctx: SessionContext = BallistaSessionContext("df://127.0.0.1:50050")
 
-tables = ["part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation", "region"]
+tables = [
+    "part",
+    "supplier",
+    "partsupp",
+    "customer",
+    "orders",
+    "lineitem",
+    "nation",
+    "region",
+]
 
 for table in tables:
     table_path = path + "/" + table
@@ -44,11 +54,9 @@ for table in tables:
     print("Registering table", table, "at path", table_path)
     ctx.register_parquet(table, table_path)
 
-with open("queries/" + query + ".sql", 'r') as file:
+with open("queries/" + query + ".sql", "r") as file:
     sql = file.read()
 
-import time
-
 start = time.time()
 
 df = ctx.sql(sql)
@@ -56,5 +64,3 @@ df.show()
 
 end = time.time()
 print("Query", query, "took", end - start, "second(s)")
-
-
diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py
index e30d72bdd..d3dd7c5dd 100644
--- a/dev/release/check-rat-report.py
+++ b/dev/release/check-rat-report.py
@@ -23,8 +23,7 @@ import sys
 import xml.etree.ElementTree as ET
 
 if len(sys.argv) != 3:
-    sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
-                     sys.argv[0])
+    sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" % sys.argv[0])
     sys.exit(1)
 
 exclude_globs_filename = sys.argv[1]
@@ -34,26 +33,28 @@ globs = [line.strip() for line in open(exclude_globs_filename, "r")]
 
 tree = ET.parse(xml_filename)
 root = tree.getroot()
-resources = root.findall('resource')
+resources = root.findall("resource")
 
 all_ok = True
 for r in resources:
-    approvals = r.findall('license-approval')
-    if not approvals or approvals[0].attrib['name'] == 'true':
+    approvals = r.findall("license-approval")
+    if not approvals or approvals[0].attrib["name"] == "true":
         continue
-    clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
+    clean_name = re.sub("^[^/]+/", "", r.attrib["name"])
     excluded = False
     for g in globs:
         if fnmatch.fnmatch(clean_name, g):
             excluded = True
             break
     if not excluded:
-        sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
-            clean_name, r.attrib['name'], approvals[0].attrib['name']))
+        sys.stdout.write(
+            "NOT APPROVED: %s (%s): %s\n"
+            % (clean_name, r.attrib["name"], approvals[0].attrib["name"])
+        )
         all_ok = False
 
 if not all_ok:
     sys.exit(1)
 
-print('OK')
+print("OK")
 sys.exit(0)
diff --git a/dev/release/download-python-wheels.py b/dev/release/download-python-wheels.py
index 0395938a9..ef6852ddb 100644
--- a/dev/release/download-python-wheels.py
+++ b/dev/release/download-python-wheels.py
@@ -35,8 +35,9 @@ import io
 
 def main():
     parser = argparse.ArgumentParser(
-        description='Download python binary wheels from release candidate workflow runs.')
-    parser.add_argument('tag', type=str, help='datafusion RC release tag')
+        description="Download python binary wheels from release candidate workflow runs."
+    )
+    parser.add_argument("tag", type=str, help="datafusion RC release tag")
     args = parser.parse_args()
 
     tag = args.tag
@@ -44,7 +45,8 @@ def main():
     if not ghp_token:
         print(
             "ERROR: Personal Github token is required to download workflow artifacts. "
-            "Please specify a token through GH_TOKEN environment variable.")
+            "Please specify a token through GH_TOKEN environment variable."
+        )
         sys.exit(1)
 
     print(f"Downloading latest python wheels for RC tag {tag}...")
@@ -59,7 +61,9 @@ def main():
     # of a release branch (e.g. branch-51), GitHub associates the successful
     # workflow run with the branch rather than the tag, and the branch query
     # only returns the (possibly cancelled) tag-triggered run.
-    commit_url = f"https://api.github.com/repos/apache/datafusion-ballista/commits/{tag}"
+    commit_url = (
+        f"https://api.github.com/repos/apache/datafusion-ballista/commits/{tag}"
+    )
     resp = requests.get(commit_url, headers=headers)
     resp.raise_for_status()
     sha = resp.json()["sha"]
@@ -81,7 +85,8 @@ def main():
     if artifacts_url is None:
         print(
             f"ERROR: Could not find a successful 'Python Release Build' run for "
-            f"commit {sha}. Re-run the workflow or cut a new RC.")
+            f"commit {sha}. Re-run the workflow or cut a new RC."
+        )
         sys.exit(1)
     print(f"Found artifacts url: {artifacts_url}")
 
@@ -94,7 +99,9 @@ def main():
         break
 
     if download_url is None:
-        print(f"ERROR: Could not resolve python wheel download URL from list of artifacts: {artifacts}")
+        print(
+            f"ERROR: Could not resolve python wheel download URL from list of artifacts: {artifacts}"
+        )
         sys.exit(1)
     print(f"Extracting archive from: {download_url}...")
 
@@ -106,11 +113,16 @@ def main():
     for entry in os.listdir("./"):
         if entry.endswith(".whl") or entry.endswith(".tar.gz"):
             print(f"Sign and checksum artifact: {entry}")
-            subprocess.check_output([
-                "gpg", "--armor",
-                "--output", entry+".asc",
-                "--detach-sig", entry,
-            ])
+            subprocess.check_output(
+                [
+                    "gpg",
+                    "--armor",
+                    "--output",
+                    entry + ".asc",
+                    "--detach-sig",
+                    entry,
+                ]
+            )
 
             sha256 = hashlib.sha256()
             sha512 = hashlib.sha512()
@@ -121,12 +133,12 @@ def main():
                         break
                     sha256.update(data)
                     sha512.update(data)
-            with open(entry+".sha256", "w") as fd:
+            with open(entry + ".sha256", "w") as fd:
                 fd.write(sha256.hexdigest())
                 fd.write("  ")
                 fd.write(entry)
                 fd.write("\n")
-            with open(entry+".sha512", "w") as fd:
+            with open(entry + ".sha512", "w") as fd:
                 fd.write(sha512.hexdigest())
                 fd.write("  ")
                 fd.write(entry)
diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py
index 613c651d0..b44ab484a 100755
--- a/dev/release/generate-changelog.py
+++ b/dev/release/generate-changelog.py
@@ -22,18 +22,22 @@ import os
 import re
 import subprocess
 
+
 def print_pulls(repo_name, title, pulls):
-    if len(pulls)  > 0:
+    if len(pulls) > 0:
         print("**{}:**".format(title))
         print()
-        for (pull, commit) in pulls:
+        for pull, commit in pulls:
             url = "https://github.com/{}/pull/{}".format(repo_name, pull.number)
-            print("- {} [#{}]({}) ({})".format(pull.title, pull.number, url, commit.author.login))
+            print(
+                "- {} [#{}]({}) ({})".format(
+                    pull.title, pull.number, url, commit.author.login
+                )
+            )
         print()
 
 
 def generate_changelog(repo, repo_name, tag1, tag2, version):
-
     # get a list of commits between two tags
     print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr)
     comparison = repo.compare(tag1, tag2)
@@ -61,34 +65,31 @@ def generate_changelog(repo, repo_name, tag1, tag2, version):
 
     # categorize the pull requests based on GitHub labels
     print("Categorizing pull requests", file=sys.stderr)
-    for (pull, commit) in all_pulls:
-
+    for pull, commit in all_pulls:
         labels = [label.name for label in pull.labels]
 
         # skip Dependabot dependency-bump PRs (labeled 'auto-dependencies')
-        if 'auto-dependencies' in labels:
+        if "auto-dependencies" in labels:
             continue
 
         # see if PR title uses Conventional Commits
-        cc_type = ''
-        cc_scope = ''
-        cc_breaking = ''
-        parts = re.findall(r'^([a-zA-Z]+)(\([a-zA-Z0-9_-]+\))?(!)?:', pull.title)
+        cc_type = ""
+        cc_breaking = ""
+        parts = re.findall(r"^([a-zA-Z]+)(\([a-zA-Z0-9_-]+\))?(!)?:", pull.title)
         if len(parts) == 1:
             parts_tuple = parts[0]
-            cc_type = parts_tuple[0] # fix, feat, docs, chore
-            cc_scope = parts_tuple[1] # component within project
-            cc_breaking = parts_tuple[2] == '!'
+            cc_type = parts_tuple[0]  # fix, feat, docs, chore
+            cc_breaking = parts_tuple[2] == "!"
 
-        if 'api change' in labels or cc_breaking:
+        if "api change" in labels or cc_breaking:
             breaking.append((pull, commit))
-        elif 'performance' in labels or cc_type == 'perf':
+        elif "performance" in labels or cc_type == "perf":
             performance.append((pull, commit))
-        elif 'bug' in labels or cc_type == 'fix':
+        elif "bug" in labels or cc_type == "fix":
             bugs.append((pull, commit))
-        elif 'enhancement' in labels or cc_type == 'feat':
+        elif "enhancement" in labels or cc_type == "feat":
             enhancements.append((pull, commit))
-        elif 'documentation' in labels or cc_type == 'docs' or cc_type == 'doc':
+        elif "documentation" in labels or cc_type == "docs" or cc_type == "doc":
             docs.append((pull, commit))
         else:
             other.append((pull, commit))
@@ -119,14 +120,20 @@ under the License.
     print(f"# Apache DataFusion Ballista {version} Changelog\n")
 
     # get the number of commits
-    commit_count = subprocess.check_output(["git", "rev-list", "--count", f"{tag1}..{tag2}"], text=True).strip()
+    commit_count = subprocess.check_output(
+        ["git", "rev-list", "--count", f"{tag1}..{tag2}"], text=True
+    ).strip()
 
     # get number of contributors
-    shortlog_output = subprocess.check_output(["git", "shortlog", "-sn", f"{tag1}..{tag2}"], text=True)
+    shortlog_output = subprocess.check_output(
+        ["git", "shortlog", "-sn", f"{tag1}..{tag2}"], text=True
+    )
     contributor_count = len(shortlog_output.strip().splitlines())
 
-    print(f"This release consists of {commit_count} commits from {contributor_count} contributors. "
-          f"See credits at the end of this changelog for more information.\n")
+    print(
+        f"This release consists of {commit_count} commits from {contributor_count} contributors. "
+        f"See credits at the end of this changelog for more information.\n"
+    )
 
     print_pulls(repo_name, "Breaking changes", breaking)
     print_pulls(repo_name, "Fixed bugs", bugs)
@@ -139,25 +146,31 @@ under the License.
     credits = shortlog_output.rstrip()
 
     print("## Credits\n")
-    print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) "
-          "per contributor.\n")
+    print(
+        "Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) "
+        "per contributor.\n"
+    )
     print("```")
     print(credits)
     print("```\n")
 
-    print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing "
-          "PRs, and providing feedback on this release.\n")
+    print(
+        "Thank you also to everyone who contributed in other ways such as filing issues, reviewing "
+        "PRs, and providing feedback on this release.\n"
+    )
+
 
 def resolve_ref(ref):
     """Resolve a git ref (e.g. HEAD, branch name) to a full commit SHA."""
     try:
-        return subprocess.check_output(
-            ["git", "rev-parse", ref], text=True
-        ).strip()
+        return subprocess.check_output(["git", "rev-parse", ref], text=True).strip()
     except subprocess.CalledProcessError:
         # If it can't be resolved locally, return as-is (e.g. a remote tag).
         # The GitHub API will attempt to resolve it.
-        print(f"Note: Could not resolve '{ref}' locally; passing to GitHub API as-is", file=sys.stderr)
+        print(
+            f"Note: Could not resolve '{ref}' locally; passing to GitHub API as-is",
+            file=sys.stderr,
+        )
         return ref
 
 
@@ -169,7 +182,9 @@ def cli(args=None):
     parser = argparse.ArgumentParser()
     parser.add_argument("tag1", help="The previous commit or tag (e.g. 52.0.0)")
     parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)")
-    parser.add_argument("version", help="The version number to include in the changelog")
+    parser.add_argument(
+        "version", help="The version number to include in the changelog"
+    )
     args = parser.parse_args()
 
     # Resolve refs to SHAs so the GitHub API compares the same commits
@@ -185,5 +200,6 @@ def cli(args=None):
     repo = g.get_repo(project)
     generate_changelog(repo, project, tag1, tag2, args.version)
 
+
 if __name__ == "__main__":
     cli()
diff --git a/dev/update_arrow_deps.py b/dev/update_arrow_deps.py
index a991de49e..15ff408cf 100755
--- a/dev/update_arrow_deps.py
+++ b/dev/update_arrow_deps.py
@@ -44,9 +44,9 @@ import requests
 
 # find latest arrow-rs sha
 def get_arrow_sha():
-    url = 'https://api.github.com/repos/apache/arrow-rs/branches/master'
+    url = "https://api.github.com/repos/apache/arrow-rs/branches/master"
     response = requests.get(url)
-    return response.json()['commit']['sha']
+    return response.json()["commit"]["sha"]
 
 
 # Update all entries that look like
@@ -60,66 +60,68 @@ def update_commit_dependencies(dependencies, new_sha):
         return
     for dep_name in dependencies:
         dep = dependencies[dep_name]
-        if hasattr(dep, 'get'):
-            if dep.get('git') == 'https://github.com/apache/arrow-rs':
-                dep['rev'] = new_sha
+        if hasattr(dep, "get"):
+            if dep.get("git") == "https://github.com/apache/arrow-rs":
+                dep["rev"] = new_sha
 
 
 def update_commit_cargo_toml(cargo_toml, new_sha):
-    print('updating {}'.format(cargo_toml.absolute()))
+    print("updating {}".format(cargo_toml.absolute()))
     with open(cargo_toml) as f:
         data = f.read()
 
     doc = tomlkit.parse(data)
 
-    update_commit_dependencies(doc.get('dependencies'), new_sha)
-    update_commit_dependencies(doc.get('dev-dependencies'), new_sha)
+    update_commit_dependencies(doc.get("dependencies"), new_sha)
+    update_commit_dependencies(doc.get("dev-dependencies"), new_sha)
 
-    with open(cargo_toml, 'w') as f:
+    with open(cargo_toml, "w") as f:
         f.write(tomlkit.dumps(doc))
 
 
 def update_version_cargo_toml(cargo_toml, new_version):
-    print('updating {}'.format(cargo_toml.absolute()))
+    print("updating {}".format(cargo_toml.absolute()))
     with open(cargo_toml) as f:
         data = f.read()
 
     doc = tomlkit.parse(data)
 
     for section in ("dependencies", "dev-dependencies"):
-        for (dep_name, constraint) in doc.get(section, {}).items():
-            if dep_name in ("arrow", "parquet", "arrow-flight") and constraint.get("version") is not None:
+        for dep_name, constraint in doc.get(section, {}).items():
+            if (
+                dep_name in ("arrow", "parquet", "arrow-flight")
+                and constraint.get("version") is not None
+            ):
                 doc[section][dep_name]["version"] = new_version
 
-    with open(cargo_toml, 'w') as f:
+    with open(cargo_toml, "w") as f:
         f.write(tomlkit.dumps(doc))
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Update arrow dep versions.')
-    sub_parsers = parser.add_subparsers(help='sub-command help')
+    parser = argparse.ArgumentParser(description="Update arrow dep versions.")
+    sub_parsers = parser.add_subparsers(help="sub-command help")
 
-    parser_version = sub_parsers.add_parser('version', help='update arrow version')
-    parser_version.add_argument('new_version', type=str, help='new arrow version')
-    parser_version.set_defaults(which='version')
+    parser_version = sub_parsers.add_parser("version", help="update arrow version")
+    parser_version.add_argument("new_version", type=str, help="new arrow version")
+    parser_version.set_defaults(which="version")
 
-    parser_commit = sub_parsers.add_parser('commit', help='update arrow commit')
-    parser_commit.set_defaults(which='commit')
+    parser_commit = sub_parsers.add_parser("commit", help="update arrow commit")
+    parser_commit.set_defaults(which="commit")
 
     args = parser.parse_args()
 
     repo_root = Path(__file__).parent.parent.absolute()
 
     if args.which == "version":
-        for cargo_toml in repo_root.rglob('Cargo.toml'):
+        for cargo_toml in repo_root.rglob("Cargo.toml"):
             update_version_cargo_toml(cargo_toml, args.new_version)
     elif args.which == "commit":
         new_sha = get_arrow_sha()
-        print('Updating files in {} to use sha {}'.format(repo_root, new_sha))
-        for cargo_toml in repo_root.rglob('Cargo.toml'):
+        print("Updating files in {} to use sha {}".format(repo_root, new_sha))
+        for cargo_toml in repo_root.rglob("Cargo.toml"):
             update_commit_cargo_toml(cargo_toml, new_sha)
 
 
-
 if __name__ == "__main__":
     main()
diff --git a/dev/update_ballista_versions.py b/dev/update_ballista_versions.py
index 926520f42..bd271b27b 100755
--- a/dev/update_ballista_versions.py
+++ b/dev/update_ballista_versions.py
@@ -30,78 +30,84 @@ import tomlkit
 
 
 def update_cargo_toml(cargo_toml: str, new_version: str):
-    print(f'updating {cargo_toml}')
+    print(f"updating {cargo_toml}")
     with open(cargo_toml) as f:
         data = f.read()
 
     doc = tomlkit.parse(data)
-    if "ballista/" in cargo_toml or "ballista-cli/" in cargo_toml or "python/Cargo.toml" in cargo_toml:
-        doc.get('package')['version'] = new_version
+    if (
+        "ballista/" in cargo_toml
+        or "ballista-cli/" in cargo_toml
+        or "python/Cargo.toml" in cargo_toml
+    ):
+        doc.get("package")["version"] = new_version
 
     # ballista crates also depend on each other
     ballista_deps = (
-        'ballista',
-        'ballista-core',
-        'ballista-executor',
-        'ballista-api-types',
-        'ballista-history',
-        'ballista-scheduler',
-        'ballista-cli',
+        "ballista",
+        "ballista-core",
+        "ballista-executor",
+        "ballista-api-types",
+        "ballista-history",
+        "ballista-scheduler",
+        "ballista-cli",
     )
     for ballista_dep in ballista_deps:
-        dep = doc.get('dependencies', {}).get(ballista_dep)
+        dep = doc.get("dependencies", {}).get(ballista_dep)
         if dep is not None:
-            dep['version'] = new_version
-        dep = doc.get('dev-dependencies', {}).get(ballista_dep)
+            dep["version"] = new_version
+        dep = doc.get("dev-dependencies", {}).get(ballista_dep)
         if dep is not None:
-            dep['version'] = new_version
+            dep["version"] = new_version
 
-    with open(cargo_toml, 'w') as f:
+    with open(cargo_toml, "w") as f:
         f.write(tomlkit.dumps(doc))
 
 
 def update_docker_compose(docker_compose_path: str, new_version: str):
-    print(f'Updating ballista versions in {docker_compose_path}')
+    print(f"Updating ballista versions in {docker_compose_path}")
     with open(docker_compose_path, "r+") as fd:
         data = fd.read()
-        pattern = re.compile(r'(^\s+image:\sballista:)\d+\.\d+\.\d+(-SNAPSHOT)?', re.MULTILINE)
-        data = pattern.sub(r"\g<1>"+new_version, data)
+        pattern = re.compile(
+            r"(^\s+image:\sballista:)\d+\.\d+\.\d+(-SNAPSHOT)?", re.MULTILINE
+        )
+        data = pattern.sub(r"\g<1>" + new_version, data)
         fd.truncate(0)
         fd.seek(0)
         fd.write(data)
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Update ballista crate versions.')
-    parser.add_argument('new_version', type=str, help='new ballista version')
+    parser = argparse.ArgumentParser(description="Update ballista crate versions.")
+    parser.add_argument("new_version", type=str, help="new ballista version")
     args = parser.parse_args()
 
     repo_root = Path(__file__).parent.parent.absolute()
-    ballista_crates = set([
-        os.path.join(repo_root, rel_path, "Cargo.toml")
-        for rel_path in [
-            'ballista-cli',
-            'ballista/core',
-            'ballista/api-types',
-            'ballista/history',
-            'ballista/scheduler',
-            'ballista/executor',
-            'ballista/client',
-            'benchmarks',
-            'examples',
-            'python',
+    ballista_crates = set(
+        [
+            os.path.join(repo_root, rel_path, "Cargo.toml")
+            for rel_path in [
+                "ballista-cli",
+                "ballista/core",
+                "ballista/api-types",
+                "ballista/history",
+                "ballista/scheduler",
+                "ballista/executor",
+                "ballista/client",
+                "benchmarks",
+                "examples",
+                "python",
+            ]
         ]
-    ])
+    )
     new_version = args.new_version
 
-    print(f'Updating ballista versions in {repo_root} to {new_version}')
+    print(f"Updating ballista versions in {repo_root} to {new_version}")
 
     for cargo_toml in ballista_crates:
         update_cargo_toml(cargo_toml, new_version)
 
-    for path in (
-            "docker-compose.yml",
-    ):
+    for path in ("docker-compose.yml",):
         path = os.path.join(repo_root, path)
         update_docker_compose(path, new_version)
 
diff --git a/dev/update_datafusion_proto.py b/dev/update_datafusion_proto.py
index 0e5d6534a..e3599db9a 100755
--- a/dev/update_datafusion_proto.py
+++ b/dev/update_datafusion_proto.py
@@ -64,11 +64,7 @@ FILES = {
 }
 
 # Every crate that might be referenced in FILES must be looked up.
-_ALL_CRATES = {
-    crate
-    for crate_list, _ in FILES.values()
-    for crate in crate_list
-}
+_ALL_CRATES = {crate for crate_list, _ in FILES.values() for crate in crate_list}
 
 
 def crate_source_dirs():
diff --git a/dev/update_datafusion_versions.py b/dev/update_datafusion_versions.py
index d21096ea1..6478d9559 100755
--- a/dev/update_datafusion_versions.py
+++ b/dev/update_datafusion_versions.py
@@ -29,36 +29,37 @@ from pathlib import Path
 import tomlkit
 
 crates = {
-    'datafusion': 'datafusion/core/Cargo.toml',
-    'datafusion-cli': 'datafusion-cli/Cargo.toml',
-    'datafusion-common': 'datafusion/common/Cargo.toml',
-    'datafusion-data-access': 'datafusion/data-access/Cargo.toml',
-    'datafusion-expr': 'datafusion/expr/Cargo.toml',
-    'datafusion-jit': 'datafusion/jit/Cargo.toml',
-    'datafusion-physical-expr': 'datafusion/physical-expr/Cargo.toml',
-    'datafusion-proto': 'datafusion/proto/Cargo.toml',
-    'datafusion-python': 'datafusion-python/Cargo.toml',
-    'datafusion-row': 'datafusion/row/Cargo.toml'
+    "datafusion": "datafusion/core/Cargo.toml",
+    "datafusion-cli": "datafusion-cli/Cargo.toml",
+    "datafusion-common": "datafusion/common/Cargo.toml",
+    "datafusion-data-access": "datafusion/data-access/Cargo.toml",
+    "datafusion-expr": "datafusion/expr/Cargo.toml",
+    "datafusion-jit": "datafusion/jit/Cargo.toml",
+    "datafusion-physical-expr": "datafusion/physical-expr/Cargo.toml",
+    "datafusion-proto": "datafusion/proto/Cargo.toml",
+    "datafusion-python": "datafusion-python/Cargo.toml",
+    "datafusion-row": "datafusion/row/Cargo.toml",
 }
 
 ballista_crates = {
-    'ballista-cli': 'ballista-cli/Cargo.toml',
-    'core': 'ballista/core/Cargo.toml',
-    'client': 'ballista/client/Cargo.toml',
-    'executor': 'ballista/executor/Cargo.toml',
-    'scheduler': 'ballista/scheduler/Cargo.toml',
-    'pyballista': 'python/Cargo.toml',
+    "ballista-cli": "ballista-cli/Cargo.toml",
+    "core": "ballista/core/Cargo.toml",
+    "client": "ballista/client/Cargo.toml",
+    "executor": "ballista/executor/Cargo.toml",
+    "scheduler": "ballista/scheduler/Cargo.toml",
+    "pyballista": "python/Cargo.toml",
 }
 
+
 def update_datafusion_version(cargo_toml: str, new_version: str):
-    print(f'updating {cargo_toml}')
+    print(f"updating {cargo_toml}")
     with open(cargo_toml) as f:
         data = f.read()
 
     doc = tomlkit.parse(data)
-    doc.get('package')['version'] = new_version
+    doc.get("package")["version"] = new_version
 
-    with open(cargo_toml, 'w') as f:
+    with open(cargo_toml, "w") as f:
         f.write(tomlkit.dumps(doc))
 
 
@@ -69,46 +70,49 @@ def update_downstream_versions(cargo_toml: str, new_version: str):
     doc = tomlkit.parse(data)
 
     for crate in crates.keys():
-        df_dep = doc.get('dependencies', {}).get(crate)
+        df_dep = doc.get("dependencies", {}).get(crate)
         # skip crates that pin datafusion using git hash
-        if df_dep is not None and df_dep.get('version') is not None:
-            print(f'updating {crate} dependency in {cargo_toml}')
-            df_dep['version'] = new_version
+        if df_dep is not None and df_dep.get("version") is not None:
+            print(f"updating {crate} dependency in {cargo_toml}")
+            df_dep["version"] = new_version
 
-        df_dep = doc.get('dev-dependencies', {}).get(crate)
-        if df_dep is not None and df_dep.get('version') is not None:
-            print(f'updating {crate} dev-dependency in {cargo_toml}')
-            df_dep['version'] = new_version
+        df_dep = doc.get("dev-dependencies", {}).get(crate)
+        if df_dep is not None and df_dep.get("version") is not None:
+            print(f"updating {crate} dev-dependency in {cargo_toml}")
+            df_dep["version"] = new_version
 
-    with open(cargo_toml, 'w') as f:
+    with open(cargo_toml, "w") as f:
         f.write(tomlkit.dumps(doc))
 
 
 def update_docs(path: str, new_version: str):
     print(f"updating docs in {path}")
-    with open(path, 'r+') as fd:
+    with open(path, "r+") as fd:
         content = fd.read()
         fd.seek(0)
-        content = re.sub(r'datafusion = "(.+)"', f'datafusion = "{new_version}"', content)
+        content = re.sub(
+            r'datafusion = "(.+)"', f'datafusion = "{new_version}"', content
+        )
         fd.write(content)
 
 
 def update_pyproject_toml(path: str, new_version: str):
-    print(f'updating datafusion pin in {path}')
+    print(f"updating datafusion pin in {path}")
     with open(path) as f:
         data = f.read()
     new_data = re.sub(r'"datafusion==[^"]+"', f'"datafusion=={new_version}"', data)
-    with open(path, 'w') as f:
+    with open(path, "w") as f:
         f.write(new_data)
 
 
 def main():
     parser = argparse.ArgumentParser(
         description=(
-            'Update datafusion crate version and corresponding version pins '
-            'in downstream crates.'
-        ))
-    parser.add_argument('new_version', type=str, help='new datafusion version')
+            "Update datafusion crate version and corresponding version pins "
+            "in downstream crates."
+        )
+    )
+    parser.add_argument("new_version", type=str, help="new datafusion version")
     args = parser.parse_args()
 
     new_version = args.new_version
@@ -118,12 +122,12 @@ def main():
     # the upstream apache/datafusion repo; those paths don't exist here. Only the keys are
     # load-bearing in this repo (used to find which deps to bump in ballista crates), so
     # skip paths that aren't present locally rather than erroring.
-    print(f'Updating datafusion crate versions in {repo_root} to {new_version}')
+    print(f"Updating datafusion crate versions in {repo_root} to {new_version}")
     for cargo_toml in crates.values():
         if os.path.exists(os.path.join(repo_root, cargo_toml)):
             update_datafusion_version(cargo_toml, new_version)
 
-    print(f'Updating datafusion dependency versions in {repo_root} to {new_version}')
+    print(f"Updating datafusion dependency versions in {repo_root} to {new_version}")
     for cargo_toml in crates.values():
         if os.path.exists(os.path.join(repo_root, cargo_toml)):
             update_downstream_versions(cargo_toml, new_version)
@@ -131,14 +135,16 @@ def main():
         update_downstream_versions(os.path.join(repo_root, cargo_toml), new_version)
 
     update_docs("README.md", new_version)
-    update_pyproject_toml(os.path.join(repo_root, "python", "pyproject.toml"), new_version)
+    update_pyproject_toml(
+        os.path.join(repo_root, "python", "pyproject.toml"), new_version
+    )
 
     print(
-        '\nNext steps:\n'
-        '  1. Run `cargo update` (or `cargo build`) so Cargo.lock resolves the new '
-        'datafusion crates.\n'
-        '  2. Run `dev/update_datafusion_proto.py` to re-sync the vendored '
-        'DataFusion proto files, then commit the result.'
+        "\nNext steps:\n"
+        "  1. Run `cargo update` (or `cargo build`) so Cargo.lock resolves the new "
+        "datafusion crates.\n"
+        "  2. Run `dev/update_datafusion_proto.py` to re-sync the vendored "
+        "DataFusion proto files, then commit the result."
     )
 
 
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 3f604d8af..249542862 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -33,9 +33,9 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'Apache DataFusion Ballista'
-copyright = '2019-2024, Apache Software Foundation'
-author = 'Apache Software Foundation'
+project = "Apache DataFusion Ballista"
+copyright = "2019-2024, Apache Software Foundation"
+author = "Apache Software Foundation"
 
 
 # -- General configuration ---------------------------------------------------
@@ -44,23 +44,23 @@ author = 'Apache Software Foundation'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
-    'sphinx.ext.doctest',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
-    'myst_parser',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.doctest",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.napoleon",
+    "myst_parser",
 ]
 
 source_suffix = {
-    '.rst': 'restructuredtext',
-    '.md': 'markdown',
+    ".rst": "restructuredtext",
+    ".md": "markdown",
 }
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -82,7 +82,7 @@ autosummary_generate = True
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'pydata_sphinx_theme'
+html_theme = "pydata_sphinx_theme"
 
 html_theme_options = {
     "logo": {
@@ -104,7 +104,7 @@ html_context = {
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 html_logo = "_static/images/ballista-logo.png"
 
diff --git a/ruff.toml b/ruff.toml
new file mode 100644
index 000000000..2a2fb6df1
--- /dev/null
+++ b/ruff.toml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Config for Python outside python/, which has its own in pyproject.toml.
+
+[lint.per-file-ignores]
+# Benchmark scripts intentionally run code before their imports.
+"benchmarks/db-benchmark/*.py" = ["E402"]
+"benchmarks/tpch.py" = ["E402"]


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org