You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by al...@apache.org on 2025/04/05 11:09:10 UTC

(datafusion) branch main updated: Improve spill performance: Disable re-validation of spilled files (#15454)

This is an automated email from the ASF dual-hosted git repository.

alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 5c31692944 Improve spill performance: Disable re-validation of spilled files (#15454)
5c31692944 is described below

commit 5c316929445552f0aff570627a0e232c4f36c59e
Author: zeb <ze...@gmail.com>
AuthorDate: Sat Apr 5 19:09:06 2025 +0800

    Improve spill performance: Disable re-validation of spilled files (#15454)
    
    * add skip_validation and comment
    
    * add benchmark
    
    * fmt
    
    * update sqllogictests and fmt Cargo.toml
    
    * use SpillManager in benchmark
    
    * fix clippy
    
    * upgrade dependencies
    
    * Update comment
    
    Co-authored-by: Daniël Heres <da...@gmail.com>
    
    ---------
    
    Co-authored-by: zeb <ze...@zeb-2.local>
    Co-authored-by: Daniël Heres <da...@gmail.com>
    Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
---
 Cargo.lock                                        |  80 +++++++-------
 Cargo.toml                                        |  14 +--
 datafusion/physical-plan/Cargo.toml               |   5 +
 datafusion/physical-plan/benches/spill_io.rs      | 123 ++++++++++++++++++++++
 datafusion/physical-plan/src/lib.rs               |   1 +
 datafusion/physical-plan/src/spill/mod.rs         |   5 +-
 datafusion/sqllogictest/test_files/dates.slt      |   2 +-
 datafusion/sqllogictest/test_files/timestamps.slt |  10 +-
 8 files changed, 190 insertions(+), 50 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index c1e5208a51..2bc56b070a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -246,9 +246,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
 [[package]]
 name = "arrow"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc208515aa0151028e464cc94a692156e945ce5126abd3537bb7fd6ba2143ed1"
+checksum = "84ef243634a39fb6e9d1710737e7a5ef96c9bacabd2326859ff889bc9ef755e5"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -270,9 +270,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e07e726e2b3f7816a85c6a45b6ec118eeeabf0b2a8c208122ad949437181f49a"
+checksum = "8f420c6aef51dad2e4a96ce29c0ec90ad84880bdb60b321c74c652a6be07b93f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -284,9 +284,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2262eba4f16c78496adfd559a29fe4b24df6088efc9985a873d58e92be022d5"
+checksum = "24bda5ff6461a4ff9739959b3d57b377f45e3f878f7be1a4f28137c0a8f339fa"
 dependencies = [
  "ahash 0.8.11",
  "arrow-buffer",
@@ -301,9 +301,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e899dade2c3b7f5642eb8366cfd898958bcca099cde6dfea543c7e8d3ad88d4"
+checksum = "bc6ed265c73f134a583d02c3cab5e16afab9446d8048ede8707e31f85fad58a0"
 dependencies = [
  "bytes",
  "half",
@@ -312,9 +312,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4103d88c5b441525ed4ac23153be7458494c2b0c9a11115848fdb9b81f6f886a"
+checksum = "01c648572391edcef10e5fd458db70ba27ed6f71bcaee04397d0cfb100b34f8b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -333,9 +333,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43d3cb0914486a3cae19a5cad2598e44e225d53157926d0ada03c20521191a65"
+checksum = "a02fb265a6d8011a7d3ad1a36f25816ad0a3bb04cb8e9fe7929c165b98c0cbcd"
 dependencies = [
  "arrow-array",
  "arrow-cast",
@@ -349,9 +349,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a329fb064477c9ec5f0870d2f5130966f91055c7c5bce2b3a084f116bc28c3b"
+checksum = "5f2cebf504bb6a92a134a87fff98f01b14fbb3a93ecf7aef90cd0f888c5fffa4"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -361,9 +361,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-flight"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7408f2bf3b978eddda272c7699f439760ebc4ac70feca25fefa82c5b8ce808d"
+checksum = "aecab6e2ed46c553354ffbc6cb76ac3eaa61dc07bd13d7de8ece233933bcf679"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -388,9 +388,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddecdeab02491b1ce88885986e25002a3da34dd349f682c7cfe67bab7cc17b86"
+checksum = "8e6405b287671c88846e7751f7291f717b164911474cabac6d3d8614d5aa7374"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -402,9 +402,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d03b9340013413eb84868682ace00a1098c81a5ebc96d279f7ebf9a4cac3c0fd"
+checksum = "5329bf9e7390cbb6b117ddd4d82e94c5362ea4cab5095697139429f36a38350c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -415,16 +415,18 @@ dependencies = [
  "half",
  "indexmap 2.8.0",
  "lexical-core",
+ "memchr",
  "num",
  "serde",
  "serde_json",
+ "simdutf8",
 ]
 
 [[package]]
 name = "arrow-ord"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f841bfcc1997ef6ac48ee0305c4dfceb1f7c786fe31e67c1186edf775e1f1160"
+checksum = "e103c13d4b80da28339c1d7aa23dd85bd59f42158acc45d39eeb6770627909ce"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -435,9 +437,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1eeb55b0a0a83851aa01f2ca5ee5648f607e8506ba6802577afdda9d75cdedcd"
+checksum = "170549a11b8534f3097a0619cfe89c42812345dc998bcf81128fc700b84345b8"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -448,9 +450,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85934a9d0261e0fa5d4e2a5295107d743b543a6e0484a835d4b8db2da15306f9"
+checksum = "a5c53775bba63f319189f366d2b86e9a8889373eb198f07d8544938fc9f8ed9a"
 dependencies = [
  "bitflags 2.8.0",
  "serde",
@@ -458,9 +460,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-select"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e2932aece2d0c869dd2125feb9bd1709ef5c445daa3838ac4112dcfa0fda52c"
+checksum = "0a99003b2eb562b8d9c99dfb672306f15e94b20d3734179d596895703e821dcf"
 dependencies = [
  "ahash 0.8.11",
  "arrow-array",
@@ -472,9 +474,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "912e38bd6a7a7714c1d9b61df80315685553b7455e8a6045c27531d8ecd5b458"
+checksum = "90fdb130ee8325f4cd8262e19bb6baa3cbcef2b2573c4bee8c6fda7ea08199d7"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -1361,9 +1363,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
 [[package]]
 name = "chrono"
-version = "0.4.39"
+version = "0.4.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
+checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -1371,7 +1373,7 @@ dependencies = [
  "num-traits",
  "serde",
  "wasm-bindgen",
- "windows-targets 0.52.6",
+ "windows-link",
 ]
 
 [[package]]
@@ -2490,6 +2492,7 @@ dependencies = [
  "rand 0.8.5",
  "rstest",
  "rstest_reuse",
+ "tempfile",
  "tokio",
 ]
 
@@ -4364,9 +4367,9 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "54.2.1"
+version = "54.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f88838dca3b84d41444a0341b19f347e8098a3898b0f21536654b8b799e11abd"
+checksum = "94243778210509a5a5e9e012872127180c155d73a9cd6e2df9243d213e81e100"
 dependencies = [
  "ahash 0.8.11",
  "arrow-array",
@@ -4396,7 +4399,6 @@ dependencies = [
  "tokio",
  "twox-hash",
  "zstd",
- "zstd-sys",
 ]
 
 [[package]]
@@ -7126,6 +7128,12 @@ dependencies = [
  "syn 2.0.100",
 ]
 
+[[package]]
+name = "windows-link"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
+
 [[package]]
 name = "windows-registry"
 version = "0.2.0"
diff --git a/Cargo.toml b/Cargo.toml
index d26446c111..a930174e2a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -87,19 +87,19 @@ ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
 apache-avro = { version = "0.17", default-features = false }
-arrow = { version = "54.2.1", features = [
+arrow = { version = "54.3.0", features = [
     "prettyprint",
     "chrono-tz",
 ] }
-arrow-buffer = { version = "54.1.0", default-features = false }
-arrow-flight = { version = "54.2.1", features = [
+arrow-buffer = { version = "54.3.0", default-features = false }
+arrow-flight = { version = "54.3.0", features = [
     "flight-sql-experimental",
 ] }
-arrow-ipc = { version = "54.2.0", default-features = false, features = [
+arrow-ipc = { version = "54.3.0", default-features = false, features = [
     "lz4",
 ] }
-arrow-ord = { version = "54.1.0", default-features = false }
-arrow-schema = { version = "54.1.0", default-features = false }
+arrow-ord = { version = "54.3.0", default-features = false }
+arrow-schema = { version = "54.3.0", default-features = false }
 async-trait = "0.1.88"
 bigdecimal = "0.4.7"
 bytes = "1.10"
@@ -149,7 +149,7 @@ itertools = "0.14"
 log = "^0.4"
 object_store = { version = "0.11.0", default-features = false }
 parking_lot = "0.12"
-parquet = { version = "54.2.1", default-features = false, features = [
+parquet = { version = "54.3.0", default-features = false, features = [
     "arrow",
     "async",
     "object_store",
diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml
index 1f38e2ed31..5210ee2675 100644
--- a/datafusion/physical-plan/Cargo.toml
+++ b/datafusion/physical-plan/Cargo.toml
@@ -72,6 +72,7 @@ insta = { workspace = true }
 rand = { workspace = true }
 rstest = { workspace = true }
 rstest_reuse = "0.7.0"
+tempfile = "3.19.1"
 tokio = { workspace = true, features = [
     "rt-multi-thread",
     "fs",
@@ -81,3 +82,7 @@ tokio = { workspace = true, features = [
 [[bench]]
 harness = false
 name = "partial_ordering"
+
+[[bench]]
+harness = false
+name = "spill_io"
diff --git a/datafusion/physical-plan/benches/spill_io.rs b/datafusion/physical-plan/benches/spill_io.rs
new file mode 100644
index 0000000000..3b877671ad
--- /dev/null
+++ b/datafusion/physical-plan/benches/spill_io.rs
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{
+    Date32Builder, Decimal128Builder, Int32Builder, RecordBatch, StringBuilder,
+};
+use arrow::datatypes::{DataType, Field, Schema};
+use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion};
+use datafusion_execution::runtime_env::RuntimeEnv;
+use datafusion_physical_plan::common::collect;
+use datafusion_physical_plan::metrics::{ExecutionPlanMetricsSet, SpillMetrics};
+use datafusion_physical_plan::SpillManager;
+use std::sync::Arc;
+use tokio::runtime::Runtime;
+
+pub fn create_batch(num_rows: usize, allow_nulls: bool) -> RecordBatch {
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("c0", DataType::Int32, true),
+        Field::new("c1", DataType::Utf8, true),
+        Field::new("c2", DataType::Date32, true),
+        Field::new("c3", DataType::Decimal128(11, 2), true),
+    ]));
+
+    let mut a = Int32Builder::new();
+    let mut b = StringBuilder::new();
+    let mut c = Date32Builder::new();
+    let mut d = Decimal128Builder::new()
+        .with_precision_and_scale(11, 2)
+        .unwrap();
+
+    for i in 0..num_rows {
+        a.append_value(i as i32);
+        c.append_value(i as i32);
+        d.append_value((i * 1000000) as i128);
+        if allow_nulls && i % 10 == 0 {
+            b.append_null();
+        } else {
+            b.append_value(format!("this is string number {i}"));
+        }
+    }
+
+    let a = a.finish();
+    let b = b.finish();
+    let c = c.finish();
+    let d = d.finish();
+
+    RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(a), Arc::new(b), Arc::new(c), Arc::new(d)],
+    )
+    .unwrap()
+}
+
+// BENCHMARK: REVALIDATION OVERHEAD COMPARISON
+// ---------------------------------------------------------
+// To compare performance with/without Arrow IPC validation:
+//
+// 1. Locate the function `read_spill`
+// 2. Modify the `skip_validation` flag:
+//    - Set to `false` to enable validation
+// 3. Rerun `cargo bench --bench spill_io`
+fn bench_spill_io(c: &mut Criterion) {
+    let env = Arc::new(RuntimeEnv::default());
+    let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("c0", DataType::Int32, true),
+        Field::new("c1", DataType::Utf8, true),
+        Field::new("c2", DataType::Date32, true),
+        Field::new("c3", DataType::Decimal128(11, 2), true),
+    ]));
+    let spill_manager = SpillManager::new(env, metrics, schema);
+
+    let mut group = c.benchmark_group("spill_io");
+    let rt = Runtime::new().unwrap();
+
+    group.bench_with_input(
+        BenchmarkId::new("StreamReader/read_100", ""),
+        &spill_manager,
+        |b, spill_manager| {
+            b.iter_batched(
+                // Setup phase: Create fresh state for each benchmark iteration.
+                // - generate an ipc file.
+                // This ensures each iteration starts with clean resources.
+                || {
+                    let batch = create_batch(8192, true);
+                    spill_manager
+                        .spill_record_batch_and_finish(&vec![batch; 100], "Test")
+                        .unwrap()
+                        .unwrap()
+                },
+                // Benchmark phase:
+                // - Execute the read operation via SpillManager
+                // - Wait for the consumer to finish processing
+                |spill_file| {
+                    rt.block_on(async {
+                        let stream =
+                            spill_manager.read_spill_as_stream(spill_file).unwrap();
+                        let _ = collect(stream).await.unwrap();
+                    })
+                },
+                BatchSize::LargeInput,
+            )
+        },
+    );
+    group.finish();
+}
+
+criterion_group!(benches, bench_spill_io);
+criterion_main!(benches);
diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs
index 04fbd06fab..b256e615b2 100644
--- a/datafusion/physical-plan/src/lib.rs
+++ b/datafusion/physical-plan/src/lib.rs
@@ -50,6 +50,7 @@ pub use crate::ordering::InputOrderMode;
 pub use crate::stream::EmptyRecordBatchStream;
 pub use crate::topk::TopK;
 pub use crate::visitor::{accept, visit_execution_plan, ExecutionPlanVisitor};
+pub use spill::spill_manager::SpillManager;
 
 mod ordering;
 mod render_tree;
diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs
index 88bf7953da..c32711a96b 100644
--- a/datafusion/physical-plan/src/spill/mod.rs
+++ b/datafusion/physical-plan/src/spill/mod.rs
@@ -35,7 +35,10 @@ use datafusion_common::{exec_datafusion_err, HashSet, Result};
 
 fn read_spill(sender: Sender<Result<RecordBatch>>, path: &Path) -> Result<()> {
     let file = BufReader::new(File::open(path)?);
-    let reader = StreamReader::try_new(file, None)?;
+    // SAFETY: DataFusion's spill writer strictly follows Arrow IPC specifications
+    // with validated schemas and buffers. Skip redundant validation during read
+    // to speedup read operation. This is safe for DataFusion as input guaranteed to be correct when written.
+    let reader = unsafe { StreamReader::try_new(file, None)?.with_skip_validation(true) };
     for batch in reader {
         sender
             .blocking_send(batch.map_err(Into::into))
diff --git a/datafusion/sqllogictest/test_files/dates.slt b/datafusion/sqllogictest/test_files/dates.slt
index 4425eee333..148f0dfe64 100644
--- a/datafusion/sqllogictest/test_files/dates.slt
+++ b/datafusion/sqllogictest/test_files/dates.slt
@@ -183,7 +183,7 @@ query error input contains invalid characters
 SELECT to_date('2020-09-08 12/00/00+00:00', '%c', '%+')
 
 # to_date with broken formatting
-query error bad or unsupported format string
+query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input
 SELECT to_date('2020-09-08 12/00/00+00:00', '%q')
 
 statement ok
diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt
index e042e3863b..16168eeae4 100644
--- a/datafusion/sqllogictest/test_files/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/timestamps.slt
@@ -2268,23 +2268,23 @@ query error input contains invalid characters
 SELECT to_timestamp_seconds('2020-09-08 12/00/00+00:00', '%c', '%+')
 
 # to_timestamp with broken formatting
-query error bad or unsupported format string
+query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input
 SELECT to_timestamp('2020-09-08 12/00/00+00:00', '%q')
 
 # to_timestamp_nanos with broken formatting
-query error bad or unsupported format string
+query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input
 SELECT to_timestamp_nanos('2020-09-08 12/00/00+00:00', '%q')
 
 # to_timestamp_millis with broken formatting
-query error bad or unsupported format string
+query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input
 SELECT to_timestamp_millis('2020-09-08 12/00/00+00:00', '%q')
 
 # to_timestamp_micros with broken formatting
-query error bad or unsupported format string
+query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input
 SELECT to_timestamp_micros('2020-09-08 12/00/00+00:00', '%q')
 
 # to_timestamp_seconds with broken formatting
-query error bad or unsupported format string
+query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input
 SELECT to_timestamp_seconds('2020-09-08 12/00/00+00:00', '%q')
 
 # Create string timestamp table with different formats


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org