You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/09/06 12:34:15 UTC
(datafusion-comet) branch main updated: fix: distinguish "nothing spilled" from a spill backend with no local path (#5726)
This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 135ee53958 fix: distinguish "nothing spilled" from a spill backend with no local path (#5726)
135ee53958 is described below
commit 135ee53958a3aa52b8e38089fe1f6457c1b57003
Author: Andy Grove <ag...@apache.org>
AuthorDate: Sun Sep 6 06:34:09 2026 -0600
fix: distinguish "nothing spilled" from a spill backend with no local path (#5726)
---
native/shuffle/src/shuffle_writer.rs | 2 +-
.../src/writers/local/local_partition_writer.rs | 61 ++++++--
native/shuffle/src/writers/local/spill.rs | 165 +++++++++++++++++++--
3 files changed, 203 insertions(+), 25 deletions(-)
diff --git a/native/shuffle/src/shuffle_writer.rs b/native/shuffle/src/shuffle_writer.rs
index a771be20cc..8d668de772 100644
--- a/native/shuffle/src/shuffle_writer.rs
+++ b/native/shuffle/src/shuffle_writer.rs
@@ -856,7 +856,7 @@ mod test {
.partition_writer()
.get_spill_writers()
.iter()
- .filter_map(|writer| writer.path())
+ .filter_map(|writer| writer.path().unwrap())
.map(|path| usize::try_from(std::fs::metadata(path).unwrap().len()).unwrap())
.sum();
assert_eq!(
diff --git a/native/shuffle/src/writers/local/local_partition_writer.rs b/native/shuffle/src/writers/local/local_partition_writer.rs
index 7f683d95bb..e22e339f94 100644
--- a/native/shuffle/src/writers/local/local_partition_writer.rs
+++ b/native/shuffle/src/writers/local/local_partition_writer.rs
@@ -240,7 +240,7 @@ impl PartitionWriter for LocalPartitionWriter {
// if we wrote a spill file for this partition then copy the
// contents into the shuffle file
if let Some(writer) = spill_writers.get(pid) {
- if let Some(spill_path) = writer.path() {
+ if let Some(spill_path) = writer.path()? {
// Use raw File handle (not BufReader) so that std::io::copy
// can use copy_file_range/sendfile for zero-copy on Linux.
let mut spill_file = File::open(spill_path)?;
@@ -330,25 +330,30 @@ impl PartitionWriter for LocalPartitionWriter {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::writers::local::spill::pathless_backend;
use crate::CompressionCodec;
use arrow::array::Int64Array;
use arrow::datatypes::{DataType, Field, Schema};
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
- /// A partition whose batch iterator fails after a batch was already encoded must hand
- /// back a drained scratch; leftover bytes would land in the next partition's block.
- #[test]
- fn finish_partition_error_drains_recycled_buffer() {
+ fn test_batch() -> RecordBatch {
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let batch = RecordBatch::try_new(
+ RecordBatch::try_new(
Arc::clone(&schema),
vec![Arc::new(Int64Array::from_iter_values(0..100))],
)
- .unwrap();
+ .unwrap()
+ }
+
+ fn partition_writer(
+ batch: &RecordBatch,
+ dir: &tempfile::TempDir,
+ runtime: Arc<RuntimeEnv>,
+ ) -> LocalPartitionWriter {
let block_writer =
- ShuffleBlockWriter::try_new(schema.as_ref(), CompressionCodec::None).unwrap();
- let dir = tempfile::tempdir().unwrap();
- let mut writer = LocalPartitionWriter::try_new(
+ ShuffleBlockWriter::try_new(batch.schema_ref().as_ref(), CompressionCodec::None)
+ .unwrap();
+ LocalPartitionWriter::try_new(
dir.path().join("data.out").to_str().unwrap().to_string(),
dir.path().join("index.out").to_str().unwrap().to_string(),
block_writer,
@@ -356,9 +361,18 @@ mod tests {
// batch_size below the row count so the write serializes into the scratch.
10,
1 << 20,
- Arc::new(RuntimeEnv::default()),
+ runtime,
)
- .unwrap();
+ .unwrap()
+ }
+
+ /// A partition whose batch iterator fails after a batch was already encoded must hand
+ /// back a drained scratch; leftover bytes would land in the next partition's block.
+ #[test]
+ fn finish_partition_error_drains_recycled_buffer() {
+ let batch = test_batch();
+ let dir = tempfile::tempdir().unwrap();
+ let mut writer = partition_writer(&batch, &dir, Arc::new(RuntimeEnv::default()));
let metrics = ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
let mut iter = vec![
@@ -379,4 +393,27 @@ mod tests {
DataOutput::Single { .. } => unreachable!("two partitions use the multi output"),
}
}
+
+ /// Spilled bytes the writer cannot reach must fail the task. Skipping the copy the way
+ /// an unspilled partition is skipped would leave the offsets claiming bytes that were
+ /// never written, so the reader would decode the next partition's block as this one's.
+ #[test]
+ fn finish_partition_fails_when_spill_has_no_local_path() {
+ let batch = test_batch();
+ let dir = tempfile::tempdir().unwrap();
+ let mut writer = partition_writer(&batch, &dir, Arc::new(pathless_backend::runtime()));
+ let metrics = ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+
+ writer
+ .write(0, &mut vec![Ok(batch)].into_iter(), &metrics)
+ .unwrap();
+
+ let err = writer
+ .finish_partition(0, &mut std::iter::empty(), &metrics)
+ .expect_err("unreachable spilled data must fail rather than truncate the partition");
+ assert!(
+ err.to_string().contains("no local path"),
+ "unexpected error: {err}"
+ );
+ }
}
diff --git a/native/shuffle/src/writers/local/spill.rs b/native/shuffle/src/writers/local/spill.rs
index 5aeb2f8877..77fe009046 100644
--- a/native/shuffle/src/writers/local/spill.rs
+++ b/native/shuffle/src/writers/local/spill.rs
@@ -123,10 +123,27 @@ impl SpillWriter {
Ok(())
}
- pub(crate) fn path(&self) -> Option<&std::path::Path> {
- self.spill_file
- .as_ref()
- .and_then(|spill_file| spill_file.temp_file.path())
+ /// Local filesystem path holding this partition's spilled bytes.
+ ///
+ /// * `Ok(None)` — nothing was spilled for this partition.
+ /// * `Ok(Some(path))` — the spilled bytes live at `path`.
+ /// * `Err(..)` — bytes were spilled but the backend exposes no local path.
+ ///
+ /// The last case must stay distinct from `Ok(None)`: a caller that treated it as
+ /// "nothing to copy" would drop the spilled bytes while still recording the
+ /// partition offsets, silently truncating the partition in the shuffle file.
+ pub(crate) fn path(&self) -> datafusion::common::Result<Option<&std::path::Path>> {
+ match self.spill_file.as_ref() {
+ None => Ok(None),
+ Some(spill_file) => match spill_file.temp_file.path() {
+ Some(path) => Ok(Some(path)),
+ None => Err(DataFusionError::Execution(
+ "Shuffle spill file has no local path; the shuffle writer requires a \
+ spill backend backed by local files."
+ .to_string(),
+ )),
+ },
+ }
}
#[cfg(test)]
@@ -135,6 +152,87 @@ impl SpillWriter {
}
}
+#[cfg(test)]
+pub(crate) mod pathless_backend {
+ //! A spill backend that stores bytes somewhere other than the local filesystem, so
+ //! `SpillFile::path()` returns `None`. `DiskManager`'s default backend always has a
+ //! local path, but `TempFileFactory` is pluggable, so the shuffle writer has to cope
+ //! with a backend that does not.
+
+ use bytes::Bytes;
+ use datafusion::common::DataFusionError;
+ use datafusion::execution::disk_manager::DiskManagerBuilder;
+ use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
+ use datafusion::execution::{SpillFile, SpillWriter, TempFileFactory};
+ use futures::Stream;
+ use std::pin::Pin;
+ use std::sync::Arc;
+
+ /// A [`RuntimeEnv`] whose spill files report no local path.
+ pub(crate) fn runtime() -> RuntimeEnv {
+ RuntimeEnvBuilder::new()
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default().with_temp_file_factory(Arc::new(PathlessFactory)),
+ )
+ .build()
+ .unwrap()
+ }
+
+ struct PathlessFactory;
+
+ impl TempFileFactory for PathlessFactory {
+ fn create_temp_file(
+ &self,
+ _description: &str,
+ ) -> datafusion::common::Result<Arc<dyn SpillFile>> {
+ Ok(Arc::new(PathlessSpillFile))
+ }
+ }
+
+ struct PathlessSpillFile;
+
+ impl SpillFile for PathlessSpillFile {
+ // `path()` is left at the trait default, which returns `None`.
+
+ fn size(&self) -> Option<u64> {
+ None
+ }
+
+ fn read_stream(
+ &self,
+ ) -> datafusion::common::Result<
+ Pin<Box<dyn Stream<Item = datafusion::common::Result<Bytes>> + Send>>,
+ > {
+ Err(DataFusionError::NotImplemented(
+ "PathlessSpillFile::read_stream".to_string(),
+ ))
+ }
+
+ fn open_writer(&self) -> datafusion::common::Result<Box<dyn SpillWriter>> {
+ Ok(Box::new(SinkWriter))
+ }
+ }
+
+ /// Accepts and discards every byte, standing in for a backend that writes elsewhere.
+ struct SinkWriter;
+
+ impl std::io::Write for SinkWriter {
+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+ Ok(buf.len())
+ }
+
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
+ }
+ }
+
+ impl SpillWriter for SinkWriter {
+ fn finish(&mut self) -> datafusion::common::Result<()> {
+ Ok(())
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -144,20 +242,29 @@ mod tests {
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
use std::sync::Arc;
- /// A spill whose batch iterator fails after a batch was already encoded must hand
- /// back a drained scratch; leftover bytes would land in the next partition's block.
- #[test]
- fn write_error_drains_recycled_buffer() {
+ fn test_batch() -> RecordBatch {
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
- let batch = RecordBatch::try_new(
+ RecordBatch::try_new(
Arc::clone(&schema),
vec![Arc::new(Int64Array::from_iter_values(0..100))],
)
- .unwrap();
+ .unwrap()
+ }
+
+ fn spill_writer(batch: &RecordBatch, batch_size: usize) -> SpillWriter {
let block_writer =
- ShuffleBlockWriter::try_new(schema.as_ref(), CompressionCodec::None).unwrap();
+ ShuffleBlockWriter::try_new(batch.schema_ref().as_ref(), CompressionCodec::None)
+ .unwrap();
+ SpillWriter::try_new(block_writer, 1 << 20, batch_size).unwrap()
+ }
+
+ /// A spill whose batch iterator fails after a batch was already encoded must hand
+ /// back a drained scratch; leftover bytes would land in the next partition's block.
+ #[test]
+ fn write_error_drains_recycled_buffer() {
+ let batch = test_batch();
// batch_size below the row count so the first write serializes into the scratch.
- let mut spill = SpillWriter::try_new(block_writer, 1 << 20, 10).unwrap();
+ let mut spill = spill_writer(&batch, 10);
let runtime = RuntimeEnv::default();
let metrics = ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
let mut recycled = Vec::new();
@@ -176,4 +283,38 @@ mod tests {
recycled.len()
);
}
+
+ /// A partition that never spilled has no path, and that is not an error.
+ #[test]
+ fn path_is_none_when_nothing_spilled() {
+ let batch = test_batch();
+ let spill = spill_writer(&batch, 10);
+ assert!(!spill.has_spill_file());
+ assert_eq!(spill.path().unwrap(), None);
+ }
+
+ /// Spilling to a backend with no local path must report an error rather than the
+ /// `None` that means "nothing spilled" — see `path`'s doc comment.
+ #[test]
+ fn path_errors_when_backend_has_no_local_path() {
+ let batch = test_batch();
+ let mut spill = spill_writer(&batch, 10);
+ let runtime = pathless_backend::runtime();
+ let metrics = ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+ let mut recycled = Vec::new();
+ let mut iter = vec![Ok(batch)].into_iter();
+
+ spill
+ .write(&mut iter, &runtime, &metrics, &mut recycled)
+ .unwrap();
+ assert!(spill.has_spill_file());
+
+ let err = spill
+ .path()
+ .expect_err("a spill file with no local path must not look like an empty partition");
+ assert!(
+ err.to_string().contains("no local path"),
+ "unexpected error: {err}"
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org