You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/09/06 12:34:15 UTC

(datafusion-comet) branch main updated: fix: distinguish "nothing spilled" from a spill backend with no local path (#5726)

This is an automated email from the ASF dual-hosted git repository.

andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


The following commit(s) were added to refs/heads/main by this push:
     new 135ee53958 fix: distinguish "nothing spilled" from a spill backend with no local path (#5726)
135ee53958 is described below

commit 135ee53958a3aa52b8e38089fe1f6457c1b57003
Author: Andy Grove <ag...@apache.org>
AuthorDate: Sun Sep 6 06:34:09 2026 -0600

    fix: distinguish "nothing spilled" from a spill backend with no local path (#5726)
---
 native/shuffle/src/shuffle_writer.rs               |   2 +-
 .../src/writers/local/local_partition_writer.rs    |  61 ++++++--
 native/shuffle/src/writers/local/spill.rs          | 165 +++++++++++++++++++--
 3 files changed, 203 insertions(+), 25 deletions(-)

diff --git a/native/shuffle/src/shuffle_writer.rs b/native/shuffle/src/shuffle_writer.rs
index a771be20cc..8d668de772 100644
--- a/native/shuffle/src/shuffle_writer.rs
+++ b/native/shuffle/src/shuffle_writer.rs
@@ -856,7 +856,7 @@ mod test {
             .partition_writer()
             .get_spill_writers()
             .iter()
-            .filter_map(|writer| writer.path())
+            .filter_map(|writer| writer.path().unwrap())
             .map(|path| usize::try_from(std::fs::metadata(path).unwrap().len()).unwrap())
             .sum();
         assert_eq!(
diff --git a/native/shuffle/src/writers/local/local_partition_writer.rs b/native/shuffle/src/writers/local/local_partition_writer.rs
index 7f683d95bb..e22e339f94 100644
--- a/native/shuffle/src/writers/local/local_partition_writer.rs
+++ b/native/shuffle/src/writers/local/local_partition_writer.rs
@@ -240,7 +240,7 @@ impl PartitionWriter for LocalPartitionWriter {
                 // if we wrote a spill file for this partition then copy the
                 // contents into the shuffle file
                 if let Some(writer) = spill_writers.get(pid) {
-                    if let Some(spill_path) = writer.path() {
+                    if let Some(spill_path) = writer.path()? {
                         // Use raw File handle (not BufReader) so that std::io::copy
                         // can use copy_file_range/sendfile for zero-copy on Linux.
                         let mut spill_file = File::open(spill_path)?;
@@ -330,25 +330,30 @@ impl PartitionWriter for LocalPartitionWriter {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::writers::local::spill::pathless_backend;
     use crate::CompressionCodec;
     use arrow::array::Int64Array;
     use arrow::datatypes::{DataType, Field, Schema};
     use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 
-    /// A partition whose batch iterator fails after a batch was already encoded must hand
-    /// back a drained scratch; leftover bytes would land in the next partition's block.
-    #[test]
-    fn finish_partition_error_drains_recycled_buffer() {
+    fn test_batch() -> RecordBatch {
         let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
-        let batch = RecordBatch::try_new(
+        RecordBatch::try_new(
             Arc::clone(&schema),
             vec![Arc::new(Int64Array::from_iter_values(0..100))],
         )
-        .unwrap();
+        .unwrap()
+    }
+
+    fn partition_writer(
+        batch: &RecordBatch,
+        dir: &tempfile::TempDir,
+        runtime: Arc<RuntimeEnv>,
+    ) -> LocalPartitionWriter {
         let block_writer =
-            ShuffleBlockWriter::try_new(schema.as_ref(), CompressionCodec::None).unwrap();
-        let dir = tempfile::tempdir().unwrap();
-        let mut writer = LocalPartitionWriter::try_new(
+            ShuffleBlockWriter::try_new(batch.schema_ref().as_ref(), CompressionCodec::None)
+                .unwrap();
+        LocalPartitionWriter::try_new(
             dir.path().join("data.out").to_str().unwrap().to_string(),
             dir.path().join("index.out").to_str().unwrap().to_string(),
             block_writer,
@@ -356,9 +361,18 @@ mod tests {
             // batch_size below the row count so the write serializes into the scratch.
             10,
             1 << 20,
-            Arc::new(RuntimeEnv::default()),
+            runtime,
         )
-        .unwrap();
+        .unwrap()
+    }
+
+    /// A partition whose batch iterator fails after a batch was already encoded must hand
+    /// back a drained scratch; leftover bytes would land in the next partition's block.
+    #[test]
+    fn finish_partition_error_drains_recycled_buffer() {
+        let batch = test_batch();
+        let dir = tempfile::tempdir().unwrap();
+        let mut writer = partition_writer(&batch, &dir, Arc::new(RuntimeEnv::default()));
 
         let metrics = ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
         let mut iter = vec![
@@ -379,4 +393,27 @@ mod tests {
             DataOutput::Single { .. } => unreachable!("two partitions use the multi output"),
         }
     }
+
+    /// Spilled bytes the writer cannot reach must fail the task. Skipping the copy the way
+    /// an unspilled partition is skipped would leave the offsets claiming bytes that were
+    /// never written, so the reader would decode the next partition's block as this one's.
+    #[test]
+    fn finish_partition_fails_when_spill_has_no_local_path() {
+        let batch = test_batch();
+        let dir = tempfile::tempdir().unwrap();
+        let mut writer = partition_writer(&batch, &dir, Arc::new(pathless_backend::runtime()));
+        let metrics = ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+
+        writer
+            .write(0, &mut vec![Ok(batch)].into_iter(), &metrics)
+            .unwrap();
+
+        let err = writer
+            .finish_partition(0, &mut std::iter::empty(), &metrics)
+            .expect_err("unreachable spilled data must fail rather than truncate the partition");
+        assert!(
+            err.to_string().contains("no local path"),
+            "unexpected error: {err}"
+        );
+    }
 }
diff --git a/native/shuffle/src/writers/local/spill.rs b/native/shuffle/src/writers/local/spill.rs
index 5aeb2f8877..77fe009046 100644
--- a/native/shuffle/src/writers/local/spill.rs
+++ b/native/shuffle/src/writers/local/spill.rs
@@ -123,10 +123,27 @@ impl SpillWriter {
         Ok(())
     }
 
-    pub(crate) fn path(&self) -> Option<&std::path::Path> {
-        self.spill_file
-            .as_ref()
-            .and_then(|spill_file| spill_file.temp_file.path())
+    /// Local filesystem path holding this partition's spilled bytes.
+    ///
+    /// * `Ok(None)` — nothing was spilled for this partition.
+    /// * `Ok(Some(path))` — the spilled bytes live at `path`.
+    /// * `Err(..)` — bytes were spilled but the backend exposes no local path.
+    ///
+    /// The last case must stay distinct from `Ok(None)`: a caller that treated it as
+    /// "nothing to copy" would drop the spilled bytes while still recording the
+    /// partition offsets, silently truncating the partition in the shuffle file.
+    pub(crate) fn path(&self) -> datafusion::common::Result<Option<&std::path::Path>> {
+        match self.spill_file.as_ref() {
+            None => Ok(None),
+            Some(spill_file) => match spill_file.temp_file.path() {
+                Some(path) => Ok(Some(path)),
+                None => Err(DataFusionError::Execution(
+                    "Shuffle spill file has no local path; the shuffle writer requires a \
+                     spill backend backed by local files."
+                        .to_string(),
+                )),
+            },
+        }
     }
 
     #[cfg(test)]
@@ -135,6 +152,87 @@ impl SpillWriter {
     }
 }
 
+#[cfg(test)]
+pub(crate) mod pathless_backend {
+    //! A spill backend that stores bytes somewhere other than the local filesystem, so
+    //! `SpillFile::path()` returns `None`. `DiskManager`'s default backend always has a
+    //! local path, but `TempFileFactory` is pluggable, so the shuffle writer has to cope
+    //! with a backend that does not.
+
+    use bytes::Bytes;
+    use datafusion::common::DataFusionError;
+    use datafusion::execution::disk_manager::DiskManagerBuilder;
+    use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
+    use datafusion::execution::{SpillFile, SpillWriter, TempFileFactory};
+    use futures::Stream;
+    use std::pin::Pin;
+    use std::sync::Arc;
+
+    /// A [`RuntimeEnv`] whose spill files report no local path.
+    pub(crate) fn runtime() -> RuntimeEnv {
+        RuntimeEnvBuilder::new()
+            .with_disk_manager_builder(
+                DiskManagerBuilder::default().with_temp_file_factory(Arc::new(PathlessFactory)),
+            )
+            .build()
+            .unwrap()
+    }
+
+    struct PathlessFactory;
+
+    impl TempFileFactory for PathlessFactory {
+        fn create_temp_file(
+            &self,
+            _description: &str,
+        ) -> datafusion::common::Result<Arc<dyn SpillFile>> {
+            Ok(Arc::new(PathlessSpillFile))
+        }
+    }
+
+    struct PathlessSpillFile;
+
+    impl SpillFile for PathlessSpillFile {
+        // `path()` is left at the trait default, which returns `None`.
+
+        fn size(&self) -> Option<u64> {
+            None
+        }
+
+        fn read_stream(
+            &self,
+        ) -> datafusion::common::Result<
+            Pin<Box<dyn Stream<Item = datafusion::common::Result<Bytes>> + Send>>,
+        > {
+            Err(DataFusionError::NotImplemented(
+                "PathlessSpillFile::read_stream".to_string(),
+            ))
+        }
+
+        fn open_writer(&self) -> datafusion::common::Result<Box<dyn SpillWriter>> {
+            Ok(Box::new(SinkWriter))
+        }
+    }
+
+    /// Accepts and discards every byte, standing in for a backend that writes elsewhere.
+    struct SinkWriter;
+
+    impl std::io::Write for SinkWriter {
+        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+            Ok(buf.len())
+        }
+
+        fn flush(&mut self) -> std::io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl SpillWriter for SinkWriter {
+        fn finish(&mut self) -> datafusion::common::Result<()> {
+            Ok(())
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -144,20 +242,29 @@ mod tests {
     use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
     use std::sync::Arc;
 
-    /// A spill whose batch iterator fails after a batch was already encoded must hand
-    /// back a drained scratch; leftover bytes would land in the next partition's block.
-    #[test]
-    fn write_error_drains_recycled_buffer() {
+    fn test_batch() -> RecordBatch {
         let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
-        let batch = RecordBatch::try_new(
+        RecordBatch::try_new(
             Arc::clone(&schema),
             vec![Arc::new(Int64Array::from_iter_values(0..100))],
         )
-        .unwrap();
+        .unwrap()
+    }
+
+    fn spill_writer(batch: &RecordBatch, batch_size: usize) -> SpillWriter {
         let block_writer =
-            ShuffleBlockWriter::try_new(schema.as_ref(), CompressionCodec::None).unwrap();
+            ShuffleBlockWriter::try_new(batch.schema_ref().as_ref(), CompressionCodec::None)
+                .unwrap();
+        SpillWriter::try_new(block_writer, 1 << 20, batch_size).unwrap()
+    }
+
+    /// A spill whose batch iterator fails after a batch was already encoded must hand
+    /// back a drained scratch; leftover bytes would land in the next partition's block.
+    #[test]
+    fn write_error_drains_recycled_buffer() {
+        let batch = test_batch();
         // batch_size below the row count so the first write serializes into the scratch.
-        let mut spill = SpillWriter::try_new(block_writer, 1 << 20, 10).unwrap();
+        let mut spill = spill_writer(&batch, 10);
         let runtime = RuntimeEnv::default();
         let metrics = ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
         let mut recycled = Vec::new();
@@ -176,4 +283,38 @@ mod tests {
             recycled.len()
         );
     }
+
+    /// A partition that never spilled has no path, and that is not an error.
+    #[test]
+    fn path_is_none_when_nothing_spilled() {
+        let batch = test_batch();
+        let spill = spill_writer(&batch, 10);
+        assert!(!spill.has_spill_file());
+        assert_eq!(spill.path().unwrap(), None);
+    }
+
+    /// Spilling to a backend with no local path must report an error rather than the
+    /// `None` that means "nothing spilled" — see `path`'s doc comment.
+    #[test]
+    fn path_errors_when_backend_has_no_local_path() {
+        let batch = test_batch();
+        let mut spill = spill_writer(&batch, 10);
+        let runtime = pathless_backend::runtime();
+        let metrics = ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+        let mut recycled = Vec::new();
+        let mut iter = vec![Ok(batch)].into_iter();
+
+        spill
+            .write(&mut iter, &runtime, &metrics, &mut recycled)
+            .unwrap();
+        assert!(spill.has_spill_file());
+
+        let err = spill
+            .path()
+            .expect_err("a spill file with no local path must not look like an empty partition");
+        assert!(
+            err.to_string().contains("no local path"),
+            "unexpected error: {err}"
+        );
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org