You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@auron.apache.org by ri...@apache.org on 2026/09/11 03:20:57 UTC

(auron) branch master updated: [AURON #2513] Optimize spill error handling and add an IOExceptiion for spill file missing (#2514)

This is an automated email from the ASF dual-hosted git repository.

richox pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git


The following commit(s) were added to refs/heads/master by this push:
     new 81b292296 [AURON #2513] Optimize spill error handling and add an IOExceptiion for spill file missing (#2514)
81b292296 is described below

commit 81b292296007461fa28259704f44d996f2294c2e
Author: Yizhong Zhang <zy...@gmail.com>
AuthorDate: Fri Sep 11 11:20:52 2026 +0800

    [AURON #2513] Optimize spill error handling and add an IOExceptiion for spill file missing (#2514)
    
    # Which issue does this PR close?
    
    Closes #2513
    
    # Rationale for this change
    Memory accounting leaks and misleading exception
    
    # What changes are included in this PR?
    Add an IOException and count memory correctly when spilling
    
    # Are there any user-facing changes?
    No.
    
    # How was this patch tested?
    UTs.
    
    # Was this patch authored or co-authored using generative AI tooling?
    - [ ] Yes
    - [x] No
    
    Co-authored-by: zhangyizhong <zh...@kuaishou.com>
---
 .../spark/sql/auron/memory/OnHeapSpill.scala       | 21 ++++++-
 .../apache/spark/sql/auron/memory/SpillBuf.scala   | 64 +++++++++++++++++++---
 2 files changed, 77 insertions(+), 8 deletions(-)

diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/OnHeapSpill.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/OnHeapSpill.scala
index 23f3e6c42..9ed6431dd 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/OnHeapSpill.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/OnHeapSpill.scala
@@ -113,9 +113,28 @@ case class OnHeapSpill(hsm: SparkOnHeapSpillManager, id: Int) extends Logging {
     spillBuf match {
       case memBasedBuf: MemBasedSpillBuf =>
         val releasingMemory = memUsed
-        spillBuf = memBasedBuf.spill(hsm)
+        val fileBasedBuf =
+          try {
+            memBasedBuf.spill(hsm)
+          } catch {
+            case e: SpillIOException =>
+              // buffers were already released by spill(); give the memory back to
+              // the manager so the accounting does not leak, then rethrow so the
+              // caller sees the real IO cause.
+              spillBuf = new ReleasedSpillBuf(memBasedBuf)
+              hsm.freeMemory(releasingMemory)
+              logError(s"failed to spill in-mem buffer to disk, id=$id", e)
+              throw e
+          }
+        spillBuf = fileBasedBuf
         hsm.freeMemory(releasingMemory)
         releasingMemory
+
+      case _: FileBasedSpillBuf =>
+        0L
+
+      case _: ReleasedSpillBuf =>
+        0L
     }
   }
 }
diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/SpillBuf.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/SpillBuf.scala
index ba2a57492..6b4f0686e 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/SpillBuf.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/SpillBuf.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.spark.sql.auron.memory
 
-import java.io.{File, RandomAccessFile}
+import java.io.{File, IOException, RandomAccessFile}
 import java.nio.ByteBuffer
 import java.nio.channels.FileChannel
 import java.util
@@ -26,6 +26,12 @@ import io.netty.buffer.Unpooled
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.Utils
 
+/**
+ * Thrown when spilling fails because of an underlying IO problem (for example the spark local
+ * directory tree being removed by an external process) rather than because of memory exhaustion.
+ */
+class SpillIOException(message: String, cause: Throwable) extends IOException(message, cause)
+
 abstract class SpillBuf {
   def write(buf: ByteBuffer): Unit
   def read(buf: ByteBuffer): Unit
@@ -85,15 +91,59 @@ class MemBasedSpillBuf extends SpillBuf with Logging {
     logWarning(s"spilling in-mem spill buffer to disk, size=${Utils.bytesToString(size)}")
 
     val startTimeNs = System.nanoTime()
-    val file = hsm.blockManager.diskBlockManager.createTempLocalBlock()._2
-    val channel = new RandomAccessFile(file, "rw").getChannel
+    val file =
+      try {
+        hsm.blockManager.diskBlockManager.createTempLocalBlock()._2
+      } catch {
+        case e: IOException =>
+          throw new SpillIOException(
+            s"failed to create spill file under spark local dirs, the local directory tree" +
+              s" may have been removed by an external process: ${e.getMessage}",
+            e)
+      }
 
-    while (!bufs.isEmpty) {
-      val buf = bufs.removeFirst().nioBuffer()
-      while (buf.remaining() > 0) {
-        channel.write(buf)
+    val channel =
+      try {
+        new RandomAccessFile(file, "rw").getChannel
+      } catch {
+        case e: IOException =>
+          throw new SpillIOException(
+            s"failed to open spill file ${file.getAbsolutePath}, it may have been removed" +
+              s" by an external process: ${e.getMessage}",
+            e)
       }
+
+    // Keep each buffer in the deque until it has been fully written, so that a
+    // partial failure does not silently discard buffered data. On failure the
+    // remaining buffers are released and the accounted memory is handed back to
+    // the caller-visible state via `release()`.
+    try {
+      while (!bufs.isEmpty) {
+        val head = bufs.peekFirst()
+        val buf = head.nioBuffer()
+        while (buf.remaining() > 0) {
+          channel.write(buf)
+        }
+        bufs.removeFirst()
+        mem -= head.capacity()
+      }
+    } catch {
+      case e: IOException =>
+        // the spill file is unusable, drop it and free everything we still hold
+        try {
+          channel.close()
+        } catch {
+          case _: IOException => // ignore, we are already failing
+        }
+        if (file.exists() && !file.delete()) {
+          logWarning(s"Was unable to delete partial spill file: ${file.getAbsolutePath}")
+        }
+        release()
+        throw new SpillIOException(
+          s"failed to write spill file ${file.getAbsolutePath}: ${e.getMessage}",
+          e)
     }
+
     val endTimeNs = System.nanoTime
     new FileBasedSpillBuf(numWrittenBytes, file, channel, endTimeNs - startTimeNs)
   }