You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@auron.apache.org by ri...@apache.org on 2026/09/11 03:20:57 UTC
(auron) branch master updated: [AURON #2513] Optimize spill error handling and add an IOExceptiion for spill file missing (#2514)
This is an automated email from the ASF dual-hosted git repository.
richox pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 81b292296 [AURON #2513] Optimize spill error handling and add an IOExceptiion for spill file missing (#2514)
81b292296 is described below
commit 81b292296007461fa28259704f44d996f2294c2e
Author: Yizhong Zhang <zy...@gmail.com>
AuthorDate: Fri Sep 11 11:20:52 2026 +0800
[AURON #2513] Optimize spill error handling and add an IOExceptiion for spill file missing (#2514)
# Which issue does this PR close?
Closes #2513
# Rationale for this change
Memory accounting leaks and misleading exception
# What changes are included in this PR?
Add an IOException and count memory correctly when spilling
# Are there any user-facing changes?
No.
# How was this patch tested?
UTs.
# Was this patch authored or co-authored using generative AI tooling?
- [ ] Yes
- [x] No
Co-authored-by: zhangyizhong <zh...@kuaishou.com>
---
.../spark/sql/auron/memory/OnHeapSpill.scala | 21 ++++++-
.../apache/spark/sql/auron/memory/SpillBuf.scala | 64 +++++++++++++++++++---
2 files changed, 77 insertions(+), 8 deletions(-)
diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/OnHeapSpill.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/OnHeapSpill.scala
index 23f3e6c42..9ed6431dd 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/OnHeapSpill.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/OnHeapSpill.scala
@@ -113,9 +113,28 @@ case class OnHeapSpill(hsm: SparkOnHeapSpillManager, id: Int) extends Logging {
spillBuf match {
case memBasedBuf: MemBasedSpillBuf =>
val releasingMemory = memUsed
- spillBuf = memBasedBuf.spill(hsm)
+ val fileBasedBuf =
+ try {
+ memBasedBuf.spill(hsm)
+ } catch {
+ case e: SpillIOException =>
+ // buffers were already released by spill(); give the memory back to
+ // the manager so the accounting does not leak, then rethrow so the
+ // caller sees the real IO cause.
+ spillBuf = new ReleasedSpillBuf(memBasedBuf)
+ hsm.freeMemory(releasingMemory)
+ logError(s"failed to spill in-mem buffer to disk, id=$id", e)
+ throw e
+ }
+ spillBuf = fileBasedBuf
hsm.freeMemory(releasingMemory)
releasingMemory
+
+ case _: FileBasedSpillBuf =>
+ 0L
+
+ case _: ReleasedSpillBuf =>
+ 0L
}
}
}
diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/SpillBuf.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/SpillBuf.scala
index ba2a57492..6b4f0686e 100644
--- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/SpillBuf.scala
+++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/memory/SpillBuf.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.auron.memory
-import java.io.{File, RandomAccessFile}
+import java.io.{File, IOException, RandomAccessFile}
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
import java.util
@@ -26,6 +26,12 @@ import io.netty.buffer.Unpooled
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
+/**
+ * Thrown when spilling fails because of an underlying IO problem (for example the spark local
+ * directory tree being removed by an external process) rather than because of memory exhaustion.
+ */
+class SpillIOException(message: String, cause: Throwable) extends IOException(message, cause)
+
abstract class SpillBuf {
def write(buf: ByteBuffer): Unit
def read(buf: ByteBuffer): Unit
@@ -85,15 +91,59 @@ class MemBasedSpillBuf extends SpillBuf with Logging {
logWarning(s"spilling in-mem spill buffer to disk, size=${Utils.bytesToString(size)}")
val startTimeNs = System.nanoTime()
- val file = hsm.blockManager.diskBlockManager.createTempLocalBlock()._2
- val channel = new RandomAccessFile(file, "rw").getChannel
+ val file =
+ try {
+ hsm.blockManager.diskBlockManager.createTempLocalBlock()._2
+ } catch {
+ case e: IOException =>
+ throw new SpillIOException(
+ s"failed to create spill file under spark local dirs, the local directory tree" +
+ s" may have been removed by an external process: ${e.getMessage}",
+ e)
+ }
- while (!bufs.isEmpty) {
- val buf = bufs.removeFirst().nioBuffer()
- while (buf.remaining() > 0) {
- channel.write(buf)
+ val channel =
+ try {
+ new RandomAccessFile(file, "rw").getChannel
+ } catch {
+ case e: IOException =>
+ throw new SpillIOException(
+ s"failed to open spill file ${file.getAbsolutePath}, it may have been removed" +
+ s" by an external process: ${e.getMessage}",
+ e)
}
+
+ // Keep each buffer in the deque until it has been fully written, so that a
+ // partial failure does not silently discard buffered data. On failure the
+ // remaining buffers are released and the accounted memory is handed back to
+ // the caller-visible state via `release()`.
+ try {
+ while (!bufs.isEmpty) {
+ val head = bufs.peekFirst()
+ val buf = head.nioBuffer()
+ while (buf.remaining() > 0) {
+ channel.write(buf)
+ }
+ bufs.removeFirst()
+ mem -= head.capacity()
+ }
+ } catch {
+ case e: IOException =>
+ // the spill file is unusable, drop it and free everything we still hold
+ try {
+ channel.close()
+ } catch {
+ case _: IOException => // ignore, we are already failing
+ }
+ if (file.exists() && !file.delete()) {
+ logWarning(s"Was unable to delete partial spill file: ${file.getAbsolutePath}")
+ }
+ release()
+ throw new SpillIOException(
+ s"failed to write spill file ${file.getAbsolutePath}: ${e.getMessage}",
+ e)
}
+
val endTimeNs = System.nanoTime
new FileBasedSpillBuf(numWrittenBytes, file, channel, endTimeNs - startTimeNs)
}