You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/10/15 13:05:37 UTC

(spark) branch branch-4.3 updated: [SPARK-59832][4.X][SQL] Fail on truncated input in the vectorized RLE reader

This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.3 by this push:
     new 95fd6623a67e [SPARK-59832][4.X][SQL] Fail on truncated input in the vectorized RLE reader
95fd6623a67e is described below

commit 95fd6623a67e4f1b558b4b5fab8b958cf909343f
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Thu Oct 15 14:05:01 2026 +0100

    [SPARK-59832][4.X][SQL] Fail on truncated input in the vectorized RLE reader
    
    ### What changes were proposed in this pull request?
    
    Backport of #59117 to branch-4.x.
    
    The cherry-pick conflicted only on code that SPARK-59620 added to master, which is not on this branch: an import in `VectorizedColumnReader` and `VectorizedRleValuesReaderSuite`, and a doc comment next to the new `readBatchRepeated` bridge in `ParquetTestAccess`. They are left out; otherwise the change is the same as on master.
    
    This commit also applies cleanly to branch-4.3, so it can be cherry-picked there when merging. The same change was built and tested on branch-4.3.
    
    ### Why are the changes needed?
    
    Same as #59117. On a corrupt page, the vectorized RLE reader, which decodes definition/repetition levels, dictionary ids and RLE booleans, silently returned wrong values or looped forever when the encoded data ended early, and a corrupt bit-packed header could overflow `currentCount` or allocate a huge buffer. The reader now fails with a `ParquetDecodingException`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Only for corrupt or non-compliant Parquet files, as described in #59117: such pages now fail with a `ParquetDecodingException` instead of returning wrong values or hanging, and with `spark.sql.files.ignoreCorruptFiles=true` the rest of such a file is skipped. A few of these pages could be read before and gave the right result by chance; #59117 lists them.
    
    ### How was this patch tested?
    
    The 18 tests added in #59117 pass on this branch, as do the existing tests in `ParquetEncodingSuite`, `ParquetVectorizedSuite`, `ParquetColumnIndexSuite`, `ParquetIOSuite`, `ParquetV1QuerySuite`, `ParquetV2QuerySuite`, `ParquetDeltaByteArrayEncodingSuite`, `ParquetDeltaLengthByteArrayEncodingSuite`, `VectorizedPlainValuesReaderSuite` and `ParquetVectorUpdaterSuite`.
    
    ```
    build/sbt 'sql/testOnly *VectorizedRleValuesReaderSuite *ParquetEncodingSuite *ParquetVectorizedSuite *ParquetIOSuite'
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Code (Claude Opus 5.5)
    
    Closes #59481 from viirya/SPARK-59832-branch-4.x.
    
    Authored-by: Liang-Chi Hsieh <vi...@gmail.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
    (cherry picked from commit 45f1cc8f1efe165eecf45f80302f8a4c0b74ee11)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../parquet/VectorizedColumnReader.java            |  22 +-
 .../parquet/VectorizedRleValuesReader.java         |  80 ++++--
 .../datasources/parquet/ParquetIOSuite.scala       |  89 ++++++
 .../datasources/parquet/ParquetTestAccess.scala    |  24 ++
 .../parquet/ParquetVectorizedSuite.scala           |  27 ++
 .../parquet/VectorizedRleValuesReaderSuite.scala   | 317 ++++++++++++++++++++-
 6 files changed, 530 insertions(+), 29 deletions(-)

diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index eff701082589..edb4638eaf22 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -32,6 +32,7 @@ import org.apache.parquet.column.Encoding;
 import org.apache.parquet.column.page.*;
 import org.apache.parquet.column.values.RequiresPreviousReader;
 import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.ParquetDecodingException;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation;
 import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
@@ -132,6 +133,9 @@ public class VectorizedColumnReader {
         this.isCurrentPageDictionaryEncoded = true;
       } catch (IOException e) {
         throw new IOException("could not decode the dictionary for " + descriptor, e);
+      } catch (ParquetDecodingException e) {
+        throw new ParquetDecodingException(
+          "could not decode the dictionary for " + descriptor + ": " + e.getMessage(), e);
       }
     } else {
       this.dictionary = null;
@@ -343,9 +347,7 @@ public class VectorizedColumnReader {
     if (dataEncoding.usesDictionary()) {
       this.dataColumn = null;
       if (dictionary == null) {
-        throw new IOException(
-            "could not read page in col " + descriptor +
-                " as the dictionary was missing for encoding " + dataEncoding);
+        throw new IOException("the dictionary was missing for encoding " + dataEncoding);
       }
       @SuppressWarnings("deprecation")
       Encoding plainDict = Encoding.PLAIN_DICTIONARY; // var to allow warning suppression
@@ -360,11 +362,9 @@ public class VectorizedColumnReader {
       this.isCurrentPageDictionaryEncoded = false;
     }
 
-    try {
-      dataColumn.initFromPage(pageValueCount, in);
-    } catch (IOException e) {
-      throw new IOException("could not read page in col " + descriptor, e);
-    }
+    // readPageV1 and readPageV2 add the page and the column to an IOException or a
+    // ParquetDecodingException thrown here.
+    dataColumn.initFromPage(pageValueCount, in);
     // for PARQUET-246 (See VectorizedDeltaByteArrayReader.setPreviousValues)
     if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, dataEncoding) &&
         previousReader instanceof RequiresPreviousReader) {
@@ -434,6 +434,9 @@ public class VectorizedColumnReader {
       return pageValueCount;
     } catch (IOException e) {
       throw new IOException("could not read page " + page + " in col " + descriptor, e);
+    } catch (ParquetDecodingException e) {
+      throw new ParquetDecodingException(
+        "could not read page " + page + " in col " + descriptor + ": " + e.getMessage(), e);
     }
   }
 
@@ -454,6 +457,9 @@ public class VectorizedColumnReader {
       return pageValueCount;
     } catch (IOException e) {
       throw new IOException("could not read page " + page + " in col " + descriptor, e);
+    } catch (ParquetDecodingException e) {
+      throw new ParquetDecodingException(
+        "could not read page " + page + " in col " + descriptor + ": " + e.getMessage(), e);
     }
   }
 }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
index d1b2726ceeb0..8f240c13c164 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
@@ -62,6 +62,9 @@ public final class VectorizedRleValuesReader extends ValuesReader
   private int bytesWidth;
   private BytePacker packer;
 
+  // True if the dictionary id section of the page is empty, not even holding the bit width.
+  private boolean emptyIdSection;
+
   // Current decoding mode and values
   private MODE mode;
   private int currentCount;
@@ -96,16 +99,35 @@ public final class VectorizedRleValuesReader extends ValuesReader
   @Override
   public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException {
     this.in = in;
+    this.emptyIdSection = false;
     if (fixedWidth) {
       // Initialize for repetition and definition levels
       if (readLength) {
+        if (in.available() < 4) {
+          throw new ParquetDecodingException("Corrupted RLE data: the 4-byte length is cut off, " +
+            "only " + in.available() + " bytes are left in the page");
+        }
         int length = readIntLittleEndian();
+        if (length < 0 || length > in.available()) {
+          throw new ParquetDecodingException("Corrupted RLE data: invalid length " + length +
+            ", " + in.available() + " bytes are left in the page");
+        }
         this.in = in.sliceStream(length);
       }
     } else {
       // Initialize for values
       if (in.available() > 0) {
-        init(in.read());
+        int width = in.read();
+        if (width > 32) {
+          throw new ParquetDecodingException("Corrupted RLE data: invalid bit width " + width);
+        }
+        init(width);
+      } else {
+        // No encoded values (e.g. an all-null page). Do not treat this as a bit width of 0,
+        // which would decode every value as 0; any read fails in `readNextGroup` instead.
+        this.emptyIdSection = true;
+        this.currentCount = 0;
+        return;
       }
     }
     if (bitWidth == 0) {
@@ -208,7 +230,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
     int leftInPage = state.valuesToReadInPage;
 
     while (leftInBatch > 0 && leftInPage > 0) {
-      if (currentCount == 0 && !readNextGroup()) break;
+      if (currentCount == 0) readNextGroup();
       int n = Math.min(leftInBatch, Math.min(leftInPage, this.currentCount));
 
       long rangeStart = state.currentRangeStart();
@@ -318,7 +340,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
     int leftInPage = state.valuesToReadInPage;
 
     while (leftInBatch > 0 && leftInPage > 0) {
-      if (currentCount == 0 && !readNextGroup()) break;
+      if (currentCount == 0) readNextGroup();
       int n = Math.min(leftInBatch, Math.min(leftInPage, this.currentCount));
 
       long rangeStart = state.currentRangeStart();
@@ -431,7 +453,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
       values, nulls, valuesReused, valueReader, updater);
 
     while ((leftInBatch > 0 || !state.lastListCompleted) && leftInPage > 0) {
-      if (currentCount == 0 && !readNextGroup()) break;
+      if (currentCount == 0) readNextGroup();
 
       // Values to read in the current RLE/PACKED block, must be <= what's left in the page
       int valuesLeftInBlock = Math.min(leftInPage, currentCount);
@@ -637,7 +659,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
     int n = total;
     int initialValueOffset = state.valueOffset;
     while (n > 0) {
-      if (currentCount == 0 && !readNextGroup()) break;
+      if (currentCount == 0) readNextGroup();
       int num = Math.min(n, this.currentCount);
       readValuesN(num, state, defLevels, values, nulls, valueReader, updater);
       state.levelOffset += num;
@@ -731,7 +753,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
       VectorizedValuesReader valuesReader,
       ParquetVectorUpdater updater) {
     while (n > 0) {
-      if (currentCount == 0 && !readNextGroup()) break;
+      if (currentCount == 0) readNextGroup();
       int num = Math.min(n, this.currentCount);
       switch (mode) {
         case RLE -> {
@@ -764,7 +786,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
   public void readIntegers(int total, WritableColumnVector c, int rowId) {
     int left = total;
     while (left > 0) {
-      if (currentCount == 0 && !readNextGroup()) break;
+      if (currentCount == 0) readNextGroup();
       int n = Math.min(left, this.currentCount);
       switch (mode) {
         case RLE -> c.putInts(rowId, n, currentValue);
@@ -957,7 +979,6 @@ public final class VectorizedRleValuesReader extends ValuesReader
    */
   private int readIntLittleEndianPaddedOnBitWidth() throws IOException {
     return switch (bytesWidth) {
-      case 0 -> 0;
       case 1 -> in.read();
       case 2 -> {
         int ch2 = in.read();
@@ -976,14 +997,30 @@ public final class VectorizedRleValuesReader extends ValuesReader
   }
 
   /**
-   * Reads the next group. Returns false if no more group available.
+   * Reads the next non-empty group, so that `currentCount` is positive on return.
+   * <p>
+   * Callers never read more values than the page declares, so running out of encoded data here
+   * means the page is corrupted. With a bit width of 0, `initFromPage` already provides all the
+   * values of the page, so reaching here also means reading past the end.
    */
-  private boolean readNextGroup() {
-    if (in.available() <= 0) {
-      currentCount = 0;
-      return false;
+  private void readNextGroup() {
+    if (emptyIdSection) {
+      throw new ParquetDecodingException(
+        "Corrupted RLE data: the dictionary id section is empty, not even the bit width");
     }
+    do {
+      if (bitWidth == 0 || in.available() <= 0) {
+        throw new ParquetDecodingException(
+          "Corrupted RLE data: reading past the end of the encoded values");
+      }
+      readGroup();
+    } while (this.currentCount == 0);
+  }
 
+  /**
+   * Reads the next group, which may be empty.
+   */
+  private void readGroup() {
     try {
       int header = readUnsignedVarInt();
       this.mode = (header & 1) == 0 ? MODE.RLE : MODE.PACKED;
@@ -994,6 +1031,17 @@ public final class VectorizedRleValuesReader extends ValuesReader
         }
         case PACKED -> {
           int numGroups = header >>> 1;
+          // Validate before allocating the buffer, so a corrupted header can neither overflow
+          // `currentCount` nor make us allocate more than the remaining bytes can fill.
+          if (numGroups > Integer.MAX_VALUE / 8) {
+            throw new ParquetDecodingException(
+              "Corrupted RLE data: bit-packed run of " + numGroups + " groups is too long");
+          }
+          if ((long) numGroups * bitWidth > in.available()) {
+            throw new ParquetDecodingException("Corrupted RLE data: bit-packed run of " +
+              numGroups + " groups needs " + (long) numGroups * bitWidth + " bytes, but only " +
+              in.available() + " bytes are left");
+          }
           this.currentCount = numGroups * 8;
 
           if (this.currentBuffer.length < this.currentCount) {
@@ -1014,10 +1062,8 @@ public final class VectorizedRleValuesReader extends ValuesReader
         }
       }
     } catch (IOException e) {
-      throw new ParquetDecodingException("Failed to read from input stream", e);
+      throw new ParquetDecodingException("Corrupted RLE data: failed to read from input stream", e);
     }
-
-    return true;
   }
 
   /**
@@ -1026,7 +1072,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
   private void skipValues(int n) {
     int left = n;
     while (left > 0) {
-      if (this.currentCount == 0 && !readNextGroup()) break;
+      if (this.currentCount == 0) readNextGroup();
       int num = Math.min(left, this.currentCount);
       switch (mode) {
         case RLE -> {}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index b9ac9df8e31d..3a938a985781 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.datasources.parquet
 
 import java.math.{BigDecimal => JBigDecimal}
+import java.nio.file.Files
 import java.time.{LocalDateTime, LocalTime}
 import java.util.Locale
 
@@ -27,16 +28,21 @@ import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.TypeTag
 
 import com.google.common.primitives.UnsignedLong
+import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext}
+import org.apache.parquet.bytes.BytesUtils
 import org.apache.parquet.column.{Encoding, ParquetProperties}
 import org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0
 import org.apache.parquet.example.data.Group
 import org.apache.parquet.example.data.simple.{SimpleGroup, SimpleGroupFactory}
 import org.apache.parquet.hadoop._
 import org.apache.parquet.hadoop.example.ExampleParquetWriter
+import org.apache.parquet.io.ParquetDecodingException
 import org.apache.parquet.io.api.Binary
 import org.apache.parquet.schema.{MessageType, MessageTypeParser}
+import org.scalatest.concurrent.{Signaler, ThreadSignaler}
+import org.scalatest.time.{Seconds, Span}
 
 import org.apache.spark.{SPARK_VERSION_SHORT, SparkException, TestUtils}
 import org.apache.spark.rdd.RDD
@@ -2410,6 +2416,89 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
       }
     }
   }
+
+  /**
+   * Writes 100 rows of `column` to a single uncompressed data page, lets `corrupt` modify the
+   * RLE section that starts with `03 00 00 00 C8 01 01` (the 4-byte length 3, then one RLE run
+   * with header 100 << 1 and value 1), and returns the error of reading the file with the
+   * vectorized reader. For a nullable INT32 column that is the definition levels; for a
+   * BOOLEAN column written as a V2 page it is the RLE-encoded values.
+   */
+  private def readCorruptedRleSection(
+      column: String,
+      writerVersion: String = "PARQUET_1_0")(
+      corrupt: (Array[Byte], Int) => Unit): Throwable = {
+    var error: Throwable = null
+    withTempPath { dir =>
+      val path = dir.getCanonicalPath
+      spark.range(100).selectExpr(column).coalesce(1)
+        .write
+        .option(ParquetOutputFormat.ENABLE_DICTIONARY, "false")
+        .option(ParquetOutputFormat.WRITER_VERSION, writerVersion)
+        .option("compression", "uncompressed")
+        .parquet(path)
+      val file = dir.listFiles().filter(_.getName.endsWith(".parquet")).head
+      val bytes = Files.readAllBytes(file.toPath)
+      val section = Array[Byte](3, 0, 0, 0, 0xC8.toByte, 0x01, 1)
+      val offset = bytes.indexOfSlice(section)
+      assert(offset >= 0 && bytes.indexOfSlice(section, offset + 1) < 0)
+      corrupt(bytes, offset)
+      Files.write(file.toPath, bytes)
+      dir.listFiles().filter(_.getName.endsWith(".crc")).foreach(_.delete())
+
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
+        // Before SPARK-59832 a truncated read never returned, so run it with a timeout. The test
+        // thread waits interruptibly for the job, so it can be signaled directly.
+        implicit val signaler: Signaler = ThreadSignaler
+        error = intercept[SparkException] {
+          failAfter(Span(60, Seconds))(spark.read.parquet(path).collect())
+        }
+      }
+    }
+    error
+  }
+
+  private def causes(e: Throwable): Seq[Throwable] =
+    ExceptionUtils.getThrowableList(e).asScala.toSeq
+
+  /** Sets the 4-byte length at `offset` to Int.MaxValue, far more than the rest of the page. */
+  private def setMaxLength(bytes: Array[Byte], offset: Int): Unit =
+    BytesUtils.intToBytes(Int.MaxValue).copyToArray(bytes, offset)
+
+  /**
+   * Checks that the page error is reported once with the column, directly on top of the
+   * `ParquetDecodingException` about the invalid length.
+   */
+  private def assertInvalidLengthReportsColumn(e: Throwable, column: String): Unit = {
+    val pageErrors =
+      causes(e).filter(c => Option(c.getMessage).exists(_.startsWith("could not read page")))
+    assert(pageErrors.size == 1, e)
+    assert(pageErrors.head.getMessage.contains(s"in col [$column]"), e)
+    assert(pageErrors.head.getCause.isInstanceOf[ParquetDecodingException], e)
+    assert(pageErrors.head.getCause.getMessage.contains(
+      s"Corrupted RLE data: invalid length ${Int.MaxValue}"), e)
+  }
+
+  test("SPARK-59832: vectorized reader fails on truncated definition levels") {
+    // Shorten the run to 64 values (header 64 << 1 = varint 0x80 0x01).
+    val e = readCorruptedRleSection("if(id < 0, null, cast(id as int)) as a") {
+      (bytes, offset) => bytes(offset + 4) = 0x80.toByte
+    }
+    assert(causes(e).exists {
+      case c: ParquetDecodingException => c.getMessage.contains("Corrupted RLE data")
+      case _ => false
+    }, e)
+  }
+
+  test("SPARK-59832: invalid definition level length reports the column") {
+    val e = readCorruptedRleSection("if(id < 0, null, cast(id as int)) as a")(setMaxLength)
+    assertInvalidLengthReportsColumn(e, "a")
+  }
+
+  test("SPARK-59832: invalid length of RLE booleans in a V2 page reports the column") {
+    val e = readCorruptedRleSection("id >= 0 as b", writerVersion = "PARQUET_2_0")(setMaxLength)
+    assertInvalidLengthReportsColumn(e, "b")
+  }
 }
 
 class JobCommitFailureParquetOutputCommitter(outputPath: Path, context: TaskAttemptContext)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
index e008a7b0322e..5391d31a14ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
@@ -37,6 +37,7 @@ import org.apache.spark.util.SparkClassUtils
  * Currently bridges:
  *   - `ParquetReadState` (constructor + `resetForNewBatch` + `resetForNewPage`)
  *   - `VectorizedRleValuesReader.readBatch` (5-arg overload not exposed publicly)
+ *   - `VectorizedRleValuesReader.readBatchRepeated` (takes the package-private state)
  *   - `ParquetVectorUpdaterFactory` (constructor)
  *   - `VectorizedDeltaByteArrayReader` (no-arg constructor)
  *   - `VectorizedDeltaLengthByteArrayReader` (no-arg constructor)
@@ -79,6 +80,15 @@ object ParquetTestAccess {
       .getOrElse(throw new NoSuchMethodException(
         "VectorizedRleValuesReader.readBatch/5"))
 
+  private val readBatchRepeatedMethod: Method =
+    classOf[VectorizedRleValuesReader].getMethods
+      .find(m =>
+        m.getName == "readBatchRepeated"
+          && m.getParameterCount == 7
+          && m.getParameterTypes()(0) == stateCls)
+      .getOrElse(throw new NoSuchMethodException(
+        "VectorizedRleValuesReader.readBatchRepeated/7"))
+
   def newState(
       descriptor: ColumnDescriptor,
       isRequired: Boolean,
@@ -118,6 +128,20 @@ object ParquetTestAccess {
         reader, state, values, defLevels, valueReader, updater)
     } catch { case e: ReflectiveOperationException => throw rethrow(e) }
 
+  def readBatchRepeated(
+      reader: VectorizedRleValuesReader,
+      state: AnyRef,
+      repLevels: WritableColumnVector,
+      defLevelsReader: VectorizedRleValuesReader,
+      defLevels: WritableColumnVector,
+      values: WritableColumnVector,
+      valueReader: VectorizedValuesReader,
+      updater: ParquetVectorUpdater): Unit =
+    try {
+      readBatchRepeatedMethod.invoke(
+        reader, state, repLevels, defLevelsReader, defLevels, values, valueReader, updater)
+    } catch { case e: ReflectiveOperationException => throw rethrow(e) }
+
   // -------- ParquetVectorUpdaterFactory --------
 
   private val factoryCtor = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
index 916884580068..c1f45b54767a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
@@ -524,6 +524,33 @@ class ParquetVectorizedSuite extends ParquetTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-59832: a corrupted dictionary page reports the column") {
+    // A PLAIN INT32 dictionary page that declares 3 entries but holds only 2.
+    val parquetSchema = MessageTypeParser.parseMessageType("message root { optional int32 a; }")
+    val ty = parquetSchema.asGroupType().getType("a").asPrimitiveType()
+    val cd = new ColumnDescriptor(Array("a"), ty, 0, 1)
+    val memPageStore = new MemPageStore(1)
+    val pageWriter = memPageStore.getPageWriter(cd)
+    pageWriter.writeDictionaryPage(new DictionaryPage(
+      BytesInput.from(Array[Byte](1, 0, 0, 0, 2, 0, 0, 0)), 3, Encoding.PLAIN_DICTIONARY))
+    // One non-null value: the definition levels (length 2, an RLE run of 1 x 1), then the
+    // dictionary ids (bit width 2, an RLE run of 1 x 0).
+    pageWriter.writePage(BytesInput.from(Array[Byte](2, 0, 0, 0, 2, 1, 2, 2, 0)), 1, 1,
+      Statistics.createStats(ty), Encoding.RLE, Encoding.RLE, Encoding.RLE_DICTIONARY)
+    val recordReader = new VectorizedParquetRecordReader(
+      DateTimeUtils.getZoneId("EST"), "CORRECTED", "UTC", "CORRECTED", "UTC", true, 1)
+    try {
+      recordReader.initialize(parquetSchema, parquetSchema,
+        TestParquetRowGroupReader(Seq(TestPageReadStore(memPageStore, Seq(0L)))), 1)
+      val e = intercept[ParquetDecodingException](recordReader.nextKeyValue())
+      assert(e.getMessage.contains("could not decode the dictionary for [a]"), e.getMessage)
+      assert(e.getMessage.contains(e.getCause.getMessage), e.getMessage)
+      assert(e.getCause.isInstanceOf[ParquetDecodingException], e)
+    } finally {
+      recordReader.close()
+    }
+  }
+
   truncateTypeTest("primitive type", IntegerType, LongType, IntegerType)
 
   truncateTypeTest("basic struct",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala
index 5b3997160ece..6e37351873b4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala
@@ -19,25 +19,92 @@ package org.apache.spark.sql.execution.datasources.parquet
 
 import java.nio.ByteBuffer
 import java.util.PrimitiveIterator
+import java.util.concurrent.{Callable, ExecutionException, TimeoutException, TimeUnit}
 
 import scala.jdk.CollectionConverters._
 
-import org.apache.parquet.bytes.ByteBufferInputStream
+import org.apache.parquet.bytes.{ByteBufferInputStream, BytesInput, BytesUtils}
+import org.apache.parquet.column.ColumnDescriptor
+import org.apache.parquet.io.ParquetDecodingException
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
+import org.apache.parquet.schema.Type.Repetition
+import org.apache.parquet.schema.Types
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.execution.datasources.parquet.VectorizedRleValuesReaderTestUtils._
 import org.apache.spark.sql.execution.vectorized.{OnHeapColumnVector, WritableColumnVector}
-import org.apache.spark.sql.types.IntegerType
+import org.apache.spark.sql.types.{BooleanType, IntegerType}
+import org.apache.spark.util.ThreadUtils
 
 /**
  * Focused correctness tests for `VectorizedRleValuesReader.readBatch` PACKED-mode decoding,
  * covering patterns the P0 optimization cares about: run boundaries, batch boundaries, and
  * nested def-level grouping. Uses the same reflection bridge as the benchmark.
+ *
+ * The SPARK-59832 tests check that corrupted or truncated input fails with a
+ * `ParquetDecodingException` instead of returning wrong values, looping forever or allocating
+ * a huge buffer, and that reads ending exactly at the end of the encoded values still work.
  */
 class VectorizedRleValuesReaderSuite extends SparkFunSuite {
 
   import VectorizedRleValuesReaderSuite._
 
+  // Runs the reads that may never return before SPARK-59832, a thread per read, so that a read
+  // stuck in a loop does not block the following ones. The decode loops do not check for
+  // interruption, so a stuck read keeps its thread busy until the JVM exits. After the first
+  // timeout, later reads fail without running, so that a regression leaves one busy thread.
+  private val executor = ThreadUtils.newDaemonCachedThreadPool("rle-reader-suite")
+  @volatile private var timedOut = false
+
+  override def afterAll(): Unit = {
+    try {
+      executor.shutdownNow()
+      executor.awaitTermination(10, TimeUnit.SECONDS)
+    } finally {
+      super.afterAll()
+    }
+  }
+
+  /**
+   * Checks that `f` fails with a `ParquetDecodingException` whose message contains
+   * "Corrupted RLE data: " followed by `expected`.
+   */
+  private def assertCorrupted(expected: String)(f: => Any): Unit = {
+    val e = intercept[ParquetDecodingException](f)
+    assert(e.getMessage.contains("Corrupted RLE data: " + expected), e.getMessage)
+  }
+
+  /**
+   * Like `assertCorrupted`, but runs `f` on another thread with a timeout, so that a read that
+   * never returns fails the test.
+   */
+  private def interceptCorrupted(expected: String)(f: => Any): Unit = {
+    assert(!timedOut, "Not run, since an earlier read did not return")
+    val future = executor.submit(new Callable[Any] {
+      override def call(): Any = f
+    })
+    assertCorrupted(expected) {
+      try {
+        future.get(10, TimeUnit.SECONDS)
+      } catch {
+        case e: ExecutionException => throw e.getCause
+        case e: TimeoutException =>
+          timedOut = true
+          throw e
+      }
+    }
+  }
+
+  /** Checks that reading `n` dictionary ids from `page` fails, on every stream of `page`. */
+  private def assertReadIntegersFails(page: Array[Byte], n: Int, expected: String): Unit = {
+    streams(page).foreach { in =>
+      val reader = new VectorizedRleValuesReader()
+      reader.initFromPage(n, in())
+      val c = new OnHeapColumnVector(n, IntegerType)
+      interceptCorrupted(expected)(reader.readIntegers(n, c, 0))
+    }
+  }
+
   test("PACKED: alternating null/non-null (many single-element runs)") {
     val n = 1024
     val defLevels = Array.tabulate(n)(i => i & 1)
@@ -96,7 +163,7 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
 
   test("PACKED group larger than initial 16-int currentBuffer (triggers buffer grow)") {
     // ~1024 alternating values produce one PACKED block well beyond the initial 16-int buffer,
-    // exercising `new int[currentCount]` in readNextGroup.
+    // exercising `new int[currentCount]` in readGroup.
     val defLevels = Array.tabulate(1024)(i => i & 1)
     runAndAssert(defLevels, maxDef = 1, batchSize = 1024, withDefLevels = false)
   }
@@ -134,7 +201,7 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
   test("PACKED: multi-buffer stream with packed run crossing buffer boundary") {
     // Exercises the MultiBufferInputStream.slice() path where the packed bytes span a buffer
     // boundary, causing slice() to return a freshly-allocated contiguous buffer with
-    // position() == 0 (the base-0 pos branch in readNextGroup).
+    // position() == 0 (the base-0 pos branch in readGroup).
     val n = 256
     val defLevels = Array.tabulate(n)(i => i & 1)
     val bitWidth = 1
@@ -185,10 +252,252 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
       produced += toRead
     }
   }
+
+  test("SPARK-59832: truncated dictionary ids fail instead of being partially read") {
+    // A dictionary id page with 10 ids, read as if it had 20.
+    val page = dictIdPage(Array.fill(10)(3), bitWidth = 4)
+    assertReadIntegersFails(page, 20, PastEnd)
+    streams(page).foreach { in =>
+      val reader = new VectorizedRleValuesReader()
+      reader.initFromPage(20, in())
+      interceptCorrupted(PastEnd)(reader.skipIntegers(20))
+    }
+    streams(page).foreach { in =>
+      val reader = new VectorizedRleValuesReader()
+      reader.initFromPage(20, in())
+      (0 until 10).foreach(_ => assert(reader.readInteger() == 3))
+      interceptCorrupted(PastEnd)(reader.readInteger())
+    }
+  }
+
+  test("SPARK-59832: truncated booleans fail instead of looping forever") {
+    val encoded = encodeRle(Array.fill(10)(1), 1)
+    streams(BytesUtils.intToBytes(encoded.length) ++ encoded).foreach { in =>
+      val reader = new VectorizedRleValuesReader(1)
+      reader.initFromPage(20, in())
+      val c = new OnHeapColumnVector(20, BooleanType)
+      interceptCorrupted(PastEnd)(reader.readBooleans(20, c, 0))
+    }
+  }
+
+  test("SPARK-59832: truncated definition levels fail in readBatch") {
+    // The page declares 100 values but its definition levels only encode 64. Before the fix
+    // readBatch returned without progress, so VectorizedColumnReader.readBatch spun forever.
+    val n = 100
+    val encoded = encodeRle(Array.fill(64)(1), 1)
+    val plainBytes = plainIntBytes(n)(valueAt)
+    def run(withDefLevels: Boolean, rowIndexes: PrimitiveIterator.OfLong): Unit = {
+      val reader = new VectorizedRleValuesReader(1, false)
+      reader.initFromPage(n, ByteBufferInputStream.wrap(ByteBuffer.wrap(encoded)))
+      val valueReader = new VectorizedPlainValuesReader
+      valueReader.initFromPage(n, ByteBufferInputStream.wrap(ByteBuffer.wrap(plainBytes)))
+      val state = ParquetTestAccess.newState(intColumnDescriptor(1), false, rowIndexes)
+      ParquetTestAccess.resetForNewPage(state, n, 0L)
+      ParquetTestAccess.resetForNewBatch(state, n)
+      val values = new OnHeapColumnVector(n, IntegerType)
+      val defLevels = if (withDefLevels) new OnHeapColumnVector(n, IntegerType) else null
+      interceptCorrupted(PastEnd) {
+        ParquetTestAccess.readBatch(reader, state, values, defLevels, valueReader, integerUpdater)
+      }
+    }
+    run(withDefLevels = false, rowIndexes = null)
+    run(withDefLevels = true, rowIndexes = null)
+    // Skips the truncated levels to reach rows 80 to 90.
+    run(withDefLevels = false, rowIndexes = longIterator((80 to 90).toArray))
+  }
+
+  test("SPARK-59832: truncated repetition levels fail in readBatchRepeated") {
+    // Every row is a top-level row (repetition level 0), but only 64 of 100 are encoded.
+    readRepeated(repLevels = Array.fill(64)(0), defLevels = Array.fill(100)(1), n = 100)
+  }
+
+  test("SPARK-59832: truncated definition levels fail in readBatchRepeated") {
+    // The repetition levels are complete, but only 64 of 100 definition levels are encoded.
+    // Before the fix the missing definition levels were silently left as 0.
+    readRepeated(repLevels = Array.fill(100)(0), defLevels = Array.fill(64)(1), n = 100)
+    // Skips the truncated levels to reach rows 80 to 90.
+    readRepeated(repLevels = Array.fill(100)(0), defLevels = Array.fill(64)(1), n = 100,
+      rowIndexes = longIterator((80 to 90).toArray))
+  }
+
+  private def readRepeated(
+      repLevels: Array[Int],
+      defLevels: Array[Int],
+      n: Int,
+      rowIndexes: PrimitiveIterator.OfLong = null): Unit = {
+    val prim = Types.primitive(PrimitiveTypeName.INT32, Repetition.REPEATED).named("col")
+    val descriptor = new ColumnDescriptor(Array("col"), prim, 1, 1)
+    val repReader = new VectorizedRleValuesReader(1, false)
+    repReader.initFromPage(
+      n, ByteBufferInputStream.wrap(ByteBuffer.wrap(encodeRle(repLevels, 1))))
+    val defReader = new VectorizedRleValuesReader(1, false)
+    defReader.initFromPage(
+      n, ByteBufferInputStream.wrap(ByteBuffer.wrap(encodeRle(defLevels, 1))))
+    val valueReader = new VectorizedPlainValuesReader
+    valueReader.initFromPage(
+      n, ByteBufferInputStream.wrap(ByteBuffer.wrap(plainIntBytes(n)(valueAt))))
+    val state = ParquetTestAccess.newState(descriptor, false, rowIndexes)
+    ParquetTestAccess.resetForNewPage(state, n, 0L)
+    ParquetTestAccess.resetForNewBatch(state, n)
+    val repLevelsVec = new OnHeapColumnVector(n, IntegerType)
+    val defLevelsVec = new OnHeapColumnVector(n, IntegerType)
+    val values = new OnHeapColumnVector(n, IntegerType)
+    interceptCorrupted(PastEnd) {
+      ParquetTestAccess.readBatchRepeated(repReader, state, repLevelsVec, defReader,
+        defLevelsVec, values, valueReader, integerUpdater)
+    }
+  }
+
+  test("SPARK-59832: reading past a page with bit width 0 fails") {
+    // With bit width 0 the page is one implicit run of `valueCount` zeros; the bytes that
+    // follow belong to the next section of the page and must not be decoded as levels.
+    val trailing = Array[Byte](3, 1, 2, 3)
+    val reader = new VectorizedRleValuesReader(0)
+    reader.initFromPage(5, ByteBufferInputStream.wrap(ByteBuffer.wrap(trailing)))
+    val c = new OnHeapColumnVector(6, IntegerType)
+    c.putInts(0, 6, -1)
+    reader.readIntegers(5, c, 0)
+    assert((0 until 5).forall(c.getInt(_) == 0))
+    interceptCorrupted(PastEnd)(reader.readIntegers(1, c, 5))
+  }
+
+  test("SPARK-59832: invalid level length is rejected") {
+    val payload = Array[Byte](10, 20, 30, 40)
+    Seq(-1, -4, -5, payload.length + 1, Int.MaxValue).foreach { length =>
+      streams(BytesUtils.intToBytes(length) ++ payload).foreach { in =>
+        val reader = new VectorizedRleValuesReader(1)
+        assertCorrupted(s"invalid length $length")(reader.initFromPage(8, in()))
+      }
+    }
+  }
+
+  test("SPARK-59832: invalid bit-packed run is rejected before allocating its buffer") {
+    def check(bitWidth: Int, numGroups: Long, data: Array[Byte])(expected: String): Unit =
+      assertReadIntegersFails(
+        Array(bitWidth.toByte) ++ varint((numGroups << 1) | 1) ++ data, 10, expected)
+    // Truncated last group. This was already rejected before (by `in.slice`), even when the
+    // bytes cover every value read. parquet-java and Arrow accept it, for compatibility with
+    // writers that do not pad the last group.
+    check(bitWidth = 4, numGroups = 2, data = Array.fill[Byte](5)(0x11))(
+      "bit-packed run of 2 groups needs 8 bytes, but only 5 bytes are left")
+    // numGroups * 8 and numGroups * bitWidth overflow to 0.
+    check(bitWidth = 8, numGroups = 1L << 29, data = encodeRle(Array.fill(10)(3), 8))(
+      s"bit-packed run of ${1 << 29} groups is too long")
+    // numGroups * 8 and numGroups * bitWidth overflow to negative values.
+    check(bitWidth = 2, numGroups = Int.MaxValue, data = Array.fill[Byte](16)(0))(
+      s"bit-packed run of ${Int.MaxValue} groups is too long")
+    // Would allocate an int[2^30] buffer before noticing that only 3 bytes are left.
+    check(bitWidth = 1, numGroups = 1L << 27, data = Array[Byte](1, 2, 3))(
+      s"bit-packed run of ${1 << 27} groups needs ${1 << 27} bytes, but only 3 bytes are left")
+  }
+
+  test("SPARK-59832: data ending inside a run header or an RLE value fails") {
+    // The continuation byte of the run header is missing, and an RLE run of 10 without its
+    // value.
+    Seq(Array[Byte](4, 0x80.toByte), Array[Byte](4, 0x14)).foreach { page =>
+      assertReadIntegersFails(page, 10, "failed to read from input stream")
+    }
+  }
+
+  test("SPARK-59832: cut-off level length is rejected") {
+    streams(Array[Byte](3, 0)).foreach { in =>
+      val reader = new VectorizedRleValuesReader(1)
+      assertCorrupted("the 4-byte length is cut off, only 2 bytes are left in the page") {
+        reader.initFromPage(8, in())
+      }
+    }
+  }
+
+  test("SPARK-59832: invalid dictionary id bit width is rejected") {
+    Seq(33, 255).foreach { bitWidth =>
+      streams(Array[Byte](bitWidth.toByte, 0x14, 0)).foreach { in =>
+        val reader = new VectorizedRleValuesReader()
+        assertCorrupted(s"invalid bit width $bitWidth")(reader.initFromPage(10, in()))
+      }
+    }
+  }
+
+  test("SPARK-59832: dictionary ids of an empty page are not decoded as 0") {
+    val reader = new VectorizedRleValuesReader()
+    reader.initFromPage(5, ByteBufferInputStream.wrap(ByteBuffer.wrap(Array.emptyByteArray)))
+    // An all-null page reads no ids.
+    val c = new OnHeapColumnVector(5, IntegerType)
+    reader.readIntegers(0, c, 0)
+    reader.skipIntegers(0)
+    interceptCorrupted("the dictionary id section is empty, not even the bit width") {
+      reader.readIntegers(5, c, 0)
+    }
+  }
+
+  test("SPARK-59832: runs of length 0 are skipped") {
+    // [bit width 4][RLE run of 0 x 7][bit-packed run of 0 groups][RLE run of 5 x 3]
+    val page = Array[Byte](4, 0, 7, 1, 10, 3)
+    streams(page).foreach { in =>
+      val reader = new VectorizedRleValuesReader()
+      reader.initFromPage(5, in())
+      assert((0 until 5).map(_ => reader.readInteger()) == Seq.fill(5)(3))
+    }
+    streams(page).foreach { in =>
+      val reader = new VectorizedRleValuesReader()
+      reader.initFromPage(5, in())
+      val c = new OnHeapColumnVector(5, IntegerType)
+      reader.readIntegers(5, c, 0)
+      assert((0 until 5).map(c.getInt) == Seq.fill(5)(3))
+    }
+  }
+
+  test("SPARK-59832: reads that end exactly at the end of the encoded values succeed") {
+    // A run of 20 identical ids (RLE) followed by 17 mixed ids (bit-packed, padded to 24).
+    val ids = Array.tabulate(37)(i => if (i < 20) 5 else i % 7)
+    streams(dictIdPage(ids, bitWidth = 3)).foreach { in =>
+      val reader = new VectorizedRleValuesReader()
+      reader.initFromPage(ids.length, in())
+      val c = new OnHeapColumnVector(ids.length, IntegerType)
+      reader.readIntegers(ids.length, c, 0)
+      assert((0 until ids.length).map(c.getInt) == ids.toSeq)
+    }
+    streams(dictIdPage(ids, bitWidth = 3)).foreach { in =>
+      val reader = new VectorizedRleValuesReader()
+      reader.initFromPage(ids.length, in())
+      reader.skipIntegers(30)
+      assert((30 until ids.length).map(_ => reader.readInteger()) == ids.drop(30).toSeq)
+    }
+
+    val booleans = Array.tabulate(37)(i => if (i < 20) 1 else i % 2)
+    val encoded = encodeRle(booleans, 1)
+    // The level length covers exactly the rest of the page.
+    streams(BytesUtils.intToBytes(encoded.length) ++ encoded).foreach { in =>
+      val reader = new VectorizedRleValuesReader(1)
+      reader.initFromPage(booleans.length, in())
+      val c = new OnHeapColumnVector(booleans.length, BooleanType)
+      reader.readBooleans(booleans.length, c, 0)
+      assert((0 until booleans.length).map(c.getBoolean) == booleans.map(_ == 1).toSeq)
+    }
+  }
 }
 
 private object VectorizedRleValuesReaderSuite {
 
+  /**
+   * The page as a single buffer and split into 1-byte buffers. The split keeps any part of the
+   * page longer than 1 byte, including a slice of it, on a `MultiBufferInputStream`.
+   */
+  private def streams(bytes: Array[Byte]): Seq[() => ByteBufferInputStream] = {
+    assert(bytes.length > 1, "the page must span multiple buffers")
+    Seq(
+      () => ByteBufferInputStream.wrap(ByteBuffer.wrap(bytes)),
+      () => ByteBufferInputStream.wrap(bytes.grouped(1).map(ByteBuffer.wrap).toList.asJava))
+  }
+
+  /** A dictionary id section: the bit width followed by the RLE/bit-packed ids. */
+  private def dictIdPage(ids: Array[Int], bitWidth: Int): Array[Byte] =
+    Array(bitWidth.toByte) ++ encodeRle(ids, bitWidth)
+
+  private val PastEnd = "reading past the end of the encoded values"
+
+  private def varint(value: Long): Array[Byte] =
+    BytesInput.fromUnsignedVarLong(value).toByteArray
+
   /**
    * Runs readBatch end-to-end and asserts null-bits, non-null values, and def levels.
    * Each batch uses a fresh output vector since `state.valueOffset` resets to 0 per batch,


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org