You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/10/15 13:05:37 UTC
(spark) branch branch-4.3 updated: [SPARK-59832][4.X][SQL] Fail on truncated input in the vectorized RLE reader
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new 95fd6623a67e [SPARK-59832][4.X][SQL] Fail on truncated input in the vectorized RLE reader
95fd6623a67e is described below
commit 95fd6623a67e4f1b558b4b5fab8b958cf909343f
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Thu Oct 15 14:05:01 2026 +0100
[SPARK-59832][4.X][SQL] Fail on truncated input in the vectorized RLE reader
### What changes were proposed in this pull request?
Backport of #59117 to branch-4.x.
The cherry-pick conflicted only on code that SPARK-59620 added to master, which is not on this branch: an import in `VectorizedColumnReader` and `VectorizedRleValuesReaderSuite`, and a doc comment next to the new `readBatchRepeated` bridge in `ParquetTestAccess`. They are left out; otherwise the change is the same as on master.
This commit also applies cleanly to branch-4.3, so it can be cherry-picked there when merging. The same change was built and tested on branch-4.3.
### Why are the changes needed?
Same as #59117. On a corrupt page, the vectorized RLE reader, which decodes definition/repetition levels, dictionary ids and RLE booleans, silently returned wrong values or looped forever when the encoded data ended early, and a corrupt bit-packed header could overflow `currentCount` or allocate a huge buffer. The reader now fails with a `ParquetDecodingException`.
### Does this PR introduce _any_ user-facing change?
Only for corrupt or non-compliant Parquet files, as described in #59117: such pages now fail with a `ParquetDecodingException` instead of returning wrong values or hanging, and with `spark.sql.files.ignoreCorruptFiles=true` the rest of such a file is skipped. A few of these pages could be read before and gave the right result by chance; #59117 lists them.
### How was this patch tested?
The 18 tests added in #59117 pass on this branch, as do the existing tests in `ParquetEncodingSuite`, `ParquetVectorizedSuite`, `ParquetColumnIndexSuite`, `ParquetIOSuite`, `ParquetV1QuerySuite`, `ParquetV2QuerySuite`, `ParquetDeltaByteArrayEncodingSuite`, `ParquetDeltaLengthByteArrayEncodingSuite`, `VectorizedPlainValuesReaderSuite` and `ParquetVectorUpdaterSuite`.
```
build/sbt 'sql/testOnly *VectorizedRleValuesReaderSuite *ParquetEncodingSuite *ParquetVectorizedSuite *ParquetIOSuite'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5.5)
Closes #59481 from viirya/SPARK-59832-branch-4.x.
Authored-by: Liang-Chi Hsieh <vi...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 45f1cc8f1efe165eecf45f80302f8a4c0b74ee11)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../parquet/VectorizedColumnReader.java | 22 +-
.../parquet/VectorizedRleValuesReader.java | 80 ++++--
.../datasources/parquet/ParquetIOSuite.scala | 89 ++++++
.../datasources/parquet/ParquetTestAccess.scala | 24 ++
.../parquet/ParquetVectorizedSuite.scala | 27 ++
.../parquet/VectorizedRleValuesReaderSuite.scala | 317 ++++++++++++++++++++-
6 files changed, 530 insertions(+), 29 deletions(-)
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index eff701082589..edb4638eaf22 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -32,6 +32,7 @@ import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.page.*;
import org.apache.parquet.column.values.RequiresPreviousReader;
import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.ParquetDecodingException;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation;
import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
@@ -132,6 +133,9 @@ public class VectorizedColumnReader {
this.isCurrentPageDictionaryEncoded = true;
} catch (IOException e) {
throw new IOException("could not decode the dictionary for " + descriptor, e);
+ } catch (ParquetDecodingException e) {
+ throw new ParquetDecodingException(
+ "could not decode the dictionary for " + descriptor + ": " + e.getMessage(), e);
}
} else {
this.dictionary = null;
@@ -343,9 +347,7 @@ public class VectorizedColumnReader {
if (dataEncoding.usesDictionary()) {
this.dataColumn = null;
if (dictionary == null) {
- throw new IOException(
- "could not read page in col " + descriptor +
- " as the dictionary was missing for encoding " + dataEncoding);
+ throw new IOException("the dictionary was missing for encoding " + dataEncoding);
}
@SuppressWarnings("deprecation")
Encoding plainDict = Encoding.PLAIN_DICTIONARY; // var to allow warning suppression
@@ -360,11 +362,9 @@ public class VectorizedColumnReader {
this.isCurrentPageDictionaryEncoded = false;
}
- try {
- dataColumn.initFromPage(pageValueCount, in);
- } catch (IOException e) {
- throw new IOException("could not read page in col " + descriptor, e);
- }
+ // readPageV1 and readPageV2 add the page and the column to an IOException or a
+ // ParquetDecodingException thrown here.
+ dataColumn.initFromPage(pageValueCount, in);
// for PARQUET-246 (See VectorizedDeltaByteArrayReader.setPreviousValues)
if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, dataEncoding) &&
previousReader instanceof RequiresPreviousReader) {
@@ -434,6 +434,9 @@ public class VectorizedColumnReader {
return pageValueCount;
} catch (IOException e) {
throw new IOException("could not read page " + page + " in col " + descriptor, e);
+ } catch (ParquetDecodingException e) {
+ throw new ParquetDecodingException(
+ "could not read page " + page + " in col " + descriptor + ": " + e.getMessage(), e);
}
}
@@ -454,6 +457,9 @@ public class VectorizedColumnReader {
return pageValueCount;
} catch (IOException e) {
throw new IOException("could not read page " + page + " in col " + descriptor, e);
+ } catch (ParquetDecodingException e) {
+ throw new ParquetDecodingException(
+ "could not read page " + page + " in col " + descriptor + ": " + e.getMessage(), e);
}
}
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
index d1b2726ceeb0..8f240c13c164 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
@@ -62,6 +62,9 @@ public final class VectorizedRleValuesReader extends ValuesReader
private int bytesWidth;
private BytePacker packer;
+ // True if the dictionary id section of the page is empty, not even holding the bit width.
+ private boolean emptyIdSection;
+
// Current decoding mode and values
private MODE mode;
private int currentCount;
@@ -96,16 +99,35 @@ public final class VectorizedRleValuesReader extends ValuesReader
@Override
public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException {
this.in = in;
+ this.emptyIdSection = false;
if (fixedWidth) {
// Initialize for repetition and definition levels
if (readLength) {
+ if (in.available() < 4) {
+ throw new ParquetDecodingException("Corrupted RLE data: the 4-byte length is cut off, " +
+ "only " + in.available() + " bytes are left in the page");
+ }
int length = readIntLittleEndian();
+ if (length < 0 || length > in.available()) {
+ throw new ParquetDecodingException("Corrupted RLE data: invalid length " + length +
+ ", " + in.available() + " bytes are left in the page");
+ }
this.in = in.sliceStream(length);
}
} else {
// Initialize for values
if (in.available() > 0) {
- init(in.read());
+ int width = in.read();
+ if (width > 32) {
+ throw new ParquetDecodingException("Corrupted RLE data: invalid bit width " + width);
+ }
+ init(width);
+ } else {
+ // No encoded values (e.g. an all-null page). Do not treat this as a bit width of 0,
+ // which would decode every value as 0; any read fails in `readNextGroup` instead.
+ this.emptyIdSection = true;
+ this.currentCount = 0;
+ return;
}
}
if (bitWidth == 0) {
@@ -208,7 +230,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
int leftInPage = state.valuesToReadInPage;
while (leftInBatch > 0 && leftInPage > 0) {
- if (currentCount == 0 && !readNextGroup()) break;
+ if (currentCount == 0) readNextGroup();
int n = Math.min(leftInBatch, Math.min(leftInPage, this.currentCount));
long rangeStart = state.currentRangeStart();
@@ -318,7 +340,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
int leftInPage = state.valuesToReadInPage;
while (leftInBatch > 0 && leftInPage > 0) {
- if (currentCount == 0 && !readNextGroup()) break;
+ if (currentCount == 0) readNextGroup();
int n = Math.min(leftInBatch, Math.min(leftInPage, this.currentCount));
long rangeStart = state.currentRangeStart();
@@ -431,7 +453,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
values, nulls, valuesReused, valueReader, updater);
while ((leftInBatch > 0 || !state.lastListCompleted) && leftInPage > 0) {
- if (currentCount == 0 && !readNextGroup()) break;
+ if (currentCount == 0) readNextGroup();
// Values to read in the current RLE/PACKED block, must be <= what's left in the page
int valuesLeftInBlock = Math.min(leftInPage, currentCount);
@@ -637,7 +659,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
int n = total;
int initialValueOffset = state.valueOffset;
while (n > 0) {
- if (currentCount == 0 && !readNextGroup()) break;
+ if (currentCount == 0) readNextGroup();
int num = Math.min(n, this.currentCount);
readValuesN(num, state, defLevels, values, nulls, valueReader, updater);
state.levelOffset += num;
@@ -731,7 +753,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
VectorizedValuesReader valuesReader,
ParquetVectorUpdater updater) {
while (n > 0) {
- if (currentCount == 0 && !readNextGroup()) break;
+ if (currentCount == 0) readNextGroup();
int num = Math.min(n, this.currentCount);
switch (mode) {
case RLE -> {
@@ -764,7 +786,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
public void readIntegers(int total, WritableColumnVector c, int rowId) {
int left = total;
while (left > 0) {
- if (currentCount == 0 && !readNextGroup()) break;
+ if (currentCount == 0) readNextGroup();
int n = Math.min(left, this.currentCount);
switch (mode) {
case RLE -> c.putInts(rowId, n, currentValue);
@@ -957,7 +979,6 @@ public final class VectorizedRleValuesReader extends ValuesReader
*/
private int readIntLittleEndianPaddedOnBitWidth() throws IOException {
return switch (bytesWidth) {
- case 0 -> 0;
case 1 -> in.read();
case 2 -> {
int ch2 = in.read();
@@ -976,14 +997,30 @@ public final class VectorizedRleValuesReader extends ValuesReader
}
/**
- * Reads the next group. Returns false if no more group available.
+ * Reads the next non-empty group, so that `currentCount` is positive on return.
+ * <p>
+ * Callers never read more values than the page declares, so running out of encoded data here
+ * means the page is corrupted. With a bit width of 0, `initFromPage` already provides all the
+ * values of the page, so reaching here also means reading past the end.
*/
- private boolean readNextGroup() {
- if (in.available() <= 0) {
- currentCount = 0;
- return false;
+ private void readNextGroup() {
+ if (emptyIdSection) {
+ throw new ParquetDecodingException(
+ "Corrupted RLE data: the dictionary id section is empty, not even the bit width");
}
+ do {
+ if (bitWidth == 0 || in.available() <= 0) {
+ throw new ParquetDecodingException(
+ "Corrupted RLE data: reading past the end of the encoded values");
+ }
+ readGroup();
+ } while (this.currentCount == 0);
+ }
+ /**
+ * Reads the next group, which may be empty.
+ */
+ private void readGroup() {
try {
int header = readUnsignedVarInt();
this.mode = (header & 1) == 0 ? MODE.RLE : MODE.PACKED;
@@ -994,6 +1031,17 @@ public final class VectorizedRleValuesReader extends ValuesReader
}
case PACKED -> {
int numGroups = header >>> 1;
+ // Validate before allocating the buffer, so a corrupted header can neither overflow
+ // `currentCount` nor make us allocate more than the remaining bytes can fill.
+ if (numGroups > Integer.MAX_VALUE / 8) {
+ throw new ParquetDecodingException(
+ "Corrupted RLE data: bit-packed run of " + numGroups + " groups is too long");
+ }
+ if ((long) numGroups * bitWidth > in.available()) {
+ throw new ParquetDecodingException("Corrupted RLE data: bit-packed run of " +
+ numGroups + " groups needs " + (long) numGroups * bitWidth + " bytes, but only " +
+ in.available() + " bytes are left");
+ }
this.currentCount = numGroups * 8;
if (this.currentBuffer.length < this.currentCount) {
@@ -1014,10 +1062,8 @@ public final class VectorizedRleValuesReader extends ValuesReader
}
}
} catch (IOException e) {
- throw new ParquetDecodingException("Failed to read from input stream", e);
+ throw new ParquetDecodingException("Corrupted RLE data: failed to read from input stream", e);
}
-
- return true;
}
/**
@@ -1026,7 +1072,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
private void skipValues(int n) {
int left = n;
while (left > 0) {
- if (this.currentCount == 0 && !readNextGroup()) break;
+ if (this.currentCount == 0) readNextGroup();
int num = Math.min(left, this.currentCount);
switch (mode) {
case RLE -> {}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index b9ac9df8e31d..3a938a985781 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.execution.datasources.parquet
import java.math.{BigDecimal => JBigDecimal}
+import java.nio.file.Files
import java.time.{LocalDateTime, LocalTime}
import java.util.Locale
@@ -27,16 +28,21 @@ import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import com.google.common.primitives.UnsignedLong
+import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext}
+import org.apache.parquet.bytes.BytesUtils
import org.apache.parquet.column.{Encoding, ParquetProperties}
import org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0
import org.apache.parquet.example.data.Group
import org.apache.parquet.example.data.simple.{SimpleGroup, SimpleGroupFactory}
import org.apache.parquet.hadoop._
import org.apache.parquet.hadoop.example.ExampleParquetWriter
+import org.apache.parquet.io.ParquetDecodingException
import org.apache.parquet.io.api.Binary
import org.apache.parquet.schema.{MessageType, MessageTypeParser}
+import org.scalatest.concurrent.{Signaler, ThreadSignaler}
+import org.scalatest.time.{Seconds, Span}
import org.apache.spark.{SPARK_VERSION_SHORT, SparkException, TestUtils}
import org.apache.spark.rdd.RDD
@@ -2410,6 +2416,89 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
}
}
}
+
+ /**
+ * Writes 100 rows of `column` to a single uncompressed data page, lets `corrupt` modify the
+ * RLE section that starts with `03 00 00 00 C8 01 01` (the 4-byte length 3, then one RLE run
+ * with header 100 << 1 and value 1), and returns the error of reading the file with the
+ * vectorized reader. For a nullable INT32 column that is the definition levels; for a
+ * BOOLEAN column written as a V2 page it is the RLE-encoded values.
+ */
+ private def readCorruptedRleSection(
+ column: String,
+ writerVersion: String = "PARQUET_1_0")(
+ corrupt: (Array[Byte], Int) => Unit): Throwable = {
+ var error: Throwable = null
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+ spark.range(100).selectExpr(column).coalesce(1)
+ .write
+ .option(ParquetOutputFormat.ENABLE_DICTIONARY, "false")
+ .option(ParquetOutputFormat.WRITER_VERSION, writerVersion)
+ .option("compression", "uncompressed")
+ .parquet(path)
+ val file = dir.listFiles().filter(_.getName.endsWith(".parquet")).head
+ val bytes = Files.readAllBytes(file.toPath)
+ val section = Array[Byte](3, 0, 0, 0, 0xC8.toByte, 0x01, 1)
+ val offset = bytes.indexOfSlice(section)
+ assert(offset >= 0 && bytes.indexOfSlice(section, offset + 1) < 0)
+ corrupt(bytes, offset)
+ Files.write(file.toPath, bytes)
+ dir.listFiles().filter(_.getName.endsWith(".crc")).foreach(_.delete())
+
+ withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
+ // Before SPARK-59832 a truncated read never returned, so run it with a timeout. The test
+ // thread waits interruptibly for the job, so it can be signaled directly.
+ implicit val signaler: Signaler = ThreadSignaler
+ error = intercept[SparkException] {
+ failAfter(Span(60, Seconds))(spark.read.parquet(path).collect())
+ }
+ }
+ }
+ error
+ }
+
+ private def causes(e: Throwable): Seq[Throwable] =
+ ExceptionUtils.getThrowableList(e).asScala.toSeq
+
+ /** Sets the 4-byte length at `offset` to Int.MaxValue, far more than the rest of the page. */
+ private def setMaxLength(bytes: Array[Byte], offset: Int): Unit =
+ BytesUtils.intToBytes(Int.MaxValue).copyToArray(bytes, offset)
+
+ /**
+ * Checks that the page error is reported once with the column, directly on top of the
+ * `ParquetDecodingException` about the invalid length.
+ */
+ private def assertInvalidLengthReportsColumn(e: Throwable, column: String): Unit = {
+ val pageErrors =
+ causes(e).filter(c => Option(c.getMessage).exists(_.startsWith("could not read page")))
+ assert(pageErrors.size == 1, e)
+ assert(pageErrors.head.getMessage.contains(s"in col [$column]"), e)
+ assert(pageErrors.head.getCause.isInstanceOf[ParquetDecodingException], e)
+ assert(pageErrors.head.getCause.getMessage.contains(
+ s"Corrupted RLE data: invalid length ${Int.MaxValue}"), e)
+ }
+
+ test("SPARK-59832: vectorized reader fails on truncated definition levels") {
+ // Shorten the run to 64 values (header 64 << 1 = varint 0x80 0x01).
+ val e = readCorruptedRleSection("if(id < 0, null, cast(id as int)) as a") {
+ (bytes, offset) => bytes(offset + 4) = 0x80.toByte
+ }
+ assert(causes(e).exists {
+ case c: ParquetDecodingException => c.getMessage.contains("Corrupted RLE data")
+ case _ => false
+ }, e)
+ }
+
+ test("SPARK-59832: invalid definition level length reports the column") {
+ val e = readCorruptedRleSection("if(id < 0, null, cast(id as int)) as a")(setMaxLength)
+ assertInvalidLengthReportsColumn(e, "a")
+ }
+
+ test("SPARK-59832: invalid length of RLE booleans in a V2 page reports the column") {
+ val e = readCorruptedRleSection("id >= 0 as b", writerVersion = "PARQUET_2_0")(setMaxLength)
+ assertInvalidLengthReportsColumn(e, "b")
+ }
}
class JobCommitFailureParquetOutputCommitter(outputPath: Path, context: TaskAttemptContext)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
index e008a7b0322e..5391d31a14ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
@@ -37,6 +37,7 @@ import org.apache.spark.util.SparkClassUtils
* Currently bridges:
* - `ParquetReadState` (constructor + `resetForNewBatch` + `resetForNewPage`)
* - `VectorizedRleValuesReader.readBatch` (5-arg overload not exposed publicly)
+ * - `VectorizedRleValuesReader.readBatchRepeated` (takes the package-private state)
* - `ParquetVectorUpdaterFactory` (constructor)
* - `VectorizedDeltaByteArrayReader` (no-arg constructor)
* - `VectorizedDeltaLengthByteArrayReader` (no-arg constructor)
@@ -79,6 +80,15 @@ object ParquetTestAccess {
.getOrElse(throw new NoSuchMethodException(
"VectorizedRleValuesReader.readBatch/5"))
+ private val readBatchRepeatedMethod: Method =
+ classOf[VectorizedRleValuesReader].getMethods
+ .find(m =>
+ m.getName == "readBatchRepeated"
+ && m.getParameterCount == 7
+ && m.getParameterTypes()(0) == stateCls)
+ .getOrElse(throw new NoSuchMethodException(
+ "VectorizedRleValuesReader.readBatchRepeated/7"))
+
def newState(
descriptor: ColumnDescriptor,
isRequired: Boolean,
@@ -118,6 +128,20 @@ object ParquetTestAccess {
reader, state, values, defLevels, valueReader, updater)
} catch { case e: ReflectiveOperationException => throw rethrow(e) }
+ def readBatchRepeated(
+ reader: VectorizedRleValuesReader,
+ state: AnyRef,
+ repLevels: WritableColumnVector,
+ defLevelsReader: VectorizedRleValuesReader,
+ defLevels: WritableColumnVector,
+ values: WritableColumnVector,
+ valueReader: VectorizedValuesReader,
+ updater: ParquetVectorUpdater): Unit =
+ try {
+ readBatchRepeatedMethod.invoke(
+ reader, state, repLevels, defLevelsReader, defLevels, values, valueReader, updater)
+ } catch { case e: ReflectiveOperationException => throw rethrow(e) }
+
// -------- ParquetVectorUpdaterFactory --------
private val factoryCtor = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
index 916884580068..c1f45b54767a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
@@ -524,6 +524,33 @@ class ParquetVectorizedSuite extends ParquetTest with SharedSparkSession {
}
}
+ test("SPARK-59832: a corrupted dictionary page reports the column") {
+ // A PLAIN INT32 dictionary page that declares 3 entries but holds only 2.
+ val parquetSchema = MessageTypeParser.parseMessageType("message root { optional int32 a; }")
+ val ty = parquetSchema.asGroupType().getType("a").asPrimitiveType()
+ val cd = new ColumnDescriptor(Array("a"), ty, 0, 1)
+ val memPageStore = new MemPageStore(1)
+ val pageWriter = memPageStore.getPageWriter(cd)
+ pageWriter.writeDictionaryPage(new DictionaryPage(
+ BytesInput.from(Array[Byte](1, 0, 0, 0, 2, 0, 0, 0)), 3, Encoding.PLAIN_DICTIONARY))
+ // One non-null value: the definition levels (length 2, an RLE run of 1 x 1), then the
+ // dictionary ids (bit width 2, an RLE run of 1 x 0).
+ pageWriter.writePage(BytesInput.from(Array[Byte](2, 0, 0, 0, 2, 1, 2, 2, 0)), 1, 1,
+ Statistics.createStats(ty), Encoding.RLE, Encoding.RLE, Encoding.RLE_DICTIONARY)
+ val recordReader = new VectorizedParquetRecordReader(
+ DateTimeUtils.getZoneId("EST"), "CORRECTED", "UTC", "CORRECTED", "UTC", true, 1)
+ try {
+ recordReader.initialize(parquetSchema, parquetSchema,
+ TestParquetRowGroupReader(Seq(TestPageReadStore(memPageStore, Seq(0L)))), 1)
+ val e = intercept[ParquetDecodingException](recordReader.nextKeyValue())
+ assert(e.getMessage.contains("could not decode the dictionary for [a]"), e.getMessage)
+ assert(e.getMessage.contains(e.getCause.getMessage), e.getMessage)
+ assert(e.getCause.isInstanceOf[ParquetDecodingException], e)
+ } finally {
+ recordReader.close()
+ }
+ }
+
truncateTypeTest("primitive type", IntegerType, LongType, IntegerType)
truncateTypeTest("basic struct",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala
index 5b3997160ece..6e37351873b4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala
@@ -19,25 +19,92 @@ package org.apache.spark.sql.execution.datasources.parquet
import java.nio.ByteBuffer
import java.util.PrimitiveIterator
+import java.util.concurrent.{Callable, ExecutionException, TimeoutException, TimeUnit}
import scala.jdk.CollectionConverters._
-import org.apache.parquet.bytes.ByteBufferInputStream
+import org.apache.parquet.bytes.{ByteBufferInputStream, BytesInput, BytesUtils}
+import org.apache.parquet.column.ColumnDescriptor
+import org.apache.parquet.io.ParquetDecodingException
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
+import org.apache.parquet.schema.Type.Repetition
+import org.apache.parquet.schema.Types
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.execution.datasources.parquet.VectorizedRleValuesReaderTestUtils._
import org.apache.spark.sql.execution.vectorized.{OnHeapColumnVector, WritableColumnVector}
-import org.apache.spark.sql.types.IntegerType
+import org.apache.spark.sql.types.{BooleanType, IntegerType}
+import org.apache.spark.util.ThreadUtils
/**
* Focused correctness tests for `VectorizedRleValuesReader.readBatch` PACKED-mode decoding,
* covering patterns the P0 optimization cares about: run boundaries, batch boundaries, and
* nested def-level grouping. Uses the same reflection bridge as the benchmark.
+ *
+ * The SPARK-59832 tests check that corrupted or truncated input fails with a
+ * `ParquetDecodingException` instead of returning wrong values, looping forever or allocating
+ * a huge buffer, and that reads ending exactly at the end of the encoded values still work.
*/
class VectorizedRleValuesReaderSuite extends SparkFunSuite {
import VectorizedRleValuesReaderSuite._
+ // Runs the reads that may never return before SPARK-59832, a thread per read, so that a read
+ // stuck in a loop does not block the following ones. The decode loops do not check for
+ // interruption, so a stuck read keeps its thread busy until the JVM exits. After the first
+ // timeout, later reads fail without running, so that a regression leaves one busy thread.
+ private val executor = ThreadUtils.newDaemonCachedThreadPool("rle-reader-suite")
+ @volatile private var timedOut = false
+
+ override def afterAll(): Unit = {
+ try {
+ executor.shutdownNow()
+ executor.awaitTermination(10, TimeUnit.SECONDS)
+ } finally {
+ super.afterAll()
+ }
+ }
+
+ /**
+ * Checks that `f` fails with a `ParquetDecodingException` whose message contains
+ * "Corrupted RLE data: " followed by `expected`.
+ */
+ private def assertCorrupted(expected: String)(f: => Any): Unit = {
+ val e = intercept[ParquetDecodingException](f)
+ assert(e.getMessage.contains("Corrupted RLE data: " + expected), e.getMessage)
+ }
+
+ /**
+ * Like `assertCorrupted`, but runs `f` on another thread with a timeout, so that a read that
+ * never returns fails the test.
+ */
+ private def interceptCorrupted(expected: String)(f: => Any): Unit = {
+ assert(!timedOut, "Not run, since an earlier read did not return")
+ val future = executor.submit(new Callable[Any] {
+ override def call(): Any = f
+ })
+ assertCorrupted(expected) {
+ try {
+ future.get(10, TimeUnit.SECONDS)
+ } catch {
+ case e: ExecutionException => throw e.getCause
+ case e: TimeoutException =>
+ timedOut = true
+ throw e
+ }
+ }
+ }
+
+ /** Checks that reading `n` dictionary ids from `page` fails, on every stream of `page`. */
+ private def assertReadIntegersFails(page: Array[Byte], n: Int, expected: String): Unit = {
+ streams(page).foreach { in =>
+ val reader = new VectorizedRleValuesReader()
+ reader.initFromPage(n, in())
+ val c = new OnHeapColumnVector(n, IntegerType)
+ interceptCorrupted(expected)(reader.readIntegers(n, c, 0))
+ }
+ }
+
test("PACKED: alternating null/non-null (many single-element runs)") {
val n = 1024
val defLevels = Array.tabulate(n)(i => i & 1)
@@ -96,7 +163,7 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
test("PACKED group larger than initial 16-int currentBuffer (triggers buffer grow)") {
// ~1024 alternating values produce one PACKED block well beyond the initial 16-int buffer,
- // exercising `new int[currentCount]` in readNextGroup.
+ // exercising `new int[currentCount]` in readGroup.
val defLevels = Array.tabulate(1024)(i => i & 1)
runAndAssert(defLevels, maxDef = 1, batchSize = 1024, withDefLevels = false)
}
@@ -134,7 +201,7 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
test("PACKED: multi-buffer stream with packed run crossing buffer boundary") {
// Exercises the MultiBufferInputStream.slice() path where the packed bytes span a buffer
// boundary, causing slice() to return a freshly-allocated contiguous buffer with
- // position() == 0 (the base-0 pos branch in readNextGroup).
+ // position() == 0 (the base-0 pos branch in readGroup).
val n = 256
val defLevels = Array.tabulate(n)(i => i & 1)
val bitWidth = 1
@@ -185,10 +252,252 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
produced += toRead
}
}
+
+ test("SPARK-59832: truncated dictionary ids fail instead of being partially read") {
+ // A dictionary id page with 10 ids, read as if it had 20.
+ val page = dictIdPage(Array.fill(10)(3), bitWidth = 4)
+ assertReadIntegersFails(page, 20, PastEnd)
+ streams(page).foreach { in =>
+ val reader = new VectorizedRleValuesReader()
+ reader.initFromPage(20, in())
+ interceptCorrupted(PastEnd)(reader.skipIntegers(20))
+ }
+ streams(page).foreach { in =>
+ val reader = new VectorizedRleValuesReader()
+ reader.initFromPage(20, in())
+ (0 until 10).foreach(_ => assert(reader.readInteger() == 3))
+ interceptCorrupted(PastEnd)(reader.readInteger())
+ }
+ }
+
+ test("SPARK-59832: truncated booleans fail instead of looping forever") {
+ val encoded = encodeRle(Array.fill(10)(1), 1)
+ streams(BytesUtils.intToBytes(encoded.length) ++ encoded).foreach { in =>
+ val reader = new VectorizedRleValuesReader(1)
+ reader.initFromPage(20, in())
+ val c = new OnHeapColumnVector(20, BooleanType)
+ interceptCorrupted(PastEnd)(reader.readBooleans(20, c, 0))
+ }
+ }
+
+ test("SPARK-59832: truncated definition levels fail in readBatch") {
+ // The page declares 100 values but its definition levels only encode 64. Before the fix
+ // readBatch returned without progress, so VectorizedColumnReader.readBatch spun forever.
+ val n = 100
+ val encoded = encodeRle(Array.fill(64)(1), 1)
+ val plainBytes = plainIntBytes(n)(valueAt)
+ def run(withDefLevels: Boolean, rowIndexes: PrimitiveIterator.OfLong): Unit = {
+ val reader = new VectorizedRleValuesReader(1, false)
+ reader.initFromPage(n, ByteBufferInputStream.wrap(ByteBuffer.wrap(encoded)))
+ val valueReader = new VectorizedPlainValuesReader
+ valueReader.initFromPage(n, ByteBufferInputStream.wrap(ByteBuffer.wrap(plainBytes)))
+ val state = ParquetTestAccess.newState(intColumnDescriptor(1), false, rowIndexes)
+ ParquetTestAccess.resetForNewPage(state, n, 0L)
+ ParquetTestAccess.resetForNewBatch(state, n)
+ val values = new OnHeapColumnVector(n, IntegerType)
+ val defLevels = if (withDefLevels) new OnHeapColumnVector(n, IntegerType) else null
+ interceptCorrupted(PastEnd) {
+ ParquetTestAccess.readBatch(reader, state, values, defLevels, valueReader, integerUpdater)
+ }
+ }
+ run(withDefLevels = false, rowIndexes = null)
+ run(withDefLevels = true, rowIndexes = null)
+ // Skips the truncated levels to reach rows 80 to 90.
+ run(withDefLevels = false, rowIndexes = longIterator((80 to 90).toArray))
+ }
+
+ test("SPARK-59832: truncated repetition levels fail in readBatchRepeated") {
+ // Every row is a top-level row (repetition level 0), but only 64 of 100 are encoded.
+ readRepeated(repLevels = Array.fill(64)(0), defLevels = Array.fill(100)(1), n = 100)
+ }
+
+ test("SPARK-59832: truncated definition levels fail in readBatchRepeated") {
+ // The repetition levels are complete, but only 64 of 100 definition levels are encoded.
+ // Before the fix the missing definition levels were silently left as 0.
+ readRepeated(repLevels = Array.fill(100)(0), defLevels = Array.fill(64)(1), n = 100)
+ // Skips the truncated levels to reach rows 80 to 90.
+ readRepeated(repLevels = Array.fill(100)(0), defLevels = Array.fill(64)(1), n = 100,
+ rowIndexes = longIterator((80 to 90).toArray))
+ }
+
+ private def readRepeated(
+ repLevels: Array[Int],
+ defLevels: Array[Int],
+ n: Int,
+ rowIndexes: PrimitiveIterator.OfLong = null): Unit = {
+ val prim = Types.primitive(PrimitiveTypeName.INT32, Repetition.REPEATED).named("col")
+ val descriptor = new ColumnDescriptor(Array("col"), prim, 1, 1)
+ val repReader = new VectorizedRleValuesReader(1, false)
+ repReader.initFromPage(
+ n, ByteBufferInputStream.wrap(ByteBuffer.wrap(encodeRle(repLevels, 1))))
+ val defReader = new VectorizedRleValuesReader(1, false)
+ defReader.initFromPage(
+ n, ByteBufferInputStream.wrap(ByteBuffer.wrap(encodeRle(defLevels, 1))))
+ val valueReader = new VectorizedPlainValuesReader
+ valueReader.initFromPage(
+ n, ByteBufferInputStream.wrap(ByteBuffer.wrap(plainIntBytes(n)(valueAt))))
+ val state = ParquetTestAccess.newState(descriptor, false, rowIndexes)
+ ParquetTestAccess.resetForNewPage(state, n, 0L)
+ ParquetTestAccess.resetForNewBatch(state, n)
+ val repLevelsVec = new OnHeapColumnVector(n, IntegerType)
+ val defLevelsVec = new OnHeapColumnVector(n, IntegerType)
+ val values = new OnHeapColumnVector(n, IntegerType)
+ interceptCorrupted(PastEnd) {
+ ParquetTestAccess.readBatchRepeated(repReader, state, repLevelsVec, defReader,
+ defLevelsVec, values, valueReader, integerUpdater)
+ }
+ }
+
+ test("SPARK-59832: reading past a page with bit width 0 fails") {
+ // With bit width 0 the page is one implicit run of `valueCount` zeros; the bytes that
+ // follow belong to the next section of the page and must not be decoded as levels.
+ val trailing = Array[Byte](3, 1, 2, 3)
+ val reader = new VectorizedRleValuesReader(0)
+ reader.initFromPage(5, ByteBufferInputStream.wrap(ByteBuffer.wrap(trailing)))
+ val c = new OnHeapColumnVector(6, IntegerType)
+ c.putInts(0, 6, -1)
+ reader.readIntegers(5, c, 0)
+ assert((0 until 5).forall(c.getInt(_) == 0))
+ interceptCorrupted(PastEnd)(reader.readIntegers(1, c, 5))
+ }
+
+ test("SPARK-59832: invalid level length is rejected") {
+ val payload = Array[Byte](10, 20, 30, 40)
+ Seq(-1, -4, -5, payload.length + 1, Int.MaxValue).foreach { length =>
+ streams(BytesUtils.intToBytes(length) ++ payload).foreach { in =>
+ val reader = new VectorizedRleValuesReader(1)
+ assertCorrupted(s"invalid length $length")(reader.initFromPage(8, in()))
+ }
+ }
+ }
+
+ test("SPARK-59832: invalid bit-packed run is rejected before allocating its buffer") {
+ def check(bitWidth: Int, numGroups: Long, data: Array[Byte])(expected: String): Unit =
+ assertReadIntegersFails(
+ Array(bitWidth.toByte) ++ varint((numGroups << 1) | 1) ++ data, 10, expected)
+ // Truncated last group. This was already rejected before (by `in.slice`), even when the
+ // bytes cover every value read. parquet-java and Arrow accept it, for compatibility with
+ // writers that do not pad the last group.
+ check(bitWidth = 4, numGroups = 2, data = Array.fill[Byte](5)(0x11))(
+ "bit-packed run of 2 groups needs 8 bytes, but only 5 bytes are left")
+ // numGroups * 8 and numGroups * bitWidth overflow to 0.
+ check(bitWidth = 8, numGroups = 1L << 29, data = encodeRle(Array.fill(10)(3), 8))(
+ s"bit-packed run of ${1 << 29} groups is too long")
+ // numGroups * 8 and numGroups * bitWidth overflow to negative values.
+ check(bitWidth = 2, numGroups = Int.MaxValue, data = Array.fill[Byte](16)(0))(
+ s"bit-packed run of ${Int.MaxValue} groups is too long")
+ // Would allocate an int[2^30] buffer before noticing that only 3 bytes are left.
+ check(bitWidth = 1, numGroups = 1L << 27, data = Array[Byte](1, 2, 3))(
+ s"bit-packed run of ${1 << 27} groups needs ${1 << 27} bytes, but only 3 bytes are left")
+ }
+
+ test("SPARK-59832: data ending inside a run header or an RLE value fails") {
+ // The continuation byte of the run header is missing, and an RLE run of 10 without its
+ // value.
+ Seq(Array[Byte](4, 0x80.toByte), Array[Byte](4, 0x14)).foreach { page =>
+ assertReadIntegersFails(page, 10, "failed to read from input stream")
+ }
+ }
+
+ test("SPARK-59832: cut-off level length is rejected") {
+ streams(Array[Byte](3, 0)).foreach { in =>
+ val reader = new VectorizedRleValuesReader(1)
+ assertCorrupted("the 4-byte length is cut off, only 2 bytes are left in the page") {
+ reader.initFromPage(8, in())
+ }
+ }
+ }
+
+ test("SPARK-59832: invalid dictionary id bit width is rejected") {
+ Seq(33, 255).foreach { bitWidth =>
+ streams(Array[Byte](bitWidth.toByte, 0x14, 0)).foreach { in =>
+ val reader = new VectorizedRleValuesReader()
+ assertCorrupted(s"invalid bit width $bitWidth")(reader.initFromPage(10, in()))
+ }
+ }
+ }
+
+ test("SPARK-59832: dictionary ids of an empty page are not decoded as 0") {
+ val reader = new VectorizedRleValuesReader()
+ reader.initFromPage(5, ByteBufferInputStream.wrap(ByteBuffer.wrap(Array.emptyByteArray)))
+ // An all-null page reads no ids.
+ val c = new OnHeapColumnVector(5, IntegerType)
+ reader.readIntegers(0, c, 0)
+ reader.skipIntegers(0)
+ interceptCorrupted("the dictionary id section is empty, not even the bit width") {
+ reader.readIntegers(5, c, 0)
+ }
+ }
+
+ test("SPARK-59832: runs of length 0 are skipped") {
+ // [bit width 4][RLE run of 0 x 7][bit-packed run of 0 groups][RLE run of 5 x 3]
+ val page = Array[Byte](4, 0, 7, 1, 10, 3)
+ streams(page).foreach { in =>
+ val reader = new VectorizedRleValuesReader()
+ reader.initFromPage(5, in())
+ assert((0 until 5).map(_ => reader.readInteger()) == Seq.fill(5)(3))
+ }
+ streams(page).foreach { in =>
+ val reader = new VectorizedRleValuesReader()
+ reader.initFromPage(5, in())
+ val c = new OnHeapColumnVector(5, IntegerType)
+ reader.readIntegers(5, c, 0)
+ assert((0 until 5).map(c.getInt) == Seq.fill(5)(3))
+ }
+ }
+
+ test("SPARK-59832: reads that end exactly at the end of the encoded values succeed") {
+ // A run of 20 identical ids (RLE) followed by 17 mixed ids (bit-packed, padded to 24).
+ val ids = Array.tabulate(37)(i => if (i < 20) 5 else i % 7)
+ streams(dictIdPage(ids, bitWidth = 3)).foreach { in =>
+ val reader = new VectorizedRleValuesReader()
+ reader.initFromPage(ids.length, in())
+ val c = new OnHeapColumnVector(ids.length, IntegerType)
+ reader.readIntegers(ids.length, c, 0)
+ assert((0 until ids.length).map(c.getInt) == ids.toSeq)
+ }
+ streams(dictIdPage(ids, bitWidth = 3)).foreach { in =>
+ val reader = new VectorizedRleValuesReader()
+ reader.initFromPage(ids.length, in())
+ reader.skipIntegers(30)
+ assert((30 until ids.length).map(_ => reader.readInteger()) == ids.drop(30).toSeq)
+ }
+
+ val booleans = Array.tabulate(37)(i => if (i < 20) 1 else i % 2)
+ val encoded = encodeRle(booleans, 1)
+ // The level length covers exactly the rest of the page.
+ streams(BytesUtils.intToBytes(encoded.length) ++ encoded).foreach { in =>
+ val reader = new VectorizedRleValuesReader(1)
+ reader.initFromPage(booleans.length, in())
+ val c = new OnHeapColumnVector(booleans.length, BooleanType)
+ reader.readBooleans(booleans.length, c, 0)
+ assert((0 until booleans.length).map(c.getBoolean) == booleans.map(_ == 1).toSeq)
+ }
+ }
}
private object VectorizedRleValuesReaderSuite {
+ /**
+ * The page as a single buffer and split into 1-byte buffers. The split keeps any part of the
+ * page longer than 1 byte, including a slice of it, on a `MultiBufferInputStream`.
+ */
+ private def streams(bytes: Array[Byte]): Seq[() => ByteBufferInputStream] = {
+ assert(bytes.length > 1, "the page must span multiple buffers")
+ Seq(
+ () => ByteBufferInputStream.wrap(ByteBuffer.wrap(bytes)),
+ () => ByteBufferInputStream.wrap(bytes.grouped(1).map(ByteBuffer.wrap).toList.asJava))
+ }
+
+ /** A dictionary id section: the bit width followed by the RLE/bit-packed ids. */
+ private def dictIdPage(ids: Array[Int], bitWidth: Int): Array[Byte] =
+ Array(bitWidth.toByte) ++ encodeRle(ids, bitWidth)
+
+ private val PastEnd = "reading past the end of the encoded values"
+
+ private def varint(value: Long): Array[Byte] =
+ BytesInput.fromUnsignedVarLong(value).toByteArray
+
/**
* Runs readBatch end-to-end and asserts null-bits, non-null values, and def levels.
* Each batch uses a fresh output vector since `state.valueOffset` resets to 0 per batch,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org