You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by do...@apache.org on 2026/07/05 10:16:45 UTC
(orc) branch branch-2.3 updated: ORC-2188: Reject invalid compression block size in PostScript
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new d291908bf ORC-2188: Reject invalid compression block size in PostScript
d291908bf is described below
commit d291908bff0e7194b2ac3850f6ae6666b537a751
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sun Jul 5 03:16:12 2026 -0700
ORC-2188: Reject invalid compression block size in PostScript
### What changes were proposed in this pull request?
This PR aims to reject invalid PostScript `compressionBlockSize` values at file open in both C++ and Java readers. If the field is present and the value is `0` or `>= 2^23 (8MB)`, the readers now throw `ParseError` (C++) / `FileFormatException` (Java) with the message `Invalid compression block size: N`. A missing field keeps the existing 256KB default.
- **C++**: check added in `getCompressionBlockSize` (`Reader.cc`), which covers all open paths.
- **Java**: new `ReaderImpl.checkCompressionBlockSize` helper, called from `extractPostScript`, the `OrcTail` constructor branch, and the deprecated `extractFileTail(ByteBuffer)`.
### Why are the changes needed?
Both writers already enforce `compressionBlockSize < 2^23` because the compressed chunk header stores the length in 23 bits, but the readers used the value without validation. Applying the same bound at read time fails fast without affecting any legitimate file.
### How was this patch tested?
Pass the CIs with newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #2650
Closes #2662 from dongjoon-hyun/ORC-2188.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 5d3cf1318b58416719e89169ae9d56f758311f7e)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
c++/src/Reader.cc | 10 ++-
c++/test/TestReader.cc | 73 ++++++++++++++++++++++
.../src/java/org/apache/orc/impl/ReaderImpl.java | 24 +++++++
.../test/org/apache/orc/impl/TestReaderImpl.java | 66 +++++++++++++++++++
4 files changed, 172 insertions(+), 1 deletion(-)
diff --git a/c++/src/Reader.cc b/c++/src/Reader.cc
index 91a6716a7..68a277fc0 100644
--- a/c++/src/Reader.cc
+++ b/c++/src/Reader.cc
@@ -67,7 +67,15 @@ namespace orc {
uint64_t getCompressionBlockSize(const proto::PostScript& ps) {
if (ps.has_compression_block_size()) {
- return ps.compression_block_size();
+ uint64_t size = ps.compression_block_size();
+ // The compressed chunk header stores the length in 23 bits, so the
+ // writer rejects sizes of 2^23 or more; zero is never valid.
+ if (size == 0 || size >= (1 << 23)) {
+ std::stringstream msg;
+ msg << "Invalid compression block size: " << size;
+ throw ParseError(msg.str());
+ }
+ return size;
} else {
return 256 * 1024;
}
diff --git a/c++/test/TestReader.cc b/c++/test/TestReader.cc
index c3c701846..fab981339 100644
--- a/c++/test/TestReader.cc
+++ b/c++/test/TestReader.cc
@@ -1351,4 +1351,77 @@ namespace orc {
EXPECT_THROW(createReader(std::move(stream), options), ParseError);
}
+ // Build a minimal ORC file tail whose PostScript carries the given
+ // compression_block_size varint bytes.
+ std::vector<char> composeInvalidCompressionBlockSizeFile(
+ const std::vector<char>& blockSizeVarint) {
+ std::vector<char> data;
+
+ // Add some dummy footer bytes
+ for (int i = 0; i < 20; i++) {
+ data.push_back(0x00);
+ }
+
+ // PostScript:
+ // footer_length = 0 (field 1)
+ data.push_back(0x08);
+ data.push_back(0x00);
+
+ // compression = ZLIB (field 2)
+ data.push_back(0x10);
+ data.push_back(0x01);
+
+ // compression_block_size (field 3)
+ data.push_back(0x18);
+ data.insert(data.end(), blockSizeVarint.begin(), blockSizeVarint.end());
+
+ // magic "ORC" (field 8000, length-delimited) so that the PostScript both
+ // parses cleanly and ends with the magic expected by ensureOrcFooter
+ data.push_back(static_cast<char>(0x82));
+ data.push_back(static_cast<char>(0xF4));
+ data.push_back(0x03);
+ data.push_back(0x03);
+ data.push_back('O');
+ data.push_back('R');
+ data.push_back('C');
+
+ // postscript length
+ uint8_t postscriptLen = static_cast<uint8_t>(12 + blockSizeVarint.size());
+ data.push_back(static_cast<char>(postscriptLen));
+ return data;
+ }
+
+ void expectInvalidCompressionBlockSize(const std::vector<char>& blockSizeVarint,
+ const std::string& errMsg) {
+ std::vector<char> data = composeInvalidCompressionBlockSizeFile(blockSizeVarint);
+ auto stream = std::make_unique<MemoryInputStream>(data.data(), data.size());
+ ReaderOptions options;
+ try {
+ createReader(std::move(stream), options);
+ FAIL() << "Should throw ParseError for invalid compression block size";
+ } catch (ParseError& e) {
+ EXPECT_EQ(e.what(), errMsg);
+ } catch (...) {
+ FAIL() << "Should only throw ParseError for invalid compression block size";
+ }
+ }
+
+ /**
+ * Test that a PostScript with compression_block_size = 0 throws ParseError.
+ */
+ TEST(TestReader, testInvalidCompressionBlockSizeZero) {
+ expectInvalidCompressionBlockSize({0x00}, "Invalid compression block size: 0");
+ }
+
+ /**
+ * Test that a PostScript with compression_block_size >= 2^23 throws
+ * ParseError since the compressed chunk header only has 23 length bits.
+ */
+ TEST(TestReader, testInvalidCompressionBlockSizeTooLarge) {
+ // 8388608 (1 << 23) as varint
+ expectInvalidCompressionBlockSize(
+ {static_cast<char>(0x80), static_cast<char>(0x80), static_cast<char>(0x80), 0x04},
+ "Invalid compression block size: 8388608");
+ }
+
} // namespace orc
diff --git a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
index eaa978c73..029197bd9 100644
--- a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
@@ -536,6 +536,27 @@ public class ReaderImpl implements Reader {
}
}
+ /**
+ * Check that the compression block size in the postscript, if present, is
+ * valid. The compressed chunk header stores the length in 23 bits, so the
+ * writers reject sizes of 2^23 or more (see
+ * {@link OutStream#assertBufferSizeValid(int)}) and zero is never valid.
+ * @param path the data source path for error messages (may be null)
+ * @param postscript the parsed postscript
+ */
+ protected static void checkCompressionBlockSize(Path path,
+ OrcProto.PostScript postscript
+ ) throws FileFormatException {
+ if (postscript.hasCompressionBlockSize()) {
+ long size = postscript.getCompressionBlockSize();
+ if (size <= 0 || size >= (1 << 23)) {
+ throw new FileFormatException("Malformed ORC file" +
+ (path == null ? "" : " " + path) +
+ ". Invalid compression block size: " + size);
+ }
+ }
+ }
+
/**
* Constructor that let's the user specify additional options.
* @param path pathname for file
@@ -578,6 +599,7 @@ public class ReaderImpl implements Reader {
options.orcTail(tail);
} else {
checkOrcVersion(path, orcTail.getPostScript());
+ checkCompressionBlockSize(path, orcTail.getPostScript());
tail = orcTail;
}
this.compressionKind = tail.getCompressionKind();
@@ -653,6 +675,7 @@ public class ReaderImpl implements Reader {
InStream.create("ps", buffer, psOffset, psLen));
OrcProto.PostScript ps = OrcProto.PostScript.parseFrom(in);
checkOrcVersion(path, ps);
+ checkCompressionBlockSize(path, ps);
// Check compression codec.
switch (ps.getCompression()) {
@@ -758,6 +781,7 @@ public class ReaderImpl implements Reader {
System.arraycopy(buffer.array(), psOffset, psBuffer, 0, psLen);
ps = OrcProto.PostScript.parseFrom(psBuffer);
+ checkCompressionBlockSize(null, ps);
int footerSize = (int) ps.getFooterLength();
CompressionKind compressionKind =
CompressionKind.valueOf(ps.getCompression().name());
diff --git a/java/core/src/test/org/apache/orc/impl/TestReaderImpl.java b/java/core/src/test/org/apache/orc/impl/TestReaderImpl.java
index 0213b6061..7ddf8ab9a 100644
--- a/java/core/src/test/org/apache/orc/impl/TestReaderImpl.java
+++ b/java/core/src/test/org/apache/orc/impl/TestReaderImpl.java
@@ -106,6 +106,72 @@ public class TestReaderImpl implements TestConf {
ReaderImpl.ensureOrcFooter(in, path, psLen, buffer);
}
+ @Test
+ public void testCheckCompressionBlockSizeBoundaries() throws IOException {
+ OrcProto.PostScript.Builder ps = OrcProto.PostScript.newBuilder();
+ // a missing field is valid
+ ReaderImpl.checkCompressionBlockSize(path, ps.build());
+ for (long valid : new long[]{1, (1 << 23) - 1}) {
+ ReaderImpl.checkCompressionBlockSize(path,
+ ps.setCompressionBlockSize(valid).build());
+ }
+ for (long invalid : new long[]{0, 1 << 23, 1L << 32}) {
+ FileFormatException e = assertThrows(FileFormatException.class, () ->
+ ReaderImpl.checkCompressionBlockSize(path,
+ ps.setCompressionBlockSize(invalid).build()));
+ assertTrue(e.getMessage().contains(
+ "Invalid compression block size: " + invalid));
+ }
+ }
+
+ @Test
+ public void testInvalidCompressionBlockSize() throws Exception {
+ Path tmpDir = new Path(System.getProperty("test.tmp.dir",
+ "target/test/tmp"));
+ FileSystem fs = tmpDir.getFileSystem(conf);
+ for (long invalid : new long[]{0, 1 << 23, 1L << 32}) {
+ byte[] fileBytes = composeInvalidCompressionBlockSizeFile(invalid);
+ // reject in OrcFile.createReader
+ Path tmpPath = new Path(tmpDir,
+ "invalid-compression-block-size-" + invalid + ".orc");
+ try (FSDataOutputStream out = fs.create(tmpPath, true)) {
+ out.write(fileBytes);
+ }
+ try {
+ FileFormatException e = assertThrows(FileFormatException.class, () ->
+ OrcFile.createReader(tmpPath, OrcFile.readerOptions(conf)).close());
+ assertTrue(e.getMessage().contains(
+ "Invalid compression block size: " + invalid));
+ } finally {
+ fs.delete(tmpPath, false);
+ }
+ // reject in the deprecated ReaderImpl.extractFileTail(ByteBuffer)
+ FileFormatException e = assertThrows(FileFormatException.class, () ->
+ ReaderImpl.extractFileTail(ByteBuffer.wrap(fileBytes)));
+ assertTrue(e.getMessage().contains(
+ "Invalid compression block size: " + invalid));
+ }
+ }
+
+ private static byte[] composeInvalidCompressionBlockSizeFile(long blockSize) {
+ // magic is the highest field number, so the serialized postscript ends
+ // with "ORC" as ensureOrcFooter expects
+ byte[] psBytes = OrcProto.PostScript.newBuilder()
+ .setFooterLength(0)
+ .setCompression(OrcProto.CompressionKind.ZLIB)
+ .setCompressionBlockSize(blockSize)
+ .addVersion(0).addVersion(12)
+ .setMetadataLength(0)
+ .setWriterVersion(OrcFile.CURRENT_WRITER.getId())
+ .setMagic(OrcFile.MAGIC)
+ .build().toByteArray();
+ ByteBuffer buf = ByteBuffer.allocate(3 + psBytes.length + 1);
+ buf.put(OrcFile.MAGIC.getBytes(StandardCharsets.UTF_8));
+ buf.put(psBytes);
+ buf.put((byte) psBytes.length);
+ return buf.array();
+ }
+
@Test
public void testOptionSafety() throws IOException {
Reader.Options options = new Reader.Options();