You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@poi.apache.org by fa...@apache.org on 2026/09/01 11:20:29 UTC
(poi) branch trunk updated: Spill unknown-size zip entries to temp files only once they exceed the threshold (#1236)
This is an automated email from the ASF dual-hosted git repository.
pjfanning pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/poi.git
The following commit(s) were added to refs/heads/trunk by this push:
new 76d93fcc0f Spill unknown-size zip entries to temp files only once they exceed the threshold (#1236)
76d93fcc0f is described below
commit 76d93fcc0fbcd65213898d9dc56fdc60baefabfe
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Tue Sep 1 12:20:24 2026 +0100
Spill unknown-size zip entries to temp files only once they exceed the threshold (#1236)
When temp-file buffering is enabled via
ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles, every entry
with an unknown declared size (entrySize == -1, e.g. data-descriptor
entries written by streaming zip writers) went straight to a temp file,
however small it actually was - so enabling the threshold cost a temp file
for practically every entry of a streamed zip.
Buffer such entries in memory up to the threshold instead, and spill to a
temp file (or encrypted temp data) only when the entry really turns out to
be that large, logging a warning when that happens. Small unknown-size
entries stay on the heap with no temp file, while an entry that hides its
size behind a data descriptor still cannot force more than the threshold
into memory - e.g. a ~1MB file whose entry inflates to ~92MB while staying
just above the minimum inflate ratio and just under the 100MB per-entry
cap.
Known-size entries behave as before: at or above the threshold they are
copied to a temp file, below it they are read into memory.
Co-authored-by: Claude Opus 4.8 <no...@anthropic.com>
---
.../poi/openxml4j/util/ZipArchiveFakeEntry.java | 57 +++++++++++++++---
.../openxml4j/util/TestZipArchiveFakeEntry.java | 68 ++++++++++++++++++++++
2 files changed, 117 insertions(+), 8 deletions(-)
diff --git a/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java b/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
index 83c581b9f3..e4a1a85545 100644
--- a/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
+++ b/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
@@ -22,6 +22,7 @@ import java.nio.file.Files;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
+import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.logging.log4j.Logger;
import org.apache.poi.logging.PoiLogManager;
import org.apache.poi.poifs.crypt.temp.EncryptedTempData;
@@ -82,17 +83,42 @@ public final class ZipArchiveFakeEntry extends ZipArchiveEntry implements Closea
if (threshold >= 0 && (entrySize >= threshold || entrySize == -1)) {
boolean success = false;
try {
- if (ZipInputStreamZipEntrySource.shouldEncryptTempFiles()) {
- encryptedTempData = new EncryptedTempData();
- try (OutputStream os = encryptedTempData.getOutputStream()) {
- numberOfBytes = IOUtils.copy(inp, os);
+ final long bytes;
+ if (entrySize == -1) {
+ // The entry does not declare its uncompressed size (e.g. it was written
+ // with a data descriptor by a streaming zip writer). Most such entries
+ // are small, so buffer in memory up to the temp-file threshold and only
+ // spill to a temp file if the entry really is that large - an entry that
+ // hides its size cannot force more than the threshold onto the heap, and
+ // small entries no longer cost a temp file each.
+ try (UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) {
+ final long bytesInMemory = IOUtils.copy(inp, baos, threshold);
+ final int nextByte = bytesInMemory < threshold ? -1 : inp.read();
+ if (nextByte == -1) {
+ data = baos.toByteArray();
+ bytes = data.length;
+ } else {
+ try (OutputStream os = createTempDataOutputStream()) {
+ LOG.atWarn().log("Zip entry {} does not declare its uncompressed size and is larger " +
+ "than the temp-file threshold of {} bytes - spilling it to {}",
+ entry.getName(), threshold,
+ tempFile != null ? tempFile.getAbsolutePath() : "encrypted temp data");
+ baos.writeTo(os);
+ os.write(nextByte);
+ bytes = bytesInMemory + 1 + IOUtils.copy(inp, os);
+ }
+ }
}
} else {
- tempFile = TempFile.createTempFile("poi-zip-entry", ".tmp");
- LOG.atInfo().log("Creating temp file {} for zip entry {} of size {} bytes",
- tempFile.getAbsolutePath(), entry.getName(), entrySize);
- numberOfBytes = IOUtils.copy(inp, tempFile);
+ try (OutputStream os = createTempDataOutputStream()) {
+ if (tempFile != null) {
+ LOG.atInfo().log("Creating temp file {} for zip entry {} of size {} bytes",
+ tempFile.getAbsolutePath(), entry.getName(), entrySize);
+ }
+ bytes = IOUtils.copy(inp, os);
+ }
}
+ numberOfBytes = bytes;
success = true;
} finally {
if (!success) {
@@ -131,6 +157,21 @@ public final class ZipArchiveFakeEntry extends ZipArchiveEntry implements Closea
}
}
+ /**
+ * Opens the output to buffer this entry's data outside the heap: encrypted temp data if
+ * {@link ZipInputStreamZipEntrySource#setEncryptTempFiles(boolean)} is enabled, a plain
+ * temp file otherwise. Sets the corresponding field so {@link #getInputStream()} and
+ * {@link #close()} can find it.
+ */
+ private OutputStream createTempDataOutputStream() throws IOException {
+ if (ZipInputStreamZipEntrySource.shouldEncryptTempFiles()) {
+ encryptedTempData = new EncryptedTempData();
+ return encryptedTempData.getOutputStream();
+ }
+ tempFile = TempFile.createTempFile("poi-zip-entry", ".tmp");
+ return Files.newOutputStream(tempFile.toPath());
+ }
+
@Override
public long getSize() {
return numberOfBytes;
diff --git a/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java b/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
index a3a8e6bddf..68bbf2f342 100644
--- a/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
+++ b/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
@@ -30,6 +30,7 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -135,6 +136,73 @@ public class TestZipArchiveFakeEntry {
}
}
+ @Test
+ void testUnknownSizeBelowThresholdStaysInMemory() throws IOException {
+ // an entry that does not declare its size (getSize() == -1) is buffered in memory
+ // as long as it stays below the temp-file threshold - no temp file is created
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(1000);
+ try {
+ ZipArchiveEntry entry = new ZipArchiveEntry("hack123");
+ UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get();
+ final byte[] data = "fakeValue".getBytes(StandardCharsets.UTF_8);
+ baos.write(data);
+ try (ZipArchiveFakeEntry zipArchiveFakeEntry = new ZipArchiveFakeEntry(entry, baos.toInputStream())) {
+ assertFalse(zipArchiveFakeEntry.isUnencryptedTempFileBacked());
+ assertFalse(zipArchiveFakeEntry.isEncryptedTempFileBacked());
+ assertEquals(data.length, zipArchiveFakeEntry.getSize());
+ UnsynchronizedByteArrayOutputStream baos2 = UnsynchronizedByteArrayOutputStream.builder().get();
+ try (InputStream stream = zipArchiveFakeEntry.getInputStream()) {
+ IOUtils.copy(stream, baos2);
+ }
+ assertEquals("fakeValue", baos2.toString(StandardCharsets.UTF_8.name()));
+ }
+ } finally {
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+ }
+ }
+
+ @Test
+ void testUnknownSizeAboveThresholdSpillsToTempFile() throws IOException {
+ // an entry that does not declare its size and turns out to be larger than the
+ // temp-file threshold spills to a temp file with no data lost
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(4);
+ try {
+ runTestSpill(true, false);
+ } finally {
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+ }
+ }
+
+ @Test
+ void testUnknownSizeAboveThresholdSpillsToEncryptedTempFile() throws IOException {
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(4);
+ ZipInputStreamZipEntrySource.setEncryptTempFiles(true);
+ try {
+ runTestSpill(false, true);
+ } finally {
+ ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+ ZipInputStreamZipEntrySource.setEncryptTempFiles(false);
+ }
+ }
+
+ private static void runTestSpill(boolean unencryptedTempFileExpected,
+ boolean encryptedTempFileExpected) throws IOException {
+ ZipArchiveEntry entry = new ZipArchiveEntry("hack123");
+ UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get();
+ final byte[] data = "fakeValue".getBytes(StandardCharsets.UTF_8);
+ baos.write(data);
+ try (ZipArchiveFakeEntry zipArchiveFakeEntry = new ZipArchiveFakeEntry(entry, baos.toInputStream())) {
+ assertEquals(unencryptedTempFileExpected, zipArchiveFakeEntry.isUnencryptedTempFileBacked());
+ assertEquals(encryptedTempFileExpected, zipArchiveFakeEntry.isEncryptedTempFileBacked());
+ assertEquals(data.length, zipArchiveFakeEntry.getSize());
+ UnsynchronizedByteArrayOutputStream baos2 = UnsynchronizedByteArrayOutputStream.builder().get();
+ try (InputStream stream = zipArchiveFakeEntry.getInputStream()) {
+ assertEquals(data.length, IOUtils.copy(stream, baos2));
+ }
+ assertEquals("fakeValue", baos2.toString(StandardCharsets.UTF_8.name()));
+ }
+ }
+
static void runTestSize(boolean unencryptedTempFileExpected,
boolean encryptedTempFileExpected) throws IOException {
final String fakeEntryName = "hack123";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@poi.apache.org
For additional commands, e-mail: commits-help@poi.apache.org