You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@poi.apache.org by fa...@apache.org on 2026/09/01 11:20:29 UTC

(poi) branch trunk updated: Spill unknown-size zip entries to temp files only once they exceed the threshold (#1236)

This is an automated email from the ASF dual-hosted git repository.

pjfanning pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/poi.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 76d93fcc0f Spill unknown-size zip entries to temp files only once they exceed the threshold (#1236)
76d93fcc0f is described below

commit 76d93fcc0fbcd65213898d9dc56fdc60baefabfe
Author: PJ Fanning <pj...@users.noreply.github.com>
AuthorDate: Tue Sep 1 12:20:24 2026 +0100

    Spill unknown-size zip entries to temp files only once they exceed the threshold (#1236)
    
    When temp-file buffering is enabled via
    ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles, every entry
    with an unknown declared size (entrySize == -1, e.g. data-descriptor
    entries written by streaming zip writers) went straight to a temp file,
    however small it actually was - so enabling the threshold cost a temp file
    for practically every entry of a streamed zip.
    
    Buffer such entries in memory up to the threshold instead, and spill to a
    temp file (or encrypted temp data) only when the entry really turns out to
    be that large, logging a warning when that happens. Small unknown-size
    entries stay on the heap with no temp file, while an entry that hides its
    size behind a data descriptor still cannot force more than the threshold
    into memory - e.g. a ~1MB file whose entry inflates to ~92MB while staying
    just above the minimum inflate ratio and just under the 100MB per-entry
    cap.
    
    Known-size entries behave as before: at or above the threshold they are
    copied to a temp file, below it they are read into memory.
    
    Co-authored-by: Claude Opus 4.8 <no...@anthropic.com>
---
 .../poi/openxml4j/util/ZipArchiveFakeEntry.java    | 57 +++++++++++++++---
 .../openxml4j/util/TestZipArchiveFakeEntry.java    | 68 ++++++++++++++++++++++
 2 files changed, 117 insertions(+), 8 deletions(-)

diff --git a/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java b/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
index 83c581b9f3..e4a1a85545 100644
--- a/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
+++ b/poi-ooxml/src/main/java/org/apache/poi/openxml4j/util/ZipArchiveFakeEntry.java
@@ -22,6 +22,7 @@ import java.nio.file.Files;
 
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
+import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
 import org.apache.logging.log4j.Logger;
 import org.apache.poi.logging.PoiLogManager;
 import org.apache.poi.poifs.crypt.temp.EncryptedTempData;
@@ -82,17 +83,42 @@ public final class ZipArchiveFakeEntry extends ZipArchiveEntry implements Closea
         if (threshold >= 0 && (entrySize >= threshold || entrySize == -1)) {
             boolean success = false;
             try {
-                if (ZipInputStreamZipEntrySource.shouldEncryptTempFiles()) {
-                    encryptedTempData = new EncryptedTempData();
-                    try (OutputStream os = encryptedTempData.getOutputStream()) {
-                        numberOfBytes = IOUtils.copy(inp, os);
+                final long bytes;
+                if (entrySize == -1) {
+                    // The entry does not declare its uncompressed size (e.g. it was written
+                    // with a data descriptor by a streaming zip writer). Most such entries
+                    // are small, so buffer in memory up to the temp-file threshold and only
+                    // spill to a temp file if the entry really is that large - an entry that
+                    // hides its size cannot force more than the threshold onto the heap, and
+                    // small entries no longer cost a temp file each.
+                    try (UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) {
+                        final long bytesInMemory = IOUtils.copy(inp, baos, threshold);
+                        final int nextByte = bytesInMemory < threshold ? -1 : inp.read();
+                        if (nextByte == -1) {
+                            data = baos.toByteArray();
+                            bytes = data.length;
+                        } else {
+                            try (OutputStream os = createTempDataOutputStream()) {
+                                LOG.atWarn().log("Zip entry {} does not declare its uncompressed size and is larger " +
+                                                "than the temp-file threshold of {} bytes - spilling it to {}",
+                                        entry.getName(), threshold,
+                                        tempFile != null ? tempFile.getAbsolutePath() : "encrypted temp data");
+                                baos.writeTo(os);
+                                os.write(nextByte);
+                                bytes = bytesInMemory + 1 + IOUtils.copy(inp, os);
+                            }
+                        }
                     }
                 } else {
-                    tempFile = TempFile.createTempFile("poi-zip-entry", ".tmp");
-                    LOG.atInfo().log("Creating temp file {} for zip entry {} of size {} bytes",
-                            tempFile.getAbsolutePath(), entry.getName(), entrySize);
-                    numberOfBytes = IOUtils.copy(inp, tempFile);
+                    try (OutputStream os = createTempDataOutputStream()) {
+                        if (tempFile != null) {
+                            LOG.atInfo().log("Creating temp file {} for zip entry {} of size {} bytes",
+                                    tempFile.getAbsolutePath(), entry.getName(), entrySize);
+                        }
+                        bytes = IOUtils.copy(inp, os);
+                    }
                 }
+                numberOfBytes = bytes;
                 success = true;
             } finally {
                 if (!success) {
@@ -131,6 +157,21 @@ public final class ZipArchiveFakeEntry extends ZipArchiveEntry implements Closea
         }
     }
 
+    /**
+     * Opens the output to buffer this entry's data outside the heap: encrypted temp data if
+     * {@link ZipInputStreamZipEntrySource#setEncryptTempFiles(boolean)} is enabled, a plain
+     * temp file otherwise. Sets the corresponding field so {@link #getInputStream()} and
+     * {@link #close()} can find it.
+     */
+    private OutputStream createTempDataOutputStream() throws IOException {
+        if (ZipInputStreamZipEntrySource.shouldEncryptTempFiles()) {
+            encryptedTempData = new EncryptedTempData();
+            return encryptedTempData.getOutputStream();
+        }
+        tempFile = TempFile.createTempFile("poi-zip-entry", ".tmp");
+        return Files.newOutputStream(tempFile.toPath());
+    }
+
     @Override
     public long getSize() {
         return numberOfBytes;
diff --git a/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java b/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
index a3a8e6bddf..68bbf2f342 100644
--- a/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
+++ b/poi-ooxml/src/test/java/org/apache/poi/openxml4j/util/TestZipArchiveFakeEntry.java
@@ -30,6 +30,7 @@ import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -135,6 +136,73 @@ public class TestZipArchiveFakeEntry {
         }
     }
 
+    @Test
+    void testUnknownSizeBelowThresholdStaysInMemory() throws IOException {
+        // an entry that does not declare its size (getSize() == -1) is buffered in memory
+        // as long as it stays below the temp-file threshold - no temp file is created
+        ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(1000);
+        try {
+            ZipArchiveEntry entry = new ZipArchiveEntry("hack123");
+            UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get();
+            final byte[] data = "fakeValue".getBytes(StandardCharsets.UTF_8);
+            baos.write(data);
+            try (ZipArchiveFakeEntry zipArchiveFakeEntry = new ZipArchiveFakeEntry(entry, baos.toInputStream())) {
+                assertFalse(zipArchiveFakeEntry.isUnencryptedTempFileBacked());
+                assertFalse(zipArchiveFakeEntry.isEncryptedTempFileBacked());
+                assertEquals(data.length, zipArchiveFakeEntry.getSize());
+                UnsynchronizedByteArrayOutputStream baos2 = UnsynchronizedByteArrayOutputStream.builder().get();
+                try (InputStream stream = zipArchiveFakeEntry.getInputStream()) {
+                    IOUtils.copy(stream, baos2);
+                }
+                assertEquals("fakeValue", baos2.toString(StandardCharsets.UTF_8.name()));
+            }
+        } finally {
+            ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+        }
+    }
+
+    @Test
+    void testUnknownSizeAboveThresholdSpillsToTempFile() throws IOException {
+        // an entry that does not declare its size and turns out to be larger than the
+        // temp-file threshold spills to a temp file with no data lost
+        ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(4);
+        try {
+            runTestSpill(true, false);
+        } finally {
+            ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+        }
+    }
+
+    @Test
+    void testUnknownSizeAboveThresholdSpillsToEncryptedTempFile() throws IOException {
+        ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(4);
+        ZipInputStreamZipEntrySource.setEncryptTempFiles(true);
+        try {
+            runTestSpill(false, true);
+        } finally {
+            ZipInputStreamZipEntrySource.setThresholdBytesForTempFiles(-1);
+            ZipInputStreamZipEntrySource.setEncryptTempFiles(false);
+        }
+    }
+
+    private static void runTestSpill(boolean unencryptedTempFileExpected,
+                                     boolean encryptedTempFileExpected) throws IOException {
+        ZipArchiveEntry entry = new ZipArchiveEntry("hack123");
+        UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get();
+        final byte[] data = "fakeValue".getBytes(StandardCharsets.UTF_8);
+        baos.write(data);
+        try (ZipArchiveFakeEntry zipArchiveFakeEntry = new ZipArchiveFakeEntry(entry, baos.toInputStream())) {
+            assertEquals(unencryptedTempFileExpected, zipArchiveFakeEntry.isUnencryptedTempFileBacked());
+            assertEquals(encryptedTempFileExpected, zipArchiveFakeEntry.isEncryptedTempFileBacked());
+            assertEquals(data.length, zipArchiveFakeEntry.getSize());
+            UnsynchronizedByteArrayOutputStream baos2 = UnsynchronizedByteArrayOutputStream.builder().get();
+            try (InputStream stream = zipArchiveFakeEntry.getInputStream()) {
+                assertEquals(data.length, IOUtils.copy(stream, baos2));
+            }
+            assertEquals("fakeValue", baos2.toString(StandardCharsets.UTF_8.name()));
+        }
+    }
+
     static void runTestSize(boolean unencryptedTempFileExpected,
                             boolean encryptedTempFileExpected) throws IOException {
         final String fakeEntryName = "hack123";


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@poi.apache.org
For additional commands, e-mail: commits-help@poi.apache.org