You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/08/26 22:03:04 UTC
(tika) branch TIKA-4835-spill-sites created (now a2eabc6943)
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a change to branch TIKA-4835-spill-sites
in repository https://gitbox.apache.org/repos/asf/tika.git
at a2eabc6943 Merge remote-tracking branch 'origin/main' into TIKA-4835-spill-sites
This branch includes the following new commits:
new 58e33ebfa3 TIKA-4835 -- read in-memory input in place instead of spooling it
new 55156aa176 TIKA-4835 -- merge primitives review fixes; POIFS budget accounting; ImageIO
new 4a687bbe54 TIKA-4835 -- merge primitives; review fixes; docs for the channel/view contract
new a2eabc6943 Merge remote-tracking branch 'origin/main' into TIKA-4835-spill-sites
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tika) 03/04: TIKA-4835 -- merge primitives; review fixes; docs for the channel/view contract
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4835-spill-sites
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 4a687bbe54bc9c55d7078e4e5486034b5dc24986
Merge: 55156aa176 3235a919f4
Author: tallison <ta...@apache.org>
AuthorDate: Wed Aug 26 16:42:42 2026 -0400
TIKA-4835 -- merge primitives; review fixes; docs for the channel/view contract
CHANGES.txt | 50 ++++++---
docs/modules/ROOT/pages/advanced/spooling.adoc | 30 +++++-
docs/modules/ROOT/pages/configuration/index.adoc | 5 +-
.../ROOT/pages/configuration/metadata-filters.adoc | 6 +-
.../pages/configuration/parsers/tess4j-parser.adoc | 5 +-
.../ROOT/pages/developers/serialization.adoc | 6 +-
.../pages/maintainers/release-guides/tika.adoc | 18 ++--
docs/modules/ROOT/pages/migration-to-4x/index.adoc | 4 +-
.../pages/migration-to-4x/metadata-changes-4x.adoc | 12 ++-
.../pages/migration-to-4x/migrating-to-4x.adoc | 9 +-
docs/modules/ROOT/pages/pipes/performance.adoc | 5 +-
.../ROOT/pages/pipes/plugins/elasticsearch.adoc | 4 +-
docs/modules/ROOT/pages/pipes/plugins/jdbc.adoc | 2 +-
docs/modules/ROOT/pages/pipes/plugins/kafka.adoc | 4 +-
.../ROOT/pages/pipes/plugins/opensearch.adoc | 4 +-
docs/modules/ROOT/pages/pipes/plugins/solr.adoc | 6 +-
.../ROOT/pages/pipes/plugins/writing-a-plugin.adoc | 12 ++-
docs/modules/ROOT/pages/pipes/unpack-config.adoc | 2 +-
docs/modules/ROOT/pages/using-tika/docker.adoc | 4 +-
.../src/main/java/org/apache/tika/gui/TikaGUI.java | 4 +-
.../apache/tika/digest/BufferingDigestSink.java | 34 ++++--
.../org/apache/tika/digest/CompositeDigester.java | 22 ++--
.../java/org/apache/tika/digest/DigestHelper.java | 36 +++++--
.../java/org/apache/tika/digest/DigestSink.java | 45 +++++---
.../main/java/org/apache/tika/digest/Digester.java | 12 +--
.../java/org/apache/tika/io/ByteArraySource.java | 12 ++-
.../org/apache/tika/io/CachingInputStream.java | 9 ++
.../java/org/apache/tika/io/CachingSource.java | 10 ++
.../main/java/org/apache/tika/io/FileSource.java | 5 +
.../java/org/apache/tika/io/ReopenableSource.java | 5 +
.../main/java/org/apache/tika/io/StreamCache.java | 5 +
.../java/org/apache/tika/io/TikaInputSource.java | 9 ++
.../java/org/apache/tika/io/TikaInputStream.java | 23 ++--
.../org/apache/tika/digest/DigestHelperTest.java | 26 +++++
.../org/apache/tika/digest/DigestSinkTest.java | 119 +++++++++++++++++----
.../apache/tika/digest/FailingTestTranslator.java | 13 ++-
.../apache/tika/io/InMemoryContentViewTest.java | 74 +++++++++++++
.../org/apache/tika/metadata/TestMetadata.java | 119 +++++++++++++++++++++
....apache.tika.extractor.EmbeddedStreamTranslator | 14 +++
tika-ml/tika-ml-junkdetect-tools/pom.xml | 5 +-
.../ml/junkdetect/JunkFilterEncodingDetector.java | 11 +-
.../tika/parser/ocr/tess4j/Tess4JConfig.java | 7 +-
.../tika/parser/image/AbstractImageParser.java | 6 +-
.../org/apache/tika/parser/image/ImageParser.java | 8 +-
.../parser/image/ImageParsersNoTempFileTest.java | 30 ------
.../detect/microsoft/POIFSContainerDetector.java | 25 ++++-
.../detect/microsoft/POIFSDeclaredSizeTest.java | 29 +++--
.../apache/tika/parser/ocr/ImagePreprocessor.java | 2 +-
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 4 -
.../java/org/apache/tika/parser/pdf/PDFParser.java | 6 +-
.../apache/tika/parser/pdf/PDFRandomAccess.java | 23 +++-
.../tika/renderer/pdf/pdfbox/PDFBoxRenderer.java | 17 ++-
.../apache/tika/pipes/core/FontCacheWarmer.java | 55 ++++++++++
.../tika/pipes/core/SharedServerModeTest.java | 2 +
.../pipes/core/async/AsyncChaosMonkeyTest.java | 2 +
.../org.junit.jupiter.api.extension.Extension | 15 +++
.../resources/configs/tika-config-passback.json | 2 +-
.../resources/configs/tika-config-truncate.json | 61 -----------
.../resources/configs/tika-config-uppercasing.json | 2 +-
.../configs/tika-config-write-limiter.json | 2 +-
.../src/test/resources/junit-platform.properties | 17 +++
.../atlassianjwt/AtlassianJwtFetcherFactory.java | 4 +-
.../tika/pipes/emitter/azblob/AZBlobEmitter.java | 4 +-
.../pipes/emitter/azblob/AZBlobEmitterFactory.java | 4 +-
.../pipes/fetcher/azblob/AZBlobFetcherFactory.java | 4 +-
.../tika/pipes/emitter/es/ESEmitterFactory.java | 4 +-
.../pipes/emitter/fs/FileSystemEmitterFactory.java | 4 +-
.../pipes/fetcher/fs/FileSystemFetcherFactory.java | 4 +-
.../apache/tika/pipes/emitter/gcs/GCSEmitter.java | 4 +-
.../tika/pipes/emitter/gcs/GCSEmitterFactory.java | 4 +-
.../tika/pipes/fetcher/gcs/GCSFetcherFactory.java | 4 +-
.../googledrive/GoogleDriveFetcherFactory.java | 4 +-
.../pipes/fetcher/http/HttpFetcherFactory.java | 4 +-
.../tika/pipes/emitter/jdbc/JDBCEmitter.java | 4 +-
.../pipes/emitter/jdbc/JDBCEmitterFactory.java | 4 +-
.../tika/pipes/emitter/kafka/KafkaEmitter.java | 4 +-
.../pipes/emitter/kafka/KafkaEmitterFactory.java | 4 +-
.../MicrosoftGraphFetcherFactory.java | 4 +-
.../opensearch/OpenSearchEmitterFactory.java | 4 +-
.../apache/tika/pipes/emitter/s3/S3Emitter.java | 4 +-
.../tika/pipes/emitter/s3/S3EmitterFactory.java | 4 +-
.../tika/pipes/fetcher/s3/S3FetcherFactory.java | 4 +-
.../tika/pipes/emitter/solr/SolrEmitter.java | 4 +-
.../pipes/emitter/solr/SolrEmitterFactory.java | 4 +-
.../server/core/TikaServerIntegrationTest.java | 4 +-
85 files changed, 887 insertions(+), 324 deletions(-)
diff --cc CHANGES.txt
index 4f886b4ba2,5ae3270114..e22631edfa
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,32 -1,5 +1,27 @@@
Release 4.1.0 - unreleased
+ * Parsers and detectors no longer spool in-memory input to a temp file just
+ to read it back with random access: the JPEG, TIFF and WebP parsers, the
+ OpenDocument parser's inline pictures, the OLE2 container detector's
+ entry-name read, and PDFParser (the incremental-update scan and the main
+ load). Together these were the bulk of the temp bytes behind 4.0.0's
+ batch slowdown on spinning disks (see docs/.../pipes/performance.adoc).
+ One rule everywhere: content goes through the stream cache (bounded by
+ the pipes cache memory budget, 1MB per object without one); what stays
+ in memory is read in place through a zero-copy view, what spills is
+ read from the file. PDFBox and metadata-extractor no longer receive a
+ second heap copy of the document. The PDF renderer now re-opens the
+ document from byte 0 for every render, which fixes per-page rendering
+ (RENDER_PAGES_AT_PAGE_END) silently producing only the first page --
- for file-backed input too. The ImageIO-based ImageParser (PNG, GIF, BMP, ...)
- reads image headers from memory instead of through ImageIO's default
- file cache, which created a temp file per image and wrote every byte it
- read to it (600k writes on a 20k-file sample; the page cache absorbed most
- of them before they reached disk, so this is syscall and file-churn
- savings more than disk bytes). PDFParser.getPDDocumentFromStream is
++ for file-backed input too. PDFParser.getPDDocumentFromStream is
+ deprecated and no longer called; override the new
+ getPDDocument(RandomAccessRead, ...) instead. ImageMetadataExtractor
+ gains TikaInputStream and InputStream overloads. One visible metadata
+ change: JPEG/TIFF/WebP images read from memory no longer carry
+ metadata-extractor's file-system tags (img:File Name, img:File Size,
+ img:File Modified Date), which described whatever file was read rather
+ than the image; input backed by a file still carries them (TIKA-4835).
+
* tika-pipes: the cache memory budget (how much rewindable content a forked
worker keeps in memory before spilling to disk; new since 4.0.0, which had
no budget at all) defaults to a quarter of the fork's heap, so raising
diff --cc docs/modules/ROOT/pages/advanced/spooling.adoc
index d3fc2d67ba,d3fc2d67ba..c66a4b191f
--- a/docs/modules/ROOT/pages/advanced/spooling.adoc
+++ b/docs/modules/ROOT/pages/advanced/spooling.adoc
@@@ -49,16 -49,16 +49,36 @@@ Several file formats are most efficient
The current architecture follows a simple principle: **each component that needs random
access is responsible for obtaining it**.
--A detector or parser is handed a `TikaInputStream`; when it needs random access
--it simply asks for a file:
++A detector or parser is handed a `TikaInputStream`; when it needs random access it asks for
++a seekable view of the content and lets `TikaInputStream` decide whether that view is backed
++by memory or by a file:
[source,java]
----
--Path path = tis.getPath();
--// or
--File file = tis.getFile();
++try (SeekableByteChannel channel = tis.getSeekableByteChannel()) {
++ ByteBuffer view = TikaInputStream.inMemoryContent(channel);
++ if (view != null) {
++ readInPlace(view); // in memory: no copy, no temp file
++ } else {
++ readFromFile(tis.getFile()); // spilled: the file already exists
++ }
++}
----
++Two rules go with that form:
++
++* The `ByteBuffer` aliases the cache's own array and is valid *only while that channel is
++ open*. Keep the channel open for as long as the view is in use. A view that outlives its
++ channel still reads correctly but is no longer counted against the memory budget.
++* `inMemoryContent()` returns `null` whenever the content is not in memory -- file-backed
++ input, or a cache that spilled. That is the normal case for large content, not an error.
++
++`getFile()` remains correct when a consumer genuinely requires a `java.io.File` -- many
++third-party libraries do -- but it always materialises one. Prefer the channel/view form when
++the consumer can accept a buffer or a stream. For a worked example see
++`PDFRandomAccess.open(...)` in `tika-parser-pdf-module`, one factory that makes this choice
++for the PDF parser, its incremental-update scan, and the renderer.
++
`TikaInputStream` handles the spooling transparently based on how it was initialized:
* **Initialized with `Path`**: The file is used directly for random access. No spooling needed.
diff --cc docs/modules/ROOT/pages/pipes/performance.adoc
index f35296d633,f35296d633..8f40cb5168
--- a/docs/modules/ROOT/pages/pipes/performance.adoc
+++ b/docs/modules/ROOT/pages/pipes/performance.adoc
@@@ -242,8 -242,8 +242,9 @@@ architecture suggested it would be
* Several parsers and detectors asked for a `java.io.File` even when the
document was already in memory: the JPEG/TIFF/WebP metadata extractors, the
OLE2 container detector, the OpenDocument parser's inline pictures, the
-- digest of translated embedded streams, and the PDF incremental-update scan
-- each wrote the bytes out just to read them back.
++ digest of translated embedded streams, the PDF incremental-update scan, and
++ PDFParser's main document load and renderer each wrote the bytes out just to
++ read them back.
On a spinning-disk host where the temp directory, the corpus, and the outputs
share spindles, every temp byte is a seek taken away from a corpus read or an
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/AbstractImageParser.java
index 4ae61217c4,4ae61217c4..954649b2b0
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/AbstractImageParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/AbstractImageParser.java
@@@ -18,7 -18,7 +18,6 @@@ package org.apache.tika.parser.image
import java.io.IOException;
import java.io.InputStream;
--import java.nio.file.Files;
import java.nio.file.Path;
import org.xml.sax.ContentHandler;
@@@ -89,7 -89,7 +88,10 @@@ public abstract class AbstractImagePars
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context);
xhtml.startDocument();
Path path = tis.getPath();
-- try (InputStream pathStream = Files.newInputStream(path)) {
++ // a TikaInputStream over the path, not a raw stream: the content is already on
++ // disk, so this takes the file path in extractMetadata instead of caching a
++ // second copy in memory whose budget reservation nothing here would release
++ try (TikaInputStream pathStream = TikaInputStream.get(path)) {
extractMetadata(pathStream, new EmbeddedContentHandler(xhtml), metadata, context);
} catch (SecurityException e) {
throw e;
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java
index 095d885b84,0000000000..9f970d28b5
mode 100644,000000..100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java
@@@ -1,173 -1,0 +1,143 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
- import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.io.ByteArrayInputStream;
- import java.io.File;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
- import java.nio.file.attribute.PosixFilePermissions;
+import java.util.stream.Stream;
- import javax.imageio.ImageIO;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TIFF;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+
+/**
+ * Embedded images usually arrive already in memory; the image parsers must not spool them to
+ * disk just to read metadata. The file-system tags are the visible consequence: they describe
+ * whatever file was read, so in-memory input must not carry them and real files must.
+ */
+public class ImageParsersNoTempFileTest extends TikaTest {
+
+ private static final String FILE_NAME_KEY = ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Name";
+ private static final String FILE_SIZE_KEY = ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Size";
+ private static final String FILE_MODIFIED_KEY =
+ ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Modified Date";
+
+ @TempDir
+ Path tempDir;
+
+ private static final String JPEG = "/test-documents/testJPEG_EXIF.jpg";
+ private static final String TIFF_RES = "/test-documents/testTIFF.tif";
+ private static final String WEBP = "/test-documents/testWebp_Alpha_Lossless.webp";
+ private static final String WEBP_WIDTH = ImageMetadataExtractor.UNKNOWN_IMG_NS + "Image Width";
- private static final String PNG = "/test-documents/testPNG.png";
+
+ private static ParseContext context() {
+ ParseContext context = new ParseContext();
+ context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
+ return context;
+ }
+
+ private byte[] bytes(String resource) throws Exception {
+ try (InputStream is = getResourceAsStream(resource)) {
+ return is.readAllBytes();
+ }
+ }
+
+ @Test
+ public void testJpegInMemory() throws Exception {
+ assertNotSpooled(new JpegParser(), JPEG, TIFF.IMAGE_WIDTH.getName());
+ }
+
+ @Test
+ public void testTiffInMemory() throws Exception {
+ assertNotSpooled(new TiffParser(), TIFF_RES, TIFF.IMAGE_WIDTH.getName());
+ }
+
+ @Test
+ public void testWebPInMemory() throws Exception {
+ assertNotSpooled(new WebPParser(), WEBP, WEBP_WIDTH);
+ }
+
- /**
- * ImageIO's own file cache is outside TemporaryResources entirely, and its cache file is
- * gone by the time a parse returns -- so the watched-directory check cannot see it. Make
- * the cache directory unwritable instead: a parser that asks ImageIO for a file-backed
- * stream then fails with "Can't create cache file"; one that reads from memory never
- * notices.
- */
- @Test
- public void testImageIoParserUsesNoFileCache() throws Exception {
- assumeTrue(Files.getFileStore(tempDir).supportsFileAttributeView("posix"), "needs POSIX permissions");
- Path cache = Files.createDirectory(tempDir.resolve("imageio-cache"));
- File before = ImageIO.getCacheDirectory();
- boolean useCache = ImageIO.getUseCache();
- ImageIO.setUseCache(true);
- ImageIO.setCacheDirectory(cache.toFile());
- Files.setPosixFilePermissions(cache, PosixFilePermissions.fromString("r-xr-xr-x"));
- try {
- assertNotSpooled(new ImageParser(), PNG, TIFF.IMAGE_WIDTH.getName());
- } finally {
- Files.setPosixFilePermissions(cache, PosixFilePermissions.fromString("rwxr-xr-x"));
- ImageIO.setCacheDirectory(before);
- ImageIO.setUseCache(useCache);
- }
- }
-
+ @Test
+ public void testJpegFromFile() throws Exception {
+ assertKeepsFileTags(new JpegParser(), JPEG, TIFF.IMAGE_WIDTH.getName(), "jpg");
+ }
+
+ @Test
+ public void testTiffFromFile() throws Exception {
+ assertKeepsFileTags(new TiffParser(), TIFF_RES, TIFF.IMAGE_WIDTH.getName(), "tif");
+ }
+
+ @Test
+ public void testWebPFromFile() throws Exception {
+ assertKeepsFileTags(new WebPParser(), WEBP, WEBP_WIDTH, "webp");
+ }
+
+ private void assertNotSpooled(Parser parser, String resource, String widthKey)
+ throws Exception {
+ byte[] bytes = bytes(resource);
+ Metadata metadata = new Metadata();
+ if (resource.endsWith(".png")) {
+ metadata.set(HttpHeaders.CONTENT_TYPE, "image/png");
+ }
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ tmp.setTemporaryFileDirectory(tempDir);
+ // a stream-backed, non-file TikaInputStream whose only spill target is tempDir
+ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
+ parser.parse(tis, new DefaultHandler(), metadata, context());
+ // temp files live until tmp closes, so any spool would be visible right here
+ try (Stream<Path> files = Files.list(tempDir)) {
+ assertEquals(0, files.filter(Files::isRegularFile).count(),
+ "parser spooled an in-memory image to disk");
+ }
+ }
+ assertNotNull(metadata.get(widthKey), "metadata was extracted");
+ assertNull(metadata.get(FILE_NAME_KEY), "in-memory input must not carry file tags");
+ assertNull(metadata.get(FILE_SIZE_KEY), "in-memory input must not carry file tags");
+ assertNull(metadata.get(FILE_MODIFIED_KEY), "in-memory input must not carry file tags");
+ }
+
+ private void assertKeepsFileTags(Parser parser, String resource, String widthKey, String ext)
+ throws Exception {
+ byte[] bytes = bytes(resource);
+ Path image = tempDir.resolve("image." + ext);
+ Files.write(image, bytes);
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(image, metadata)) {
+ parser.parse(tis, new DefaultHandler(), metadata, context());
+ }
+ assertNotNull(metadata.get(widthKey), "metadata was extracted");
+ assertEquals(image.getFileName().toString(), metadata.get(FILE_NAME_KEY),
+ "a real file keeps metadata-extractor's file-system tags");
+ assertNotNull(metadata.get(FILE_SIZE_KEY), "a real file keeps its size tag");
+ }
+}
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
index a0500ae985,356dcba2f1..cb046e3780
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
@@@ -597,107 -583,6 +597,124 @@@ public class POIFSContainerDetector imp
}
}
+ /**
+ * Opens the container from memory. POI's stream loader copies the object into its own
+ * heap array, sized from the header rather than the content, and keeps it for the
+ * container's lifetime -- so that allocation is reserved from the CacheMemoryBudget
+ * before it is made. Returns null, and the caller uses the file, when the content is not
+ * in memory, the budget has no room for the copy, or POI's stream loader rejects the
- * object (it is stricter than the file loader on truncated objects). Without a budget
- * only objects under the cache's threshold have an in-memory view, which bounds the copy.
++ * object (it is stricter than the file loader on truncated objects).
++ * <p>
++ * With no budget in the context there is no accounting at all, and a byte[]-backed
++ * stream has no spill threshold to bound it either, so the copy is capped outright:
++ * above {@link #MAX_UNBUDGETED_COPY} the file path -- which is what 4.0.0 always did --
++ * is used instead.
+ */
++ /**
++ * Ceiling on POI's in-memory copy when no CacheMemoryBudget is present to account for it.
++ * Embedded OLE2 objects are typically a few hundred KB; past this the file loader's lazy
++ * block reads cost less than the heap.
++ */
++ private static final long MAX_UNBUDGETED_COPY = 16L * 1024 * 1024;
++
+ private Set<String> getTopLevelNamesInMemory(TikaInputStream stream, ParseContext context)
+ throws IOException {
+ CacheMemoryBudget budget = context == null ? null : context.get(CacheMemoryBudget.class);
+ // the budget decides how much the drain below keeps in memory
+ stream.enableRewind(budget);
+ POIFSFileSystem fs = null;
+ long reserved = 0;
+ try (SeekableByteChannel channel = stream.getSeekableByteChannel()) {
+ if (TikaInputStream.inMemoryContent(channel) == null) {
+ return null;
+ }
+ // POI allocates what the header declares. The bytes in hand are the truth: a
+ // header that declares more than they can account for is lying, and the file
+ // loader (block by block, no such allocation) is the only safe way to read it.
+ long copy = honestDeclaredSize(channel);
+ if (copy < 0) {
+ return null;
+ }
- if (budget != null) {
++ if (budget == null) {
++ if (copy > MAX_UNBUDGETED_COPY) {
++ return null;
++ }
++ } else {
+ if (budget.tryReserve(copy) == 0) {
+ return null;
+ }
+ reserved = copy;
+ }
++ // POI reads from wherever the channel sits; do not rely on it being fresh
++ channel.position(0);
+ fs = new POIFSFileSystem(Channels.newInputStream(channel));
+ Set<String> names = getTopLevelNames(fs.getRoot());
+ stream.setOpenContainer(fs);
++ fs = null; // published: the stream owns it now, the finally must not close it
+ if (reserved > 0) {
+ long charged = reserved;
+ stream.addCloseableResource(() -> budget.release(charged));
+ reserved = 0;
+ }
- fs = null;
+ return names;
+ } catch (SecurityException e) {
+ throw e;
+ } catch (IOException | RuntimeException e) {
+ return null;
+ } finally {
+ if (reserved > 0) {
+ budget.release(reserved);
+ }
+ if (fs != null) {
+ closeQuietly(fs);
+ }
+ }
+ }
+
+ /**
+ * The heap POI's stream loader would allocate for this object -- sized from the header's
+ * declared BAT count, not the content -- or -1 when the header cannot be read or declares
+ * more than the content can account for. A valid header covers at most one BAT block of
+ * unused entries beyond the actual size; anything past that is a malformed or hostile
+ * header (a 512-byte object can declare hundreds of MB) and must not be opened from a
+ * stream at all.
+ */
+ static long honestDeclaredSize(SeekableByteChannel channel) throws IOException {
+ byte[] header = new byte[POIFSConstants.SMALLER_BIG_BLOCK_SIZE];
+ long start = channel.position();
+ try {
+ channel.position(0);
+ ByteBuffer buffer = ByteBuffer.wrap(header);
+ while (buffer.hasRemaining() && channel.read(buffer) > 0) {
+ // fill the header block
+ }
+ if (buffer.hasRemaining()) {
+ return -1;
+ }
+ } finally {
+ channel.position(start);
+ }
+ try {
+ HeaderBlock hb = new HeaderBlock(
+ UnsynchronizedByteArrayInputStream.builder().setByteArray(header).get());
+ long declared = BATBlock.calculateMaximumSize(hb);
+ long oneBatSpan = (long) hb.getBigBlockSize().getBigBlockSize() *
+ hb.getBigBlockSize().getBATEntriesPerBlock();
+ long actual = channel.size();
+ return declared > actual + oneBatSpan ? -1 : declared;
+ } catch (IOException | RuntimeException e) {
+ return -1;
+ }
+ }
+
+ private static void closeQuietly(POIFSFileSystem fs) {
+ try {
+ fs.close();
+ } catch (IOException e) {
+ LOG.debug("failed to close abandoned POIFSFileSystem", e);
+ }
+ }
+
public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException {
// Check if we have access to the document
if (tis == null) {
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java
index 6baedcb2f7,0000000000..8450153bec
mode 100644,000000..100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java
@@@ -1,137 -1,0 +1,154 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect.microsoft;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
++import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * POI sizes its in-memory OLE2 buffer from the header's declared BAT count rather than the
+ * actual length, so a 512-byte object can demand hundreds of MB. The in-memory detection path
+ * only believes a header the bytes in hand can account for, and reserves that from the
+ * budget before POI allocates it.
+ */
+public class POIFSDeclaredSizeTest extends TikaTest {
+
+ private static final int BAT_COUNT_OFFSET = 0x2C;
+ private static final int SECTOR_SHIFT_OFFSET = 0x1E;
+
+ @TempDir
+ Path tempDir;
+
+ /** A bare 512-byte OLE2 header declaring {@code batCount} BAT blocks and nothing else. */
+ private static byte[] header(int batCount) {
+ byte[] data = new byte[512];
+ byte[] magic = {(byte) 0xd0, (byte) 0xcf, 0x11, (byte) 0xe0,
+ (byte) 0xa1, (byte) 0xb1, 0x1a, (byte) 0xe1};
+ System.arraycopy(magic, 0, data, 0, magic.length);
+ data[SECTOR_SHIFT_OFFSET] = 9; // 2^9 = 512-byte blocks
+ ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).putInt(BAT_COUNT_OFFSET, batCount);
+ return data;
+ }
+
+ private SeekableByteChannel channelFor(byte[] bytes, String name) throws Exception {
+ Path p = tempDir.resolve(name);
+ Files.write(p, bytes);
+ return Files.newByteChannel(p, StandardOpenOption.READ);
+ }
+
+ @Test
+ public void testLyingHeaderIsRejected() throws Exception {
+ // (1 + 3813 * 128) * 512 == 249_889_280, just under POI's 250MB allocation ceiling,
+ // declared by 512 bytes of content
+ try (SeekableByteChannel channel = channelFor(header(3813), "hostile.ole")) {
+ assertEquals(-1, POIFSContainerDetector.honestDeclaredSize(channel),
+ "the content cannot account for what the header declares");
+ assertEquals(0, channel.position(), "the size probe must not move the channel");
+ }
+ }
+
+ @Test
+ public void testHonestHeaderIsBelievedWithinOneBatBlock() throws Exception {
+ // header only, declaring one BAT block: 129 sectors, 512 bytes present -- within slack
+ try (SeekableByteChannel channel = channelFor(header(1), "modest.ole")) {
+ assertEquals((1 + 128) * 512L, POIFSContainerDetector.honestDeclaredSize(channel));
+ }
+ // two BAT blocks declared by 512 bytes: one block past what the content covers
+ try (SeekableByteChannel channel = channelFor(header(2), "twoblocks.ole")) {
+ assertEquals(-1, POIFSContainerDetector.honestDeclaredSize(channel));
+ }
+ }
+
+ @Test
+ public void testRealDocumentHeaderIsHonest() throws Exception {
+ byte[] bytes;
+ try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) {
+ bytes = is.readAllBytes();
+ }
+ try (SeekableByteChannel channel = channelFor(bytes, "real.doc")) {
+ long declared = POIFSContainerDetector.honestDeclaredSize(channel);
+ assertTrue(declared >= bytes.length && declared <= bytes.length + 128 * 512L,
+ "a real header declares about its own size: " + declared + " vs " + bytes.length);
+ }
+ }
+
+ @Test
+ public void testTooShortForAHeaderIsRejected() throws Exception {
+ try (SeekableByteChannel channel = channelFor(new byte[16], "short.bin")) {
+ assertEquals(-1, POIFSContainerDetector.honestDeclaredSize(channel));
+ }
+ }
+
+ /**
- * End to end: the crafted object is refused before any reservation, so POI never
- * allocates its declared size and no POIFSFileSystem is retained -- even with a budget
- * large enough to have said yes.
++ * End to end. NOTE: this asserts only that the crafted object does not become an open
++ * container and leaves nothing charged -- both of which also hold if the declared-size
++ * guard is deleted, because POI throws on the truncated read either way. The guard's
++ * real effect is the ~238MB POI would allocate first, and measuring that needs
++ * com.sun.management, which forbidden-apis bans. The guard's arithmetic and its input
++ * channel type are pinned by the unit tests above instead.
+ */
+ @Test
+ public void testHostileHeaderDoesNotBecomeAnOpenContainer() throws Exception {
+ ParseContext context = new ParseContext();
+ CacheMemoryBudget budget = new CacheMemoryBudget(1024L * 1024 * 1024);
+ context.set(CacheMemoryBudget.class, budget);
+ Metadata metadata = new Metadata();
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ TikaInputStream tis = TikaInputStream.get(
+ new ByteArrayInputStream(header(3813)), tmp, metadata);
- POIFSContainerDetector detector = new POIFSContainerDetector();
- assertTrue(detector.detect(tis, metadata, context) != null);
++ new POIFSContainerDetector().detect(tis, metadata, context);
+ assertNull(tis.getOpenContainer(),
+ "a header-only object must not be opened from memory");
+ }
- assertEquals(0, budget.getReservedBytes(), "nothing was ever reserved for it");
++ assertEquals(0, budget.getReservedBytes(), "nothing left charged");
++ }
++
++ /** The channel type production actually uses is in-memory, not a file. */
++ @Test
++ public void testLyingHeaderIsRejectedOverAnInMemoryChannel() throws Exception {
++ try (TemporaryResources tmp = new TemporaryResources()) {
++ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(header(3813)), tmp,
++ new Metadata());
++ tis.enableRewind(null);
++ try (SeekableByteChannel channel = tis.getSeekableByteChannel()) {
++ assertNotNull(TikaInputStream.inMemoryContent(channel), "precondition: in memory");
++ assertEquals(-1, POIFSContainerDetector.honestDeclaredSize(channel));
++ }
++ }
+ }
+}
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index bfdf538101,c61f33cc45..9267506822
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@@ -494,18 -495,21 +494,22 @@@ public class PDFParser implements Parse
ParseContext context)
throws IOException, EncryptedDocumentException {
try {
- PDDocument pdDocument = null;
if (tis.hasFile()) {
// File based -- send file directly to PDFBox
- pdDocument =
- getPDDocument(tis.getPath(), password, streamCacheCreateFunction, metadata, context);
- } else {
- tis.setCloseShield();
+ return getPDDocument(tis.getPath(), password, streamCacheCreateFunction, metadata, context);
+ }
+ // PDFBox owns the reader and closes it with the document
+ RandomAccessRead ra = PDFRandomAccess.open(tis, context);
+ try {
+ return getPDDocument(ra, password, streamCacheCreateFunction, metadata, context);
+ } catch (IOException | RuntimeException e) {
- ra.close();
+ try {
- pdDocument = getPDDocumentFromStream(tis, password,
- streamCacheCreateFunction, metadata, context);
- } finally {
- tis.removeCloseShield();
++ ra.close();
++ } catch (IOException closeFailure) {
++ e.addSuppressed(closeFailure);
+ }
+ throw e;
}
- return pdDocument;
} catch (IOException e) {
if (e.getMessage() != null &&
e.getMessage().contains("No security handler for filter")) {
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java
index 923f5043d4,0000000000..4db4a092f2
mode 100644,000000..100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java
@@@ -1,66 -1,0 +1,79 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+
+import org.apache.pdfbox.io.RandomAccessRead;
+import org.apache.pdfbox.io.RandomAccessReadBuffer;
+import org.apache.pdfbox.io.RandomAccessReadBufferedFile;
+
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * One way to hand PDFBox a document: in-memory content is read in place through a zero-copy
+ * view, anything else through PDFBox's buffered file reader. Neither copies the document, and
+ * every call starts from byte 0 regardless of the stream's position -- so a renderer can
+ * re-open the same document per page. Closing the returned reader releases the view.
+ */
+public final class PDFRandomAccess {
+
+ private PDFRandomAccess() {
+ }
+
+ public static RandomAccessRead open(TikaInputStream tis, ParseContext context)
+ throws IOException {
+ if (tis.hasFile()) {
+ return new RandomAccessReadBufferedFile(tis.getFile());
+ }
+ tis.enableRewind(context == null ? null : context.get(CacheMemoryBudget.class));
+ SeekableByteChannel channel = tis.getSeekableByteChannel();
- ByteBuffer view = TikaInputStream.inMemoryContent(channel);
- if (view == null) {
- // the drain spilled: the content is on disk now
- channel.close();
- return new RandomAccessReadBufferedFile(tis.getFile());
++ try {
++ ByteBuffer view = TikaInputStream.inMemoryContent(channel);
++ if (view == null) {
++ // the drain spilled: the content is on disk now
++ channel.close();
++ return new RandomAccessReadBufferedFile(tis.getFile());
++ }
++ return wrap(channel, view);
++ } catch (IOException | RuntimeException e) {
++ try {
++ channel.close();
++ } catch (IOException closeFailure) {
++ e.addSuppressed(closeFailure);
++ }
++ throw e;
+ }
++ }
++
++ private static RandomAccessRead wrap(SeekableByteChannel channel, ByteBuffer view) {
+ return new RandomAccessReadBuffer(view) {
+ @Override
+ public void close() throws IOException {
+ try {
+ super.close();
+ } finally {
+ channel.close();
+ }
+ }
+ };
+ }
+}
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
index 20bc2117ef,8c5c94592a..7a421d47d2
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
@@@ -25,6 -25,7 +25,7 @@@ import java.util.Collections
import java.util.Set;
import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.io.RandomAccessReadBuffer;
++import org.apache.pdfbox.io.RandomAccessRead;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
@@@ -100,9 -100,7 +101,21 @@@ public class PDFBoxRenderer implements
if (tis.getOpenContainer() != null) {
pdDocument = (PDDocument) tis.getOpenContainer();
} else {
- pdDocument = Loader.loadPDF(new RandomAccessReadBuffer(tis));
+ // a fresh reader from byte 0 each time, so per-page renders do not depend on
- // where the stream was left; the document closes it
- pdDocument = Loader.loadPDF(PDFRandomAccess.open(tis, parseContext));
++ // where the stream was left; the document closes it -- but only once it exists,
++ // so a failed load must close the reader itself or its channel pin (and the
++ // budget behind it) leaks for the life of the stream
++ RandomAccessRead ra = PDFRandomAccess.open(tis, parseContext);
++ try {
++ pdDocument = Loader.loadPDF(ra);
++ } catch (IOException | RuntimeException e) {
++ try {
++ ra.close();
++ } catch (IOException closeFailure) {
++ e.addSuppressed(closeFailure);
++ }
++ throw e;
++ }
mustClose = true;
}
PageBasedRenderResults results = new PageBasedRenderResults(new TemporaryResources());
(tika) 01/04: TIKA-4835 -- read in-memory input in place instead of spooling it
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4835-spill-sites
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 58e33ebfa3448febb676fcacde54858f85a9eb92
Author: tallison <ta...@apache.org>
AuthorDate: Wed Aug 26 10:06:22 2026 -0400
TIKA-4835 -- read in-memory input in place instead of spooling it
---
CHANGES.txt | 22 ++++
.../tika/parser/microsoft/rtf/RTFParserTest.java | 4 +-
.../apache/tika/parser/image/ByteBufferReader.java | 74 +++++++++++
.../tika/parser/image/ImageMetadataExtractor.java | 80 ++++++++++--
.../org/apache/tika/parser/image/ImageXmp.java | 55 ++++----
.../org/apache/tika/parser/image/JpegParser.java | 8 +-
.../org/apache/tika/parser/image/TiffParser.java | 6 +-
.../org/apache/tika/parser/image/WebPParser.java | 7 +-
.../parser/image/ImageParsersNoTempFileTest.java | 138 +++++++++++++++++++++
.../detect/microsoft/POIFSContainerDetector.java | 108 ++++++++++++++--
.../POIFSContainerDetectorNoTempFileTest.java | 71 +++++++++++
.../detect/microsoft/POIFSDeclaredSizeTest.java | 115 +++++++++++++++++
.../apache/tika/parser/odf/OpenDocumentParser.java | 6 +-
.../odf/OpenDocumentParserNoTempFileTest.java | 104 ++++++++++++++++
.../java/org/apache/tika/parser/pdf/PDFParser.java | 43 ++++---
.../apache/tika/parser/pdf/PDFRandomAccess.java | 66 ++++++++++
.../tika/renderer/pdf/pdfbox/PDFBoxRenderer.java | 6 +-
.../tika/parser/pdf/PDFParserNoTempFileTest.java | 71 +++++++++++
.../tika/parser/pdf/PDFPerPageRenderTest.java | 95 ++++++++++++++
19 files changed, 1007 insertions(+), 72 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 402927e239..2c0ebe5208 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,27 @@
Release 4.1.0 - unreleased
+ * Parsers and detectors no longer spool in-memory input to a temp file just
+ to read it back with random access: the JPEG, TIFF and WebP parsers, the
+ OpenDocument parser's inline pictures, the OLE2 container detector's
+ entry-name read, and PDFParser (the incremental-update scan and the main
+ load). Together these were the bulk of the temp bytes behind 4.0.0's
+ batch slowdown on spinning disks (see docs/.../pipes/performance.adoc).
+ One rule everywhere: content goes through the stream cache (bounded by
+ the pipes cache memory budget, 1MB per object without one); what stays
+ in memory is read in place through a zero-copy view, what spills is
+ read from the file. PDFBox and metadata-extractor no longer receive a
+ second heap copy of the document. The PDF renderer now re-opens the
+ document from byte 0 for every render, which fixes per-page rendering
+ (RENDER_PAGES_AT_PAGE_END) silently producing only the first page --
+ for file-backed input too. PDFParser.getPDDocumentFromStream is
+ deprecated and no longer called; override the new
+ getPDDocument(RandomAccessRead, ...) instead. ImageMetadataExtractor
+ gains TikaInputStream and InputStream overloads. One visible metadata
+ change: JPEG/TIFF/WebP images read from memory no longer carry
+ metadata-extractor's file-system tags (img:File Name, img:File Size,
+ img:File Modified Date), which described whatever file was read rather
+ than the image; input backed by a file still carries them (TIKA-4835).
+
* tika-pipes: the cache memory budget (how much rewindable content a forked
worker keeps in memory before spilling to disk) now defaults to a quarter
of the fork's heap, so raising -Xmx raises it; it was a fixed 256MB. It is
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
index 96ba2b3c7b..9b4fe1b5ea 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
@@ -145,7 +145,9 @@ public class RTFParserTest extends TikaTest {
//need flexibility for if tesseract is installed or not
//TODO -- fix this test. It is too fragile.
- assertTrue(meta_jpg.names().length >= 52 && meta_jpg.names().length <= 60);
+ // in-memory embedded images no longer carry metadata-extractor's temp-file
+ // name/size/date tags (TIKA-4835), hence the lower bound
+ assertTrue(meta_jpg.names().length >= 49 && meta_jpg.names().length <= 60);
assertTrue(meta_jpg_exif.names().length >= 100 && meta_jpg_exif.names().length <= 130);
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ByteBufferReader.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ByteBufferReader.java
new file mode 100644
index 0000000000..43da7a2840
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ByteBufferReader.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import java.nio.ByteBuffer;
+
+import com.drew.lang.BufferBoundsException;
+import com.drew.lang.RandomAccessReader;
+
+/**
+ * drewnoakes random-access reader over a {@link ByteBuffer}, for content already in memory.
+ * The library's own stream reader retains every chunk it reads and its TIFF reader asks for
+ * the length up front, which reads the whole stream; this reads the buffer in place.
+ */
+final class ByteBufferReader extends RandomAccessReader {
+
+ private final ByteBuffer buffer;
+
+ /** Reads {@code buffer} from position 0 to its limit; the buffer's position is not used. */
+ ByteBufferReader(ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ @Override
+ public int toUnshiftedOffset(int localOffset) {
+ return localOffset;
+ }
+
+ @Override
+ public long getLength() {
+ return buffer.limit();
+ }
+
+ @Override
+ public byte getByte(int index) throws java.io.IOException {
+ validateIndex(index, 1);
+ return buffer.get(index);
+ }
+
+ @Override
+ public byte[] getBytes(int index, int count) throws java.io.IOException {
+ validateIndex(index, count);
+ byte[] bytes = new byte[count];
+ buffer.get(index, bytes);
+ return bytes;
+ }
+
+ @Override
+ protected boolean isValidIndex(int index, int bytesRequested) {
+ return bytesRequested >= 0 && index >= 0 &&
+ (long) index + bytesRequested - 1L < buffer.limit();
+ }
+
+ @Override
+ protected void validateIndex(int index, int bytesRequested) throws java.io.IOException {
+ if (!isValidIndex(index, bytesRequested)) {
+ throw new BufferBoundsException(index, bytesRequested, buffer.limit());
+ }
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
index 3272f8d72f..41f2b4bca3 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
@@ -19,6 +19,8 @@ package org.apache.tika.parser.image;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.SimpleDateFormat;
@@ -41,6 +43,7 @@ import com.drew.imaging.tiff.TiffProcessingException;
import com.drew.imaging.webp.WebpMetadataReader;
import com.drew.lang.ByteArrayReader;
import com.drew.lang.GeoLocation;
+import com.drew.lang.RandomAccessReader;
import com.drew.lang.Rational;
import com.drew.metadata.Directory;
import com.drew.metadata.MetadataException;
@@ -70,6 +73,7 @@ import org.apache.commons.io.IOUtils;
import org.xml.sax.SAXException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Geographic;
import org.apache.tika.metadata.IPTC;
import org.apache.tika.metadata.KeyPrefix;
@@ -135,6 +139,50 @@ public class ImageMetadataExtractor {
new ExifReader(), new IccReader(), new PhotoshopReader(), new DuckyReader(),
new IptcReader(), new AdobeJpegReader(), new JpegDhtReader(), new JpegDnlReader());
+ /**
+ * Reads from the file when one already exists (that path also yields metadata-extractor's
+ * file-system tags: img:File Name/Size/Modified Date, which describe the file, not the
+ * image), and from the stream otherwise. JPEG is read sequentially, so the stream costs
+ * no extra memory.
+ */
+ public void parseJpeg(TikaInputStream tis) throws IOException, SAXException, TikaException {
+ if (tis.hasFile()) {
+ parseJpeg(tis.getFile());
+ } else {
+ parseJpeg((InputStream) tis);
+ }
+ }
+
+ /**
+ * TIFF needs random access. In-memory content is read in place through a zero-copy view;
+ * anything else is read from the file. There is deliberately no InputStream overload:
+ * metadata-extractor's stream reader retains everything it reads.
+ */
+ public void parseTiff(TikaInputStream tis) throws IOException, SAXException, TikaException {
+ if (tis.hasFile()) {
+ parseTiff(tis.getFile());
+ return;
+ }
+ try (SeekableByteChannel channel = tis.getSeekableByteChannel()) {
+ ByteBuffer view = TikaInputStream.inMemoryContent(channel);
+ if (view != null) {
+ parseTiff(new ByteBufferReader(view));
+ return;
+ }
+ }
+ // the drain spilled: the content is on disk now
+ parseTiff(tis.getFile());
+ }
+
+ /** See {@link #parseJpeg(TikaInputStream)}; WebP is read sequentially too. */
+ public void parseWebP(TikaInputStream tis) throws IOException, TikaException {
+ if (tis.hasFile()) {
+ parseWebP(tis.getFile());
+ } else {
+ parseWebP((InputStream) tis);
+ }
+ }
+
public void parseJpeg(File file) throws IOException, SAXException, TikaException {
try {
com.drew.metadata.Metadata jpegMetadata =
@@ -145,23 +193,41 @@ public class ImageMetadataExtractor {
}
}
+ public void parseJpeg(InputStream stream) throws IOException, SAXException, TikaException {
+ try {
+ handle(JpegMetadataReader.readMetadata(stream, JPEG_READERS_NO_XMP));
+ } catch (JpegProcessingException | MetadataException e) {
+ throw new TikaException("Can't read JPEG metadata", e);
+ }
+ }
+
public void parseTiff(File file) throws IOException, SAXException, TikaException {
try {
- com.drew.metadata.Metadata tiffMetadata = TiffMetadataReader.readMetadata(file);
- handle(tiffMetadata);
+ handle(TiffMetadataReader.readMetadata(file));
+ } catch (MetadataException | TiffProcessingException e) {
+ throw new TikaException("Can't read TIFF metadata", e);
+ }
+ }
+
+ private void parseTiff(RandomAccessReader reader) throws IOException, SAXException, TikaException {
+ try {
+ handle(TiffMetadataReader.readMetadata(reader));
} catch (MetadataException | TiffProcessingException e) {
throw new TikaException("Can't read TIFF metadata", e);
}
}
public void parseWebP(File file) throws IOException, TikaException {
+ try {
+ handle(WebpMetadataReader.readMetadata(file));
+ } catch (RiffProcessingException | MetadataException e) {
+ throw new TikaException("Can't process Riff data", e);
+ }
+ }
+ public void parseWebP(InputStream stream) throws IOException, TikaException {
try {
- com.drew.metadata.Metadata webPMetadata = new com.drew.metadata.Metadata();
- webPMetadata = WebpMetadataReader.readMetadata(file);
- handle(webPMetadata);
- } catch (IOException e) {
- throw e;
+ handle(WebpMetadataReader.readMetadata(stream));
} catch (RiffProcessingException | MetadataException e) {
throw new TikaException("Can't process Riff data", e);
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java
index af88c13659..8f540148de 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java
@@ -17,8 +17,6 @@
package org.apache.tika.parser.image;
import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
@@ -28,6 +26,7 @@ import com.drew.imaging.jpeg.JpegProcessingException;
import com.drew.imaging.jpeg.JpegSegmentData;
import com.drew.imaging.jpeg.JpegSegmentReader;
import com.drew.imaging.jpeg.JpegSegmentType;
+import com.drew.lang.StreamReader;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.xml.sax.SAXException;
@@ -81,12 +80,12 @@ final class ImageXmp {
}
}
- /** JPEG: read APP1 segments, reassemble Extended XMP, parse each resulting packet. */
- static void extractJpeg(File file, Metadata metadata, ParseContext context) {
+ /** JPEG: read APP1 segments, reassemble Extended XMP, parse each resulting packet. Consumes the stream. */
+ static void extractJpeg(InputStream stream, Metadata metadata, ParseContext context) {
try {
Iterable<byte[]> app1;
try {
- JpegSegmentData data = JpegSegmentReader.readSegments(file,
+ JpegSegmentData data = JpegSegmentReader.readSegments(new StreamReader(stream),
Collections.singletonList(JpegSegmentType.APP1));
app1 = data.getSegments(JpegSegmentType.APP1);
} catch (JpegProcessingException e) {
@@ -105,10 +104,10 @@ final class ImageXmp {
}
}
- /** WebP: pull the raw packet out of the RIFF {@code "XMP "} chunk and parse it. */
- static void extractWebp(File file, Metadata metadata, ParseContext context) {
+ /** WebP: pull the raw packet out of the RIFF {@code "XMP "} chunk and parse it. Consumes the stream. */
+ static void extractWebp(InputStream stream, Metadata metadata, ParseContext context) {
try {
- byte[] xmp = readRiffChunk(file, "XMP ");
+ byte[] xmp = readRiffChunk(stream, "XMP ");
if (xmp != null) {
new XmpExtractor().extract(xmp, metadata, context);
}
@@ -123,28 +122,28 @@ final class ImageXmp {
private static final long MAX_CHUNK = 64L * 1024 * 1024;
/** Return the payload of the first top-level RIFF chunk with the given FourCC, or null. */
- private static byte[] readRiffChunk(File file, String fourCC) throws IOException {
- try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {
- byte[] head = new byte[12];
- if (IOUtils.read(in, head, 0, 12) < 12 || head[0] != 'R' || head[1] != 'I' ||
- head[2] != 'F' || head[3] != 'F' || head[8] != 'W' || head[9] != 'E' ||
- head[10] != 'B' || head[11] != 'P') {
- return null;
- }
- byte[] ch = new byte[8];
- while (IOUtils.read(in, ch, 0, 8) == 8) {
- long size = (ch[4] & 0xffL) | (ch[5] & 0xffL) << 8 |
- (ch[6] & 0xffL) << 16 | (ch[7] & 0xffL) << 24;
- if (fourCC.equals(new String(ch, 0, 4, StandardCharsets.US_ASCII))) {
- if (size > MAX_CHUNK) {
- return null; // target chunk too large to allocate
- }
- byte[] data = new byte[(int) size];
- return IOUtils.read(in, data, 0, data.length) == data.length ? data : null;
+ private static byte[] readRiffChunk(InputStream stream, String fourCC) throws IOException {
+ // not closed: the stream is the caller's, who rewinds it for the next pass
+ InputStream in = new BufferedInputStream(stream);
+ byte[] head = new byte[12];
+ if (IOUtils.read(in, head, 0, 12) < 12 || head[0] != 'R' || head[1] != 'I' ||
+ head[2] != 'F' || head[3] != 'F' || head[8] != 'W' || head[9] != 'E' ||
+ head[10] != 'B' || head[11] != 'P') {
+ return null;
+ }
+ byte[] ch = new byte[8];
+ while (IOUtils.read(in, ch, 0, 8) == 8) {
+ long size = (ch[4] & 0xffL) | (ch[5] & 0xffL) << 8 |
+ (ch[6] & 0xffL) << 16 | (ch[7] & 0xffL) << 24;
+ if (fourCC.equals(new String(ch, 0, 4, StandardCharsets.US_ASCII))) {
+ if (size > MAX_CHUNK) {
+ return null; // target chunk too large to allocate
}
- // a large foreign chunk before "XMP " must be skipped, not abort the scan
- IOUtils.skipFully(in, size + (size & 1L)); // RIFF pads chunks to even length
+ byte[] data = new byte[(int) size];
+ return IOUtils.read(in, data, 0, data.length) == data.length ? data : null;
}
+ // a large foreign chunk before "XMP " must be skipped, not abort the scan
+ IOUtils.skipFully(in, size + (size & 1L)); // RIFF pads chunks to even length
}
return null;
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java
index a96311857f..bf49883b5a 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java
@@ -26,6 +26,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -54,9 +55,12 @@ public class JpegParser extends AbstractImageParser {
TemporaryResources tmp = new TemporaryResources();
try {
TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata);
+ // two sequential passes; rewind rather than spool between them
+ tis.enableRewind(parseContext.get(CacheMemoryBudget.class));
// XMP first so it is canonical; the metadata-extractor handlers (IPTC/EXIF) fill gaps.
- ImageXmp.extractJpeg(tis.getFile(), metadata, parseContext);
- new ImageMetadataExtractor(metadata).parseJpeg(tis.getFile());
+ ImageXmp.extractJpeg(tis, metadata, parseContext);
+ tis.rewind();
+ new ImageMetadataExtractor(metadata).parseJpeg(tis);
} finally {
tmp.dispose();
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java
index 65664bff10..4a5cad5f48 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java
@@ -26,6 +26,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -54,10 +55,11 @@ public class TiffParser extends AbstractImageParser {
TemporaryResources tmp = new TemporaryResources();
try {
TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata);
- tis.getFile(); // spool so tis is fully re-readable below
+ tis.enableRewind(parseContext.get(CacheMemoryBudget.class));
// XMP first so it is canonical; metadata-extractor (IPTC/EXIF) fills gaps.
ImageXmp.scanAndExtract(tis, metadata, parseContext);
- new ImageMetadataExtractor(metadata).parseTiff(tis.getFile());
+ tis.rewind();
+ new ImageMetadataExtractor(metadata).parseTiff(tis);
} finally {
tmp.dispose();
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java
index 1af3b5e5a3..459aa07def 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java
@@ -25,6 +25,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -50,8 +51,10 @@ public class WebPParser implements Parser {
public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata,
ParseContext context) throws IOException, SAXException, TikaException {
// XMP first (canonical), then EXIF/etc. from metadata-extractor as fallback.
- ImageXmp.extractWebp(tis.getFile(), metadata, context);
- new ImageMetadataExtractor(metadata).parseWebP(tis.getFile());
+ tis.enableRewind(context.get(CacheMemoryBudget.class));
+ ImageXmp.extractWebp(tis, metadata, context);
+ tis.rewind();
+ new ImageMetadataExtractor(metadata).parseWebP(tis);
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context);
xhtml.startDocument();
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java
new file mode 100644
index 0000000000..7a72e536a6
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TIFF;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+
+/**
+ * Embedded images usually arrive already in memory; the image parsers must not spool them to
+ * disk just to read metadata. The file-system tags are the visible consequence: they describe
+ * whatever file was read, so in-memory input must not carry them and real files must.
+ */
+public class ImageParsersNoTempFileTest extends TikaTest {
+
+ private static final String FILE_NAME_KEY = ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Name";
+ private static final String FILE_SIZE_KEY = ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Size";
+ private static final String FILE_MODIFIED_KEY =
+ ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Modified Date";
+
+ @TempDir
+ Path tempDir;
+
+ private static final String JPEG = "/test-documents/testJPEG_EXIF.jpg";
+ private static final String TIFF_RES = "/test-documents/testTIFF.tif";
+ private static final String WEBP = "/test-documents/testWebp_Alpha_Lossless.webp";
+ private static final String WEBP_WIDTH = ImageMetadataExtractor.UNKNOWN_IMG_NS + "Image Width";
+
+ private static ParseContext context() {
+ ParseContext context = new ParseContext();
+ context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
+ return context;
+ }
+
+ private byte[] bytes(String resource) throws Exception {
+ try (InputStream is = getResourceAsStream(resource)) {
+ return is.readAllBytes();
+ }
+ }
+
+ @Test
+ public void testJpegInMemory() throws Exception {
+ assertNotSpooled(new JpegParser(), JPEG, TIFF.IMAGE_WIDTH.getName());
+ }
+
+ @Test
+ public void testTiffInMemory() throws Exception {
+ assertNotSpooled(new TiffParser(), TIFF_RES, TIFF.IMAGE_WIDTH.getName());
+ }
+
+ @Test
+ public void testWebPInMemory() throws Exception {
+ assertNotSpooled(new WebPParser(), WEBP, WEBP_WIDTH);
+ }
+
+ @Test
+ public void testJpegFromFile() throws Exception {
+ assertKeepsFileTags(new JpegParser(), JPEG, TIFF.IMAGE_WIDTH.getName(), "jpg");
+ }
+
+ @Test
+ public void testTiffFromFile() throws Exception {
+ assertKeepsFileTags(new TiffParser(), TIFF_RES, TIFF.IMAGE_WIDTH.getName(), "tif");
+ }
+
+ @Test
+ public void testWebPFromFile() throws Exception {
+ assertKeepsFileTags(new WebPParser(), WEBP, WEBP_WIDTH, "webp");
+ }
+
+ private void assertNotSpooled(Parser parser, String resource, String widthKey)
+ throws Exception {
+ byte[] bytes = bytes(resource);
+ Metadata metadata = new Metadata();
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ tmp.setTemporaryFileDirectory(tempDir);
+ // a stream-backed, non-file TikaInputStream whose only spill target is tempDir
+ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
+ parser.parse(tis, new DefaultHandler(), metadata, context());
+ // temp files live until tmp closes, so any spool would be visible right here
+ try (Stream<Path> files = Files.list(tempDir)) {
+ assertEquals(0, files.count(), "parser spooled an in-memory image to disk");
+ }
+ }
+ assertNotNull(metadata.get(widthKey), "metadata was extracted");
+ assertNull(metadata.get(FILE_NAME_KEY), "in-memory input must not carry file tags");
+ assertNull(metadata.get(FILE_SIZE_KEY), "in-memory input must not carry file tags");
+ assertNull(metadata.get(FILE_MODIFIED_KEY), "in-memory input must not carry file tags");
+ }
+
+ private void assertKeepsFileTags(Parser parser, String resource, String widthKey, String ext)
+ throws Exception {
+ byte[] bytes = bytes(resource);
+ Path image = tempDir.resolve("image." + ext);
+ Files.write(image, bytes);
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(image, metadata)) {
+ parser.parse(tis, new DefaultHandler(), metadata, context());
+ }
+ assertNotNull(metadata.get(widthKey), "metadata was extracted");
+ assertEquals(image.getFileName().toString(), metadata.get(FILE_NAME_KEY),
+ "a real file keeps metadata-extractor's file-system tags");
+ assertNotNull(metadata.get(FILE_SIZE_KEY), "a real file keeps its size tag");
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
index 356dcba2f1..4b5f470815 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
@@ -22,6 +22,9 @@ import static org.apache.tika.mime.MediaType.image;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.SeekableByteChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Collections;
@@ -32,18 +35,23 @@ import java.util.Set;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
import org.apache.poi.hssf.model.InternalWorkbook;
+import org.apache.poi.poifs.common.POIFSConstants;
import org.apache.poi.poifs.filesystem.DirectoryEntry;
import org.apache.poi.poifs.filesystem.DirectoryNode;
import org.apache.poi.poifs.filesystem.DocumentInputStream;
import org.apache.poi.poifs.filesystem.DocumentNode;
import org.apache.poi.poifs.filesystem.Entry;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
+import org.apache.poi.poifs.storage.BATBlock;
+import org.apache.poi.poifs.storage.HeaderBlock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.detect.Detector;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -554,9 +562,22 @@ public class POIFSContainerDetector implements Detector {
}
- private Set<String> getTopLevelNames(TikaInputStream stream) throws IOException {
- // Force the document stream to a (possibly temporary) file
- // so we don't modify the current position of the stream.
+ /**
+ * Largest object opened from memory. POI's stream loader copies the whole object into a
+ * heap array, so this bounds that copy; the cache underneath is already budget-bounded.
+ * Embedded OLE2 objects are typically a few hundred KB.
+ */
+ private static final long MAX_IN_MEMORY_POIFS = 8L * 1024 * 1024;
+
+ private Set<String> getTopLevelNames(TikaInputStream stream, ParseContext context)
+ throws IOException {
+ if (!stream.hasFile()) {
+ Set<String> names = getTopLevelNamesInMemory(stream, context);
+ if (names != null) {
+ return names;
+ }
+ }
+ // random access over content that is not in memory: use the file
Path file = stream.getPath();
if (file == null) {
@@ -583,16 +604,89 @@ public class POIFSContainerDetector implements Detector {
}
}
+ /**
+ * Opens the container from memory. Returns null -- and the caller uses the file -- when
+ * the object is too large, its header would make POI allocate too much, or POI's stream
+ * loader rejects it (it is stricter than the file loader on truncated objects).
+ */
+ private Set<String> getTopLevelNamesInMemory(TikaInputStream stream, ParseContext context)
+ throws IOException {
+ if (stream.hasLength() && stream.getLength() > MAX_IN_MEMORY_POIFS) {
+ return null;
+ }
+ // the budget decides how much the drain below keeps in memory
+ stream.enableRewind(context == null ? null : context.get(CacheMemoryBudget.class));
+ POIFSFileSystem fs = null;
+ try (SeekableByteChannel channel = stream.getSeekableByteChannel()) {
+ if (TikaInputStream.inMemoryContent(channel) == null ||
+ channel.size() > MAX_IN_MEMORY_POIFS ||
+ declaredInMemorySize(channel) > MAX_IN_MEMORY_POIFS) {
+ return null;
+ }
+ fs = new POIFSFileSystem(Channels.newInputStream(channel));
+ Set<String> names = getTopLevelNames(fs.getRoot());
+ stream.setOpenContainer(fs);
+ fs = null;
+ return names;
+ } catch (SecurityException e) {
+ throw e;
+ } catch (IOException | RuntimeException e) {
+ return null;
+ } finally {
+ if (fs != null) {
+ closeQuietly(fs);
+ }
+ }
+ }
+
+ /**
+ * The heap POI would allocate for this object if opened from a stream. It sizes that buffer
+ * from the header's declared BAT count rather than the actual length, so a 512-byte object
+ * can demand hundreds of MB; the file loader reads block by block and never allocates it.
+ * Returns 0 when the header cannot be read, leaving the rejection to POI.
+ */
+ static long declaredInMemorySize(SeekableByteChannel channel) throws IOException {
+ byte[] header = new byte[POIFSConstants.SMALLER_BIG_BLOCK_SIZE];
+ long start = channel.position();
+ try {
+ channel.position(0);
+ ByteBuffer buffer = ByteBuffer.wrap(header);
+ while (buffer.hasRemaining() && channel.read(buffer) > 0) {
+ // fill the header block
+ }
+ if (buffer.hasRemaining()) {
+ return 0;
+ }
+ } finally {
+ channel.position(start);
+ }
+ try {
+ return BATBlock.calculateMaximumSize(
+ new HeaderBlock(UnsynchronizedByteArrayInputStream.builder().setByteArray(header).get()));
+ } catch (IOException | RuntimeException e) {
+ return 0;
+ }
+ }
+
+ private static void closeQuietly(POIFSFileSystem fs) {
+ try {
+ fs.close();
+ } catch (IOException e) {
+ LOG.debug("failed to close abandoned POIFSFileSystem", e);
+ }
+ }
+
public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException {
// Check if we have access to the document
if (tis == null) {
return MediaType.OCTET_STREAM;
}
- return handleTikaStream(tis, metadata);
+ return handleTikaStream(tis, metadata, parseContext);
}
- private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata) throws IOException {
+ private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata, ParseContext context)
+ throws IOException {
//try for an open container
Set<String> names = tryOpenContainerOnTikaInputStream(tis, metadata);
@@ -601,10 +695,8 @@ public class POIFSContainerDetector implements Detector {
return OCTET_STREAM;
}
- // If OLE, spool to disk
if (names == null) {
- // spool to disk and try detection
- names = getTopLevelNames(tis);
+ names = getTopLevelNames(tis, context);
}
// Detect based on the names (as available)
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java
new file mode 100644
index 0000000000..b00f2e57e7
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect.microsoft;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+import org.apache.poi.poifs.filesystem.POIFSFileSystem;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * Detection used to spool every in-memory OLE2 object to a temp file to read its
+ * top-level entry names; it must now open the container from memory.
+ */
+public class POIFSContainerDetectorNoTempFileTest extends TikaTest {
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ public void testNoTempFileForInMemoryInput() throws Exception {
+ byte[] bytes;
+ try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) {
+ bytes = is.readAllBytes();
+ }
+ ParseContext context = new ParseContext();
+ context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
+ Metadata metadata = new Metadata();
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ tmp.setTemporaryFileDirectory(tempDir);
+ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
+ MediaType type = new POIFSContainerDetector().detect(tis, metadata, context);
+ assertEquals(MediaType.application("msword"), type);
+ try (Stream<Path> files = Files.list(tempDir)) {
+ assertEquals(0, files.count(), "detector spooled an in-memory OLE2 object to disk");
+ }
+ assertTrue(tis.getOpenContainer() instanceof POIFSFileSystem, "open container kept for the parser");
+ assertEquals(0, tis.getPosition(), "detection must not move the stream");
+ assertEquals(0xd0, tis.read(), "stream still readable from the start");
+ }
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java
new file mode 100644
index 0000000000..348655d6e5
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect.microsoft;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * POI sizes its in-memory OLE2 buffer from the header's declared BAT count rather than the
+ * actual length, so a 512-byte object can demand hundreds of MB. The in-memory detection path
+ * must reject those on the declared size, not on the real one.
+ */
+public class POIFSDeclaredSizeTest extends TikaTest {
+
+ private static final int BAT_COUNT_OFFSET = 0x2C;
+ private static final int SECTOR_SHIFT_OFFSET = 0x1E;
+
+ @TempDir
+ Path tempDir;
+
+ /** A bare 512-byte OLE2 header declaring {@code batCount} BAT blocks and nothing else. */
+ private static byte[] header(int batCount) {
+ byte[] data = new byte[512];
+ byte[] magic = {(byte) 0xd0, (byte) 0xcf, 0x11, (byte) 0xe0,
+ (byte) 0xa1, (byte) 0xb1, 0x1a, (byte) 0xe1};
+ System.arraycopy(magic, 0, data, 0, magic.length);
+ data[SECTOR_SHIFT_OFFSET] = 9; // 2^9 = 512-byte blocks
+ ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).putInt(BAT_COUNT_OFFSET, batCount);
+ return data;
+ }
+
+ private SeekableByteChannel channelFor(byte[] bytes, String name) throws Exception {
+ Path p = tempDir.resolve(name);
+ Files.write(p, bytes);
+ return Files.newByteChannel(p, StandardOpenOption.READ);
+ }
+
+ @Test
+ public void testDeclaredSizeFollowsHeaderNotLength() throws Exception {
+ // (1 + 3813 * 128) * 512 == 249_889_280, just under POI's 250MB allocation ceiling
+ try (SeekableByteChannel channel = channelFor(header(3813), "hostile.ole")) {
+ long declared = POIFSContainerDetector.declaredInMemorySize(channel);
+ assertEquals(249_889_280L, declared,
+ "a 512-byte object declares a ~238MB in-memory buffer");
+ assertEquals(0, channel.position(), "the size probe must not move the channel");
+ }
+ }
+
+ @Test
+ public void testModestHeaderIsNotRejected() throws Exception {
+ try (SeekableByteChannel channel = channelFor(header(1), "modest.ole")) {
+ assertEquals((1 + 128) * 512L, POIFSContainerDetector.declaredInMemorySize(channel));
+ }
+ }
+
+ @Test
+ public void testTooShortForAHeaderReportsZero() throws Exception {
+ try (SeekableByteChannel channel = channelFor(new byte[16], "short.bin")) {
+ assertEquals(0, POIFSContainerDetector.declaredInMemorySize(channel),
+ "no header to read; leave the rejection to POI");
+ }
+ }
+
+ /**
+ * End to end: the crafted object must not be opened in memory, so no POIFSFileSystem is
+ * retained. Before the declared-size check this allocated ~238MB first.
+ */
+ @Test
+ public void testHostileHeaderDoesNotBecomeAnOpenContainer() throws Exception {
+ ParseContext context = new ParseContext();
+ context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
+ Metadata metadata = new Metadata();
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ TikaInputStream tis = TikaInputStream.get(
+ new ByteArrayInputStream(header(3813)), tmp, metadata);
+ POIFSContainerDetector detector = new POIFSContainerDetector();
+ assertTrue(detector.detect(tis, metadata, context) != null);
+ assertNull(tis.getOpenContainer(),
+ "a header-only object must not be opened from memory");
+ }
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
index c500689ab4..5cc0f37a1c 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
@@ -45,6 +45,7 @@ import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
@@ -320,14 +321,13 @@ public class OpenDocumentParser implements Parser {
if (embeddedName.contains("Pictures/")) {
embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, TikaCoreProperties.EmbeddedResourceType.INLINE.toString());
- //spool
- tisZip.getFile();
+ tisZip.enableRewind(context.get(CacheMemoryBudget.class));
MediaType embeddedMimeType = EmbeddedDocumentUtil.getDetector(context)
.detect(tisZip, embeddedMetadata, context);
if (embeddedMimeType != null) {
embeddedMetadata.set(HttpHeaders.CONTENT_TYPE, embeddedMimeType.toString());
}
- tisZip.reset();
+ tisZip.rewind();
// Tag the picture with the draw:page indices it
// appears on (set populated by scanPicturePages).
// A null lookup means "not referenced by any
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java
new file mode 100644
index 0000000000..fd93ebb635
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.odf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.detect.DefaultDetector;
+import org.apache.tika.detect.Detector;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.BodyContentHandler;
+
+/**
+ * Inline pictures were spooled to a temp file before detection; with rewind support the
+ * entry stays in memory. Watching the temp directory cannot see this -- each entry gets its
+ * own TemporaryResources -- so the assertion is on the stream the detector is handed.
+ */
+public class OpenDocumentParserNoTempFileTest extends TikaTest {
+
+ /** Records whether each inline picture reached detection backed by a file. */
+ private static class SpyDetector implements Detector {
+ private final Detector delegate = new DefaultDetector();
+ private final List<Boolean> pictureHadFile = new ArrayList<>();
+
+ @Override
+ public MediaType detect(TikaInputStream input, Metadata metadata, ParseContext context)
+ throws IOException {
+ String type = metadata.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE);
+ if (TikaCoreProperties.EmbeddedResourceType.INLINE.toString().equals(type)) {
+ pictureHadFile.add(input.hasFile());
+ }
+ return delegate.detect(input, metadata, context);
+ }
+ }
+
+ @Test
+ public void testInlinePicturesAreDetectedFromMemory() throws Exception {
+ byte[] bytes;
+ try (InputStream is = getResourceAsStream("/test-documents/testODTEmbedded.odt")) {
+ bytes = is.readAllBytes();
+ }
+ SpyDetector spy = new SpyDetector();
+ ParseContext context = new ParseContext();
+ context.set(Detector.class, spy);
+ context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
+ Metadata metadata = new Metadata();
+ BodyContentHandler handler = new BodyContentHandler();
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
+ new OpenDocumentParser().parse(tis, handler, metadata, context);
+ }
+ assertFalse(spy.pictureHadFile.isEmpty(), "no inline picture reached detection");
+ for (Boolean hadFile : spy.pictureHadFile) {
+ assertFalse(hadFile, "inline picture was spooled to disk before detection");
+ }
+ assertTrue(handler.toString().length() > 0, "content was extracted");
+ }
+
+ /** The picture must be re-readable after detection, or the embedded parse sees nothing. */
+ @Test
+ public void testEmbeddedPicturesStillParseAfterRewind() throws Exception {
+ List<Metadata> metadataList = getRecursiveMetadata("testODTEmbedded.odt");
+ boolean sawImage = false;
+ for (Metadata m : metadataList) {
+ assertEquals(null, m.get(TikaCoreProperties.EMBEDDED_EXCEPTION),
+ "embedded exception for " + m.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+ String type = m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE);
+ if (TikaCoreProperties.EmbeddedResourceType.INLINE.toString().equals(type)) {
+ sawImage = true;
+ }
+ }
+ assertTrue(sawImage, "no inline picture in the recursive metadata");
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index c61f33cc45..bfdf538101 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -40,7 +40,6 @@ import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.io.MemoryUsageSetting;
import org.apache.pdfbox.io.RandomAccessRead;
import org.apache.pdfbox.io.RandomAccessReadBuffer;
-import org.apache.pdfbox.io.RandomAccessReadBufferedFile;
import org.apache.pdfbox.io.RandomAccessStreamCache;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
@@ -66,6 +65,7 @@ import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.AccessPermissions;
import org.apache.tika.metadata.HttpHeaders;
@@ -181,6 +181,8 @@ public class PDFParser implements Parser, RenderingParser {
context.set(OCRPageCounter.class, new OCRPageCounter());
try {
if (shouldSpool(localConfig)) {
+ // later stages re-open the document (xref scan, renderer, per-page renders)
+ tis.enableRewind(context.get(CacheMemoryBudget.class));
context.set(PDFRenderingState.class, new PDFRenderingState(tis));
}
@@ -313,10 +315,7 @@ public class PDFParser implements Parser, RenderingParser {
return;
}
List<StartXRefOffset> xRefOffsets = new ArrayList<>();
- //TODO -- can we use the PDFBox parser's RandomAccessRead
- //so that we don't have to reopen from file?
- try (RandomAccessRead ra =
- new RandomAccessReadBufferedFile(tikaInputStream.getFile())) {
+ try (RandomAccessRead ra = PDFRandomAccess.open(tikaInputStream, parseContext)) {
StartXRefScanner xRefScanner = new StartXRefScanner(ra);
xRefOffsets.addAll(xRefScanner.scan());
} catch (IOException e) {
@@ -495,21 +494,18 @@ public class PDFParser implements Parser, RenderingParser {
ParseContext context)
throws IOException, EncryptedDocumentException {
try {
- PDDocument pdDocument = null;
if (tis.hasFile()) {
// File based -- send file directly to PDFBox
- pdDocument =
- getPDDocument(tis.getPath(), password, streamCacheCreateFunction, metadata, context);
- } else {
- tis.setCloseShield();
- try {
- pdDocument = getPDDocumentFromStream(tis, password,
- streamCacheCreateFunction, metadata, context);
- } finally {
- tis.removeCloseShield();
- }
+ return getPDDocument(tis.getPath(), password, streamCacheCreateFunction, metadata, context);
+ }
+ // PDFBox owns the reader and closes it with the document
+ RandomAccessRead ra = PDFRandomAccess.open(tis, context);
+ try {
+ return getPDDocument(ra, password, streamCacheCreateFunction, metadata, context);
+ } catch (IOException | RuntimeException e) {
+ ra.close();
+ throw e;
}
- return pdDocument;
} catch (IOException e) {
if (e.getMessage() != null &&
e.getMessage().contains("No security handler for filter")) {
@@ -519,6 +515,12 @@ public class PDFParser implements Parser, RenderingParser {
}
}
+ /**
+ * @deprecated no longer called by this parser: stream-backed input is loaded through
+ * {@link #getPDDocument(RandomAccessRead, String, RandomAccessStreamCache.StreamCacheCreateFunction, Metadata, ParseContext)},
+ * which does not copy the document into heap the way {@code RandomAccessReadBuffer(InputStream)} does
+ */
+ @Deprecated
protected PDDocument getPDDocumentFromStream(InputStream inputStream, String password,
RandomAccessStreamCache.StreamCacheCreateFunction streamCacheCreateFunction,
Metadata metadata,
@@ -526,6 +528,13 @@ public class PDFParser implements Parser, RenderingParser {
return Loader.loadPDF(new RandomAccessReadBuffer(inputStream), password, streamCacheCreateFunction);
}
+ protected PDDocument getPDDocument(RandomAccessRead source, String password,
+ RandomAccessStreamCache.StreamCacheCreateFunction streamCacheCreateFunction,
+ Metadata metadata,
+ ParseContext parseContext) throws IOException {
+ return Loader.loadPDF(source, password, streamCacheCreateFunction);
+ }
+
protected PDDocument getPDDocument(Path path, String password,
RandomAccessStreamCache.StreamCacheCreateFunction
streamCacheCreateFunction, Metadata metadata,
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java
new file mode 100644
index 0000000000..923f5043d4
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+
+import org.apache.pdfbox.io.RandomAccessRead;
+import org.apache.pdfbox.io.RandomAccessReadBuffer;
+import org.apache.pdfbox.io.RandomAccessReadBufferedFile;
+
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * One way to hand PDFBox a document: in-memory content is read in place through a zero-copy
+ * view, anything else through PDFBox's buffered file reader. Neither copies the document, and
+ * every call starts from byte 0 regardless of the stream's position -- so a renderer can
+ * re-open the same document per page. Closing the returned reader releases the view.
+ */
+public final class PDFRandomAccess {
+
+ private PDFRandomAccess() {
+ }
+
+ public static RandomAccessRead open(TikaInputStream tis, ParseContext context)
+ throws IOException {
+ if (tis.hasFile()) {
+ return new RandomAccessReadBufferedFile(tis.getFile());
+ }
+ tis.enableRewind(context == null ? null : context.get(CacheMemoryBudget.class));
+ SeekableByteChannel channel = tis.getSeekableByteChannel();
+ ByteBuffer view = TikaInputStream.inMemoryContent(channel);
+ if (view == null) {
+ // the drain spilled: the content is on disk now
+ channel.close();
+ return new RandomAccessReadBufferedFile(tis.getFile());
+ }
+ return new RandomAccessReadBuffer(view) {
+ @Override
+ public void close() throws IOException {
+ try {
+ super.close();
+ } finally {
+ channel.close();
+ }
+ }
+ };
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
index 8c5c94592a..20bc2117ef 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.Set;
import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.io.RandomAccessReadBuffer;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
@@ -47,6 +46,7 @@ import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.pdf.PDFParser;
import org.apache.tika.parser.pdf.PDFParserConfig;
+import org.apache.tika.parser.pdf.PDFRandomAccess;
import org.apache.tika.renderer.PageBasedRenderResults;
import org.apache.tika.renderer.PageRangeRequest;
import org.apache.tika.renderer.RenderRequest;
@@ -100,7 +100,9 @@ public class PDFBoxRenderer implements PDDocumentRenderer {
if (tis.getOpenContainer() != null) {
pdDocument = (PDDocument) tis.getOpenContainer();
} else {
- pdDocument = Loader.loadPDF(new RandomAccessReadBuffer(tis));
+ // a fresh reader from byte 0 each time, so per-page renders do not depend on
+ // where the stream was left; the document closes it
+ pdDocument = Loader.loadPDF(PDFRandomAccess.open(tis, parseContext));
mustClose = true;
}
PageBasedRenderResults results = new PageBasedRenderResults(new TemporaryResources());
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java
new file mode 100644
index 0000000000..9529fee4d4
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.PDF;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * The incremental-update xref scan used to spool in-memory PDFs to a file; it must scan
+ * from memory like the main parse does.
+ */
+public class PDFParserNoTempFileTest extends TikaTest {
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ public void testXRefScanNoTempFileForInMemoryInput() throws Exception {
+ byte[] bytes;
+ try (InputStream is = getResourceAsStream("/test-documents/testPDF_incrementalUpdates.pdf")) {
+ bytes = is.readAllBytes();
+ }
+ PDFParserConfig config = new PDFParserConfig();
+ config.setExtractIncrementalUpdateInfo(true);
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
+ Metadata metadata = new Metadata();
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ tmp.setTemporaryFileDirectory(tempDir);
+ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
+ new PDFParser().parse(tis, new DefaultHandler(), metadata, context);
+ try (Stream<Path> files = Files.list(tempDir)) {
+ assertEquals(0, files.count(), "xref scan spooled an in-memory PDF to disk");
+ }
+ }
+ assertNotNull(metadata.get(PDF.PDF_INCREMENTAL_UPDATE_COUNT), "incremental update info was extracted");
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPerPageRenderTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPerPageRenderTest.java
new file mode 100644
index 0000000000..df88be575c
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPerPageRenderTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.PagedText;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * Per-page rendering re-opens the document once per page. Each open must start from byte 0
+ * regardless of where the stream was left; before PDFRandomAccess the renderer read the
+ * TikaInputStream from its current position, so page 1 drained it and every later page
+ * rendered nothing, silently.
+ */
+public class PDFPerPageRenderTest extends TikaTest {
+
+ private static final String TWO_PAGES = "/test-documents/testPDF_bookmarks.pdf";
+
+ @TempDir
+ Path tempDir;
+
+ private static ParseContext renderAtPageEnd() {
+ PDFParserConfig config = new PDFParserConfig();
+ config.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END);
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ return context;
+ }
+
+ private static void assertEveryPageRendered(List<Metadata> metadataList) {
+ Metadata container = metadataList.get(0);
+ assertNull(container.get(TikaCoreProperties.EMBEDDED_EXCEPTION),
+ "a page render failure is recorded on the container, not thrown");
+ int pages = container.getInt(PagedText.N_PAGES);
+ assertEquals(2, pages, "fixture must be multi-page for this test to mean anything");
+ long rendered = metadataList.stream()
+ .filter(m -> TikaCoreProperties.EmbeddedResourceType.RENDERING.name()
+ .equals(m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)))
+ .count();
+ assertEquals(pages, rendered, "one rendering per page");
+ }
+
+ @Test
+ public void testStreamBackedRendersEveryPage() throws Exception {
+ byte[] bytes;
+ try (InputStream is = getResourceAsStream(TWO_PAGES)) {
+ bytes = is.readAllBytes();
+ }
+ try (TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), new Metadata())) {
+ assertEveryPageRendered(getRecursiveMetadata(tis, AUTO_DETECT_PARSER, new Metadata(),
+ renderAtPageEnd(), true));
+ }
+ }
+
+ @Test
+ public void testFileBackedRendersEveryPage() throws Exception {
+ Path pdf = tempDir.resolve("two-pages.pdf");
+ try (InputStream is = getResourceAsStream(TWO_PAGES)) {
+ Files.copy(is, pdf);
+ }
+ try (TikaInputStream tis = TikaInputStream.get(pdf)) {
+ assertEveryPageRendered(getRecursiveMetadata(tis, AUTO_DETECT_PARSER, new Metadata(),
+ renderAtPageEnd(), true));
+ }
+ }
+}
(tika) 04/04: Merge remote-tracking branch 'origin/main' into TIKA-4835-spill-sites
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4835-spill-sites
in repository https://gitbox.apache.org/repos/asf/tika.git
commit a2eabc69437109a15d85ce8c81315380d44eb277
Merge: 4a687bbe54 ab22ec9c9a
Author: tallison <ta...@apache.org>
AuthorDate: Wed Aug 26 18:02:33 2026 -0400
Merge remote-tracking branch 'origin/main' into TIKA-4835-spill-sites
CHANGES.txt | 18 ++++
.../org/apache/tika/pipes/core/PipesClient.java | 22 +++--
.../org/apache/tika/pipes/core/PipesParser.java | 9 ++
.../org/apache/tika/pipes/core/ServerManager.java | 23 +++++
.../tika/pipes/core/SharedServerManager.java | 71 ++++++++++++++--
.../pipes/core/SharedServerManagerClosedTest.java | 45 ++++++++++
.../tika/pipes/core/SharedServerModeTest.java | 98 ++++++++++++++++++++++
7 files changed, 275 insertions(+), 11 deletions(-)
diff --cc CHANGES.txt
index e22631edfa,7c0d7b9084..195ea6035d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,27 -1,23 +1,45 @@@
Release 4.1.0 - unreleased
+ * Parsers and detectors no longer spool in-memory input to a temp file just
+ to read it back with random access: the JPEG, TIFF and WebP parsers, the
+ OpenDocument parser's inline pictures, the OLE2 container detector's
+ entry-name read, and PDFParser (the incremental-update scan and the main
+ load). Together these were the bulk of the temp bytes behind 4.0.0's
+ batch slowdown on spinning disks (see docs/.../pipes/performance.adoc).
+ One rule everywhere: content goes through the stream cache (bounded by
+ the pipes cache memory budget, 1MB per object without one); what stays
+ in memory is read in place through a zero-copy view, what spills is
+ read from the file. PDFBox and metadata-extractor no longer receive a
+ second heap copy of the document. The PDF renderer now re-opens the
+ document from byte 0 for every render, which fixes per-page rendering
+ (RENDER_PAGES_AT_PAGE_END) silently producing only the first page --
+ for file-backed input too. PDFParser.getPDDocumentFromStream is
+ deprecated and no longer called; override the new
+ getPDDocument(RandomAccessRead, ...) instead. ImageMetadataExtractor
+ gains TikaInputStream and InputStream overloads. One visible metadata
+ change: JPEG/TIFF/WebP images read from memory no longer carry
+ metadata-extractor's file-system tags (img:File Name, img:File Size,
+ img:File Modified Date), which described whatever file was read rather
+ than the image; input backed by a file still carries them (TIKA-4835).
+
+ * Shared pipes server (useSharedServer: true, not the default): a client whose
+ in-flight parse was killed by another client's restart could restart the
+ healthy replacement. ensureRunning holds its lock across the whole fork, so
+ siblings cannot report a dead worker until after the replacement is up, and
+ the pending-restart flag carried no process identity -- so a report about
+ the process that just died was applied to its successor, which was then
+ destroyed and re-forked. One worker death produced two restarts and a second
+ round of destroyed in-flight work; under sustained concurrent load it
+ sustained itself at one spurious restart per round, appearing as periodic
+ unexplained worker churn and intermittent parse failures that succeed on
+ retry. Each fork now carries a generation that clients capture when they
+ connect and hand back with every report, and reports about a superseded
+ process are dropped. Also fixed in shared mode: ensureRunning could fork a
+ replacement after shutdown() that nothing owned and nothing would ever
+ destroy, and an interrupt during process teardown left the process handle
+ pointing at a killed process and leaked the temp directory. Affects 4.0.0
+ and earlier (TIKA-4844).
+
* tika-pipes: the cache memory budget (how much rewindable content a forked
worker keeps in memory before spilling to disk; new since 4.0.0, which had
no budget at all) defaults to a quarter of the fork's heap, so raising
(tika) 02/04: TIKA-4835 -- merge primitives review fixes; POIFS budget accounting; ImageIO
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4835-spill-sites
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 55156aa176b139ad69b95d7bfb23d1bcc2ed2955
Merge: 58e33ebfa3 379c88a92e
Author: tallison <ta...@apache.org>
AuthorDate: Wed Aug 26 11:41:41 2026 -0400
TIKA-4835 -- merge primitives review fixes; POIFS budget accounting; ImageIO
CHANGES.txt | 35 +++--
docs/modules/ROOT/pages/pipes/configuration.adoc | 27 ++--
.../apache/tika/digest/BufferingDigestSink.java | 52 ++++---
.../org/apache/tika/digest/CompositeDigester.java | 55 +++----
.../java/org/apache/tika/digest/DigestHelper.java | 20 ++-
.../java/org/apache/tika/digest/DigestSink.java | 67 +++++++++
.../main/java/org/apache/tika/digest/Digester.java | 6 +-
.../apache/tika/digest/InputStreamDigester.java | 22 ++-
.../org/apache/tika/io/CachingInputStream.java | 5 +-
.../java/org/apache/tika/io/CachingSource.java | 4 +-
.../apache/tika/io/MemorySeekableByteChannel.java | 3 +-
.../org/apache/tika/io/TemporaryResources.java | 9 +-
.../java/org/apache/tika/io/TikaInputStream.java | 16 +-
.../org/apache/tika/digest/DigestHelperTest.java | 102 +++++++++++++
.../org/apache/tika/digest/DigestSinkTest.java | 166 +++++++++++++++++++--
.../apache/tika/digest/FailingTestTranslator.java | 54 +++++++
.../apache/tika/io/InMemoryContentViewTest.java | 40 +++--
.../org/apache/tika/io/StreamCacheBudgetTest.java | 48 ++++--
.../org/apache/tika/io/TemporaryResourcesTest.java | 22 ---
....apache.tika.extractor.EmbeddedStreamTranslator | 1 +
.../org/apache/tika/parser/image/ImageParser.java | 8 +-
.../parser/image/ImageParsersNoTempFileTest.java | 37 ++++-
.../detect/microsoft/POIFSContainerDetector.java | 74 +++++----
.../POIFSContainerDetectorNoTempFileTest.java | 45 ++++++
.../detect/microsoft/POIFSDeclaredSizeTest.java | 50 +++++--
.../apache/tika/pipes/core/server/PipesServer.java | 39 +++--
.../core/server/CacheMemoryBudgetSeedingTest.java | 18 +++
27 files changed, 796 insertions(+), 229 deletions(-)
diff --cc CHANGES.txt
index 2c0ebe5208,06ce3f95d7..4f886b4ba2
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,39 -1,23 +1,50 @@@
Release 4.1.0 - unreleased
+ * Parsers and detectors no longer spool in-memory input to a temp file just
+ to read it back with random access: the JPEG, TIFF and WebP parsers, the
+ OpenDocument parser's inline pictures, the OLE2 container detector's
+ entry-name read, and PDFParser (the incremental-update scan and the main
+ load). Together these were the bulk of the temp bytes behind 4.0.0's
+ batch slowdown on spinning disks (see docs/.../pipes/performance.adoc).
+ One rule everywhere: content goes through the stream cache (bounded by
+ the pipes cache memory budget, 1MB per object without one); what stays
+ in memory is read in place through a zero-copy view, what spills is
+ read from the file. PDFBox and metadata-extractor no longer receive a
+ second heap copy of the document. The PDF renderer now re-opens the
+ document from byte 0 for every render, which fixes per-page rendering
+ (RENDER_PAGES_AT_PAGE_END) silently producing only the first page --
- for file-backed input too. PDFParser.getPDDocumentFromStream is
++ for file-backed input too. The ImageIO-based ImageParser (PNG, GIF, BMP, ...)
++ reads image headers from memory instead of through ImageIO's default
++ file cache, which created a temp file per image and wrote every byte it
++ read to it (600k writes on a 20k-file sample; the page cache absorbed most
++ of them before they reached disk, so this is syscall and file-churn
++ savings more than disk bytes). PDFParser.getPDDocumentFromStream is
+ deprecated and no longer called; override the new
+ getPDDocument(RandomAccessRead, ...) instead. ImageMetadataExtractor
+ gains TikaInputStream and InputStream overloads. One visible metadata
+ change: JPEG/TIFF/WebP images read from memory no longer carry
+ metadata-extractor's file-system tags (img:File Name, img:File Size,
+ img:File Modified Date), which described whatever file was read rather
+ than the image; input backed by a file still carries them (TIKA-4835).
+
* tika-pipes: the cache memory budget (how much rewindable content a forked
- worker keeps in memory before spilling to disk) now defaults to a quarter
- of the fork's heap, so raising -Xmx raises it; it was a fixed 256MB. It is
- one pool per forked JVM shared by all of its threads.
- -Dtika.pipes.cacheMemoryBudgetBytes in forkedJvmArgs still overrides it
- (below the quarter-heap ceiling; <=0 disables). TikaInputStream.hasFile()
- now also reports content the stream cache spilled on its own, not only
- content a getPath() call put on disk. Digester gains digestSink(), an
- OutputStream that digests as it is written; DigestHelper uses it for
- translated embedded streams, which no longer touch a temp file.
- TemporaryResources and CachingSource now close every remaining resource
- when one close() throws an unchecked exception (TIKA-4835).
+ worker keeps in memory before spilling to disk; new since 4.0.0, which had
+ no budget at all) defaults to a quarter of the fork's heap, so raising
+ -Xmx raises it. It is one pool per forked JVM shared by all of its threads.
+ -Dtika.pipes.cacheMemoryBudgetBytes in forkedJvmArgs overrides it (below
+ the quarter-heap ceiling; <=0 disables); the fork logs the value and its
+ source at startup. TikaInputStream.hasFile() now also reports content the
+ stream cache spilled on its own, not only content a getPath() call put on
+ disk. TikaInputStream.inMemoryContent(channel) gives a zero-copy read-only
+ view of cached content for consumers that need random access. Digester
+ gains digestSink(), a DigestSink that digests as it is written and can be
+ aborted; DigestHelper uses it for translated embedded streams, which no
+ longer touch a temp file when the digester implements digestSink (all of
+ Tika's do; one that only implements digest() still buffers), and a failed
+ translation now publishes no digest instead of a digest of the fragment.
+ TemporaryResources.closeAll(Closeable...) closes every argument even when
+ one throws unchecked; TemporaryResources, CachingSource, CachingInputStream
+ and CompositeDigester use it (TIKA-4835).
* Pipes plugins no longer bundle their own Jackson: jackson-core, -databind
and -annotations are provided by the host (tika-serialization) and the
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageParser.java
index c764d9e403,c764d9e403..8f33335111
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageParser.java
@@@ -28,6 -28,6 +28,7 @@@ import javax.imageio.ImageIO
import javax.imageio.ImageReader;
import javax.imageio.metadata.IIOMetadata;
import javax.imageio.stream.ImageInputStream;
++import javax.imageio.stream.MemoryCacheImageInputStream;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.slf4j.Logger;
@@@ -169,8 -169,8 +170,11 @@@ public class ImageParser extends Abstra
if (iterator.hasNext()) {
ImageReader reader = iterator.next();
try {
-- try (ImageInputStream imageStream = ImageIO
-- .createImageInputStream(CloseShieldInputStream.wrap(stream))) {
++ // Not ImageIO.createImageInputStream(InputStream): with the JDK default
++ // useCache=true that is a FileCacheImageInputStream, which writes every byte
++ // it reads to a temp file. Header reads are small; keep them in memory.
++ try (ImageInputStream imageStream =
++ new MemoryCacheImageInputStream(CloseShieldInputStream.wrap(stream))) {
reader.setInput(imageStream);
try {
int numImages = reader.getNumImages(true);
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java
index 7a72e536a6,0000000000..095d885b84
mode 100644,000000..100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java
@@@ -1,138 -1,0 +1,173 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
++import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.io.ByteArrayInputStream;
++import java.io.File;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
++import java.nio.file.attribute.PosixFilePermissions;
+import java.util.stream.Stream;
++import javax.imageio.ImageIO;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
++import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TIFF;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+
+/**
+ * Embedded images usually arrive already in memory; the image parsers must not spool them to
+ * disk just to read metadata. The file-system tags are the visible consequence: they describe
+ * whatever file was read, so in-memory input must not carry them and real files must.
+ */
+public class ImageParsersNoTempFileTest extends TikaTest {
+
+ private static final String FILE_NAME_KEY = ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Name";
+ private static final String FILE_SIZE_KEY = ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Size";
+ private static final String FILE_MODIFIED_KEY =
+ ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Modified Date";
+
+ @TempDir
+ Path tempDir;
+
+ private static final String JPEG = "/test-documents/testJPEG_EXIF.jpg";
+ private static final String TIFF_RES = "/test-documents/testTIFF.tif";
+ private static final String WEBP = "/test-documents/testWebp_Alpha_Lossless.webp";
+ private static final String WEBP_WIDTH = ImageMetadataExtractor.UNKNOWN_IMG_NS + "Image Width";
++ private static final String PNG = "/test-documents/testPNG.png";
+
+ private static ParseContext context() {
+ ParseContext context = new ParseContext();
+ context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
+ return context;
+ }
+
+ private byte[] bytes(String resource) throws Exception {
+ try (InputStream is = getResourceAsStream(resource)) {
+ return is.readAllBytes();
+ }
+ }
+
+ @Test
+ public void testJpegInMemory() throws Exception {
+ assertNotSpooled(new JpegParser(), JPEG, TIFF.IMAGE_WIDTH.getName());
+ }
+
+ @Test
+ public void testTiffInMemory() throws Exception {
+ assertNotSpooled(new TiffParser(), TIFF_RES, TIFF.IMAGE_WIDTH.getName());
+ }
+
+ @Test
+ public void testWebPInMemory() throws Exception {
+ assertNotSpooled(new WebPParser(), WEBP, WEBP_WIDTH);
+ }
+
++ /**
++ * ImageIO's own file cache is outside TemporaryResources entirely, and its cache file is
++ * gone by the time a parse returns -- so the watched-directory check cannot see it. Make
++ * the cache directory unwritable instead: a parser that asks ImageIO for a file-backed
++ * stream then fails with "Can't create cache file"; one that reads from memory never
++ * notices.
++ */
++ @Test
++ public void testImageIoParserUsesNoFileCache() throws Exception {
++ assumeTrue(Files.getFileStore(tempDir).supportsFileAttributeView("posix"), "needs POSIX permissions");
++ Path cache = Files.createDirectory(tempDir.resolve("imageio-cache"));
++ File before = ImageIO.getCacheDirectory();
++ boolean useCache = ImageIO.getUseCache();
++ ImageIO.setUseCache(true);
++ ImageIO.setCacheDirectory(cache.toFile());
++ Files.setPosixFilePermissions(cache, PosixFilePermissions.fromString("r-xr-xr-x"));
++ try {
++ assertNotSpooled(new ImageParser(), PNG, TIFF.IMAGE_WIDTH.getName());
++ } finally {
++ Files.setPosixFilePermissions(cache, PosixFilePermissions.fromString("rwxr-xr-x"));
++ ImageIO.setCacheDirectory(before);
++ ImageIO.setUseCache(useCache);
++ }
++ }
++
+ @Test
+ public void testJpegFromFile() throws Exception {
+ assertKeepsFileTags(new JpegParser(), JPEG, TIFF.IMAGE_WIDTH.getName(), "jpg");
+ }
+
+ @Test
+ public void testTiffFromFile() throws Exception {
+ assertKeepsFileTags(new TiffParser(), TIFF_RES, TIFF.IMAGE_WIDTH.getName(), "tif");
+ }
+
+ @Test
+ public void testWebPFromFile() throws Exception {
+ assertKeepsFileTags(new WebPParser(), WEBP, WEBP_WIDTH, "webp");
+ }
+
+ private void assertNotSpooled(Parser parser, String resource, String widthKey)
+ throws Exception {
+ byte[] bytes = bytes(resource);
+ Metadata metadata = new Metadata();
++ if (resource.endsWith(".png")) {
++ metadata.set(HttpHeaders.CONTENT_TYPE, "image/png");
++ }
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ tmp.setTemporaryFileDirectory(tempDir);
+ // a stream-backed, non-file TikaInputStream whose only spill target is tempDir
+ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
+ parser.parse(tis, new DefaultHandler(), metadata, context());
+ // temp files live until tmp closes, so any spool would be visible right here
+ try (Stream<Path> files = Files.list(tempDir)) {
- assertEquals(0, files.count(), "parser spooled an in-memory image to disk");
++ assertEquals(0, files.filter(Files::isRegularFile).count(),
++ "parser spooled an in-memory image to disk");
+ }
+ }
+ assertNotNull(metadata.get(widthKey), "metadata was extracted");
+ assertNull(metadata.get(FILE_NAME_KEY), "in-memory input must not carry file tags");
+ assertNull(metadata.get(FILE_SIZE_KEY), "in-memory input must not carry file tags");
+ assertNull(metadata.get(FILE_MODIFIED_KEY), "in-memory input must not carry file tags");
+ }
+
+ private void assertKeepsFileTags(Parser parser, String resource, String widthKey, String ext)
+ throws Exception {
+ byte[] bytes = bytes(resource);
+ Path image = tempDir.resolve("image." + ext);
+ Files.write(image, bytes);
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(image, metadata)) {
+ parser.parse(tis, new DefaultHandler(), metadata, context());
+ }
+ assertNotNull(metadata.get(widthKey), "metadata was extracted");
+ assertEquals(image.getFileName().toString(), metadata.get(FILE_NAME_KEY),
+ "a real file keeps metadata-extractor's file-system tags");
+ assertNotNull(metadata.get(FILE_SIZE_KEY), "a real file keeps its size tag");
+ }
+}
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
index 4b5f470815,356dcba2f1..a0500ae985
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java
@@@ -562,22 -554,9 +562,15 @@@ public class POIFSContainerDetector imp
}
- /**
- * Largest object opened from memory. POI's stream loader copies the whole object into a
- * heap array, so this bounds that copy; the cache underneath is already budget-bounded.
- * Embedded OLE2 objects are typically a few hundred KB.
- */
- private static final long MAX_IN_MEMORY_POIFS = 8L * 1024 * 1024;
-
- private Set<String> getTopLevelNames(TikaInputStream stream) throws IOException {
- // Force the document stream to a (possibly temporary) file
- // so we don't modify the current position of the stream.
+ private Set<String> getTopLevelNames(TikaInputStream stream, ParseContext context)
+ throws IOException {
+ if (!stream.hasFile()) {
+ Set<String> names = getTopLevelNamesInMemory(stream, context);
+ if (names != null) {
+ return names;
+ }
+ }
+ // random access over content that is not in memory: use the file
Path file = stream.getPath();
if (file == null) {
@@@ -604,78 -583,6 +597,107 @@@
}
}
+ /**
- * Opens the container from memory. Returns null -- and the caller uses the file -- when
- * the object is too large, its header would make POI allocate too much, or POI's stream
- * loader rejects it (it is stricter than the file loader on truncated objects).
++ * Opens the container from memory. POI's stream loader copies the object into its own
++ * heap array, sized from the header rather than the content, and keeps it for the
++ * container's lifetime -- so that allocation is reserved from the CacheMemoryBudget
++ * before it is made. Returns null, and the caller uses the file, when the content is not
++ * in memory, the budget has no room for the copy, or POI's stream loader rejects the
++ * object (it is stricter than the file loader on truncated objects). Without a budget
++ * only objects under the cache's threshold have an in-memory view, which bounds the copy.
+ */
+ private Set<String> getTopLevelNamesInMemory(TikaInputStream stream, ParseContext context)
+ throws IOException {
- if (stream.hasLength() && stream.getLength() > MAX_IN_MEMORY_POIFS) {
- return null;
- }
++ CacheMemoryBudget budget = context == null ? null : context.get(CacheMemoryBudget.class);
+ // the budget decides how much the drain below keeps in memory
- stream.enableRewind(context == null ? null : context.get(CacheMemoryBudget.class));
++ stream.enableRewind(budget);
+ POIFSFileSystem fs = null;
++ long reserved = 0;
+ try (SeekableByteChannel channel = stream.getSeekableByteChannel()) {
- if (TikaInputStream.inMemoryContent(channel) == null ||
- channel.size() > MAX_IN_MEMORY_POIFS ||
- declaredInMemorySize(channel) > MAX_IN_MEMORY_POIFS) {
++ if (TikaInputStream.inMemoryContent(channel) == null) {
+ return null;
+ }
++ // POI allocates what the header declares. The bytes in hand are the truth: a
++ // header that declares more than they can account for is lying, and the file
++ // loader (block by block, no such allocation) is the only safe way to read it.
++ long copy = honestDeclaredSize(channel);
++ if (copy < 0) {
++ return null;
++ }
++ if (budget != null) {
++ if (budget.tryReserve(copy) == 0) {
++ return null;
++ }
++ reserved = copy;
++ }
+ fs = new POIFSFileSystem(Channels.newInputStream(channel));
+ Set<String> names = getTopLevelNames(fs.getRoot());
+ stream.setOpenContainer(fs);
++ if (reserved > 0) {
++ long charged = reserved;
++ stream.addCloseableResource(() -> budget.release(charged));
++ reserved = 0;
++ }
+ fs = null;
+ return names;
+ } catch (SecurityException e) {
+ throw e;
+ } catch (IOException | RuntimeException e) {
+ return null;
+ } finally {
++ if (reserved > 0) {
++ budget.release(reserved);
++ }
+ if (fs != null) {
+ closeQuietly(fs);
+ }
+ }
+ }
+
+ /**
- * The heap POI would allocate for this object if opened from a stream. It sizes that buffer
- * from the header's declared BAT count rather than the actual length, so a 512-byte object
- * can demand hundreds of MB; the file loader reads block by block and never allocates it.
- * Returns 0 when the header cannot be read, leaving the rejection to POI.
++ * The heap POI's stream loader would allocate for this object -- sized from the header's
++ * declared BAT count, not the content -- or -1 when the header cannot be read or declares
++ * more than the content can account for. A valid header covers at most one BAT block of
++ * unused entries beyond the actual size; anything past that is a malformed or hostile
++ * header (a 512-byte object can declare hundreds of MB) and must not be opened from a
++ * stream at all.
+ */
- static long declaredInMemorySize(SeekableByteChannel channel) throws IOException {
++ static long honestDeclaredSize(SeekableByteChannel channel) throws IOException {
+ byte[] header = new byte[POIFSConstants.SMALLER_BIG_BLOCK_SIZE];
+ long start = channel.position();
+ try {
+ channel.position(0);
+ ByteBuffer buffer = ByteBuffer.wrap(header);
+ while (buffer.hasRemaining() && channel.read(buffer) > 0) {
+ // fill the header block
+ }
+ if (buffer.hasRemaining()) {
- return 0;
++ return -1;
+ }
+ } finally {
+ channel.position(start);
+ }
+ try {
- return BATBlock.calculateMaximumSize(
- new HeaderBlock(UnsynchronizedByteArrayInputStream.builder().setByteArray(header).get()));
++ HeaderBlock hb = new HeaderBlock(
++ UnsynchronizedByteArrayInputStream.builder().setByteArray(header).get());
++ long declared = BATBlock.calculateMaximumSize(hb);
++ long oneBatSpan = (long) hb.getBigBlockSize().getBigBlockSize() *
++ hb.getBigBlockSize().getBATEntriesPerBlock();
++ long actual = channel.size();
++ return declared > actual + oneBatSpan ? -1 : declared;
+ } catch (IOException | RuntimeException e) {
- return 0;
++ return -1;
+ }
+ }
+
+ private static void closeQuietly(POIFSFileSystem fs) {
+ try {
+ fs.close();
+ } catch (IOException e) {
+ LOG.debug("failed to close abandoned POIFSFileSystem", e);
+ }
+ }
+
public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException {
// Check if we have access to the document
if (tis == null) {
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java
index b00f2e57e7,0000000000..6a06ded324
mode 100644,000000..100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java
@@@ -1,71 -1,0 +1,116 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect.microsoft;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+import org.apache.poi.poifs.filesystem.POIFSFileSystem;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * Detection used to spool every in-memory OLE2 object to a temp file to read its
+ * top-level entry names; it must now open the container from memory.
+ */
+public class POIFSContainerDetectorNoTempFileTest extends TikaTest {
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ public void testNoTempFileForInMemoryInput() throws Exception {
+ byte[] bytes;
+ try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) {
+ bytes = is.readAllBytes();
+ }
+ ParseContext context = new ParseContext();
+ context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
+ Metadata metadata = new Metadata();
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ tmp.setTemporaryFileDirectory(tempDir);
+ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
+ MediaType type = new POIFSContainerDetector().detect(tis, metadata, context);
+ assertEquals(MediaType.application("msword"), type);
+ try (Stream<Path> files = Files.list(tempDir)) {
+ assertEquals(0, files.count(), "detector spooled an in-memory OLE2 object to disk");
+ }
+ assertTrue(tis.getOpenContainer() instanceof POIFSFileSystem, "open container kept for the parser");
+ assertEquals(0, tis.getPosition(), "detection must not move the stream");
+ assertEquals(0xd0, tis.read(), "stream still readable from the start");
+ }
+ }
++
++ /** POI's copy is charged to the budget for the container's lifetime, then released. */
++ @Test
++ public void testInMemoryCopyIsChargedAndReleased() throws Exception {
++ byte[] bytes;
++ try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) {
++ bytes = is.readAllBytes();
++ }
++ CacheMemoryBudget budget = new CacheMemoryBudget(64L * 1024 * 1024);
++ ParseContext context = new ParseContext();
++ context.set(CacheMemoryBudget.class, budget);
++ Metadata metadata = new Metadata();
++ TikaInputStream tis;
++ try (TemporaryResources tmp = new TemporaryResources()) {
++ tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
++ assertEquals(MediaType.application("msword"),
++ new POIFSContainerDetector().detect(tis, metadata, context));
++ assertTrue(tis.getOpenContainer() instanceof POIFSFileSystem);
++ assertTrue(budget.getReservedBytes() >= bytes.length,
++ "POI's header-sized copy is charged while the container is open");
++ }
++ assertEquals(0, budget.getReservedBytes(), "released with the stream");
++ }
++
++ /** No room in the budget for POI's copy: detection still succeeds, from the file. */
++ @Test
++ public void testFallsBackToFileWhenBudgetRefusesTheCopy() throws Exception {
++ byte[] bytes;
++ try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) {
++ bytes = is.readAllBytes();
++ }
++ CacheMemoryBudget budget = new CacheMemoryBudget(4096);
++ ParseContext context = new ParseContext();
++ context.set(CacheMemoryBudget.class, budget);
++ Metadata metadata = new Metadata();
++ try (TemporaryResources tmp = new TemporaryResources()) {
++ tmp.setTemporaryFileDirectory(tempDir);
++ TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata);
++ assertEquals(MediaType.application("msword"),
++ new POIFSContainerDetector().detect(tis, metadata, context));
++ assertTrue(tis.getOpenContainer() instanceof POIFSFileSystem);
++ assertTrue(tis.hasFile(), "opened from the file instead");
++ }
++ assertEquals(0, budget.getReservedBytes(), "nothing left charged");
++ }
+}
diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java
index 348655d6e5,0000000000..6baedcb2f7
mode 100644,000000..100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java
@@@ -1,115 -1,0 +1,137 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect.microsoft;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
++import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * POI sizes its in-memory OLE2 buffer from the header's declared BAT count rather than the
+ * actual length, so a 512-byte object can demand hundreds of MB. The in-memory detection path
- * must reject those on the declared size, not on the real one.
++ * only believes a header the bytes in hand can account for, and reserves that from the
++ * budget before POI allocates it.
+ */
+public class POIFSDeclaredSizeTest extends TikaTest {
+
+ private static final int BAT_COUNT_OFFSET = 0x2C;
+ private static final int SECTOR_SHIFT_OFFSET = 0x1E;
+
+ @TempDir
+ Path tempDir;
+
+ /** A bare 512-byte OLE2 header declaring {@code batCount} BAT blocks and nothing else. */
+ private static byte[] header(int batCount) {
+ byte[] data = new byte[512];
+ byte[] magic = {(byte) 0xd0, (byte) 0xcf, 0x11, (byte) 0xe0,
+ (byte) 0xa1, (byte) 0xb1, 0x1a, (byte) 0xe1};
+ System.arraycopy(magic, 0, data, 0, magic.length);
+ data[SECTOR_SHIFT_OFFSET] = 9; // 2^9 = 512-byte blocks
+ ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).putInt(BAT_COUNT_OFFSET, batCount);
+ return data;
+ }
+
+ private SeekableByteChannel channelFor(byte[] bytes, String name) throws Exception {
+ Path p = tempDir.resolve(name);
+ Files.write(p, bytes);
+ return Files.newByteChannel(p, StandardOpenOption.READ);
+ }
+
+ @Test
- public void testDeclaredSizeFollowsHeaderNotLength() throws Exception {
- // (1 + 3813 * 128) * 512 == 249_889_280, just under POI's 250MB allocation ceiling
++ public void testLyingHeaderIsRejected() throws Exception {
++ // (1 + 3813 * 128) * 512 == 249_889_280, just under POI's 250MB allocation ceiling,
++ // declared by 512 bytes of content
+ try (SeekableByteChannel channel = channelFor(header(3813), "hostile.ole")) {
- long declared = POIFSContainerDetector.declaredInMemorySize(channel);
- assertEquals(249_889_280L, declared,
- "a 512-byte object declares a ~238MB in-memory buffer");
++ assertEquals(-1, POIFSContainerDetector.honestDeclaredSize(channel),
++ "the content cannot account for what the header declares");
+ assertEquals(0, channel.position(), "the size probe must not move the channel");
+ }
+ }
+
+ @Test
- public void testModestHeaderIsNotRejected() throws Exception {
++ public void testHonestHeaderIsBelievedWithinOneBatBlock() throws Exception {
++ // header only, declaring one BAT block: 129 sectors, 512 bytes present -- within slack
+ try (SeekableByteChannel channel = channelFor(header(1), "modest.ole")) {
- assertEquals((1 + 128) * 512L, POIFSContainerDetector.declaredInMemorySize(channel));
++ assertEquals((1 + 128) * 512L, POIFSContainerDetector.honestDeclaredSize(channel));
++ }
++ // two BAT blocks declared by 512 bytes: one block past what the content covers
++ try (SeekableByteChannel channel = channelFor(header(2), "twoblocks.ole")) {
++ assertEquals(-1, POIFSContainerDetector.honestDeclaredSize(channel));
++ }
++ }
++
++ @Test
++ public void testRealDocumentHeaderIsHonest() throws Exception {
++ byte[] bytes;
++ try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) {
++ bytes = is.readAllBytes();
++ }
++ try (SeekableByteChannel channel = channelFor(bytes, "real.doc")) {
++ long declared = POIFSContainerDetector.honestDeclaredSize(channel);
++ assertTrue(declared >= bytes.length && declared <= bytes.length + 128 * 512L,
++ "a real header declares about its own size: " + declared + " vs " + bytes.length);
+ }
+ }
+
+ @Test
- public void testTooShortForAHeaderReportsZero() throws Exception {
++ public void testTooShortForAHeaderIsRejected() throws Exception {
+ try (SeekableByteChannel channel = channelFor(new byte[16], "short.bin")) {
- assertEquals(0, POIFSContainerDetector.declaredInMemorySize(channel),
- "no header to read; leave the rejection to POI");
++ assertEquals(-1, POIFSContainerDetector.honestDeclaredSize(channel));
+ }
+ }
+
+ /**
- * End to end: the crafted object must not be opened in memory, so no POIFSFileSystem is
- * retained. Before the declared-size check this allocated ~238MB first.
++ * End to end: the crafted object is refused before any reservation, so POI never
++ * allocates its declared size and no POIFSFileSystem is retained -- even with a budget
++ * large enough to have said yes.
+ */
+ @Test
+ public void testHostileHeaderDoesNotBecomeAnOpenContainer() throws Exception {
+ ParseContext context = new ParseContext();
- context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024));
++ CacheMemoryBudget budget = new CacheMemoryBudget(1024L * 1024 * 1024);
++ context.set(CacheMemoryBudget.class, budget);
+ Metadata metadata = new Metadata();
+ try (TemporaryResources tmp = new TemporaryResources()) {
+ TikaInputStream tis = TikaInputStream.get(
+ new ByteArrayInputStream(header(3813)), tmp, metadata);
+ POIFSContainerDetector detector = new POIFSContainerDetector();
+ assertTrue(detector.detect(tis, metadata, context) != null);
+ assertNull(tis.getOpenContainer(),
+ "a header-only object must not be opened from memory");
+ }
++ assertEquals(0, budget.getReservedBytes(), "nothing was ever reserved for it");
+ }
+}