You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/09/02 20:07:24 UTC
(tika) branch TIKA-4868-performance-improvements created (now a1e91c362a)
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a change to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
at a1e91c362a TIKA-4868: tersify comments; CHANGES entries trimmed
This branch includes the following new commits:
new 92c53d17a5 TIKA-4868: improve md
new 17e67b4d31 don't double detect
new 5bdf823f0e TIKA-4868: overlap the intermediate-result ACK with the parse; add per-stage timing logs
new 92db81f1b9 TIKA-4868: single-pass buffer for CSVSniffer
new 6ad41e0f20 improve detector efficiency
new 25a2f6059e side channel content bytes for fewer encode/decode in tika-server
new 953b23251a TIKA-4868: single-pass text cleanup in WordExtractor and markdown handler
new be1b9cd877 TIKA-4868: gate rfc822 magic, cache the parser map per parse
new 967e988c59 fix zip rewind re-decompression
new 683e8fe9be TIKA-4868: bulk-span markdown text rendering
new 39d5e7f443 TIKA-4868: incremental HTML probe stripping; BMP table for block lookup
new a864bc897e TIKA-4868 fixes/reviewer feedback
new a1e91c362a TIKA-4868: tersify comments; CHANGES entries trimmed
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tika) 04/13: TIKA-4868: single-pass buffer for CSVSniffer
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 92db81f1b9370f38e63ba3199b27775f04cd2dbd
Author: tallison <ta...@apache.org>
AuthorDate: Tue Sep 1 17:33:01 2026 -0400
TIKA-4868: single-pass buffer for CSVSniffer
---
CHANGES.txt | 7 +
.../org/apache/tika/parser/csv/CSVSniffer.java | 175 +++++++++++----------
2 files changed, 100 insertions(+), 82 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index ecfea4793e..db2f14e11f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,12 @@
Release 4.1.0 - unreleased
+ * CSVSniffer reads its detection window once into a shared buffer and
+ runs every delimiter hypothesis against it, instead of re-reading the
+ stream through a pushback/mark-reset stack per delimiter; windows with
+ no delimiter and no quote skip the scan outright. Identical results
+ on a 1,125-file differential; ~28% off small-text parse time in
+ tika-server (TIKA-4868).
+
* Pipes workers no longer stall between pre-parse and parse waiting
for the client to acknowledge the intermediate-result frame; the
ACK round trip now overlaps the parse. Adds opt-in per-request
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
index ca0d8499de..164eacfd15 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
@@ -19,7 +19,6 @@ package org.apache.tika.parser.csv;
import java.io.BufferedReader;
import java.io.EOFException;
import java.io.IOException;
-import java.io.PushbackReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
@@ -28,8 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.io.input.ProxyReader;
-
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -39,7 +36,6 @@ class CSVSniffer {
static final int CARRIAGE_RETURN = '\r';
private static final int DEFAULT_MARK_LIMIT = 10000;
private static final double DEFAULT_MIN_CONFIDENCE = 0.50;
- private static final int PUSH_BACK = 2;
private static final int SPACE = ' ';
private final Set<Character> delimiters;
@@ -60,20 +56,55 @@ class CSVSniffer {
if (!reader.markSupported()) {
reader = new BufferedReader(reader);
}
+ // Every snifflet examines the same window, so read it once into a buffer
+ // instead of once per delimiter through a mark/reset + pushback stack.
+ char[] buf = new char[markLimit];
+ reader.mark(markLimit);
+ int len = 0;
+ try {
+ while (len < markLimit) {
+ int n = reader.read(buf, len, markLimit - len);
+ if (n == -1) {
+ break;
+ }
+ len += n;
+ }
+ } finally {
+ reader.reset();
+ }
+ boolean hasQuote = false;
+ for (int i = 0; i < len; i++) {
+ if (buf[i] == '"') {
+ hasQuote = true;
+ break;
+ }
+ }
List<CSVResult> ret = new ArrayList<>();
for (char delimiter : delimiters) {
- reader.mark(markLimit);
- try {
- CSVResult result = new Snifflet(delimiter).sniff(reader);
- ret.add(result);
- } finally {
- reader.reset();
+ // A window with no delimiter and no quote can only produce single-column
+ // rows: consistency and encapsulation are both 0, so the full pass is a
+ // foregone conclusion.
+ if (!hasQuote && !contains(buf, len, delimiter)) {
+ ret.add(new CSVResult(0.0,
+ delimiter == '\t' ? TextAndCSVParser.TSV : TextAndCSVParser.CSV,
+ delimiter));
+ continue;
}
+ ret.add(new Snifflet(delimiter, buf, len).sniff());
}
Collections.sort(ret);
return ret;
}
+ private static boolean contains(char[] buf, int len, char c) {
+ for (int i = 0; i < len; i++) {
+ if (buf[i] == c) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* @param reader
* @param metadata
@@ -135,28 +166,37 @@ class CSVSniffer {
//hardcode this for now
private final char quoteCharacter = '"';
+ // The shared window read once by sniff(Reader); pos is the cursor, so
+ // "unread" is a decrement and the mark-limit check is a bounds check.
+ private final char[] buf;
+ private final int len;
+ private int pos = 0;
+
Map<Integer, MutableInt> rowLengthCounts = new HashMap<>();
- int charsRead = 0;
int colCount = 0;
boolean rowZero = true;
boolean rowZeroEmpty = false;
int encapsulated = 0; //number of cells that are encapsulated in dquotes (for now)
boolean parseException = false;
+ // Cell content is never analyzed (see the unquoted() TODO that was here);
+ // only whether the current cell is non-empty matters.
+ private int unquotedLen = 0;
- public Snifflet(char delimiter) {
+ public Snifflet(char delimiter, char[] buf, int len) {
this.delimiter = delimiter;
+ this.buf = buf;
+ this.len = len;
}
- CSVResult sniff(Reader r) throws IOException {
+ CSVResult sniff() throws IOException {
boolean eof = false;
boolean hitMarkLimit = false;
int lastC = -1;
- StringBuilder unquoted = new StringBuilder();
- try (PushbackReader reader = new PushbackReader(new CloseShieldReader(r), PUSH_BACK)) {
- int c = read(reader);
+ try {
+ int c = read();
while (c != EOF) {
if (c == quoteCharacter) {
- handleUnquoted(unquoted);
+ unquotedLen = 0;
//test to make sure there isn't an unencapsulated quote character
// in the middle of a cell
if (lastC > -1 && lastC != delimiter && lastC != NEW_LINE &&
@@ -166,27 +206,27 @@ class CSVSniffer {
}
//TODO: test to make sure cell doesn't start with escaped
// ""the quick brown cat"
- boolean correctlyEncapsulated = consumeQuoted(reader, quoteCharacter);
+ boolean correctlyEncapsulated = consumeQuoted(quoteCharacter);
if (!correctlyEncapsulated) {
parseException = true;
return calcResult();
}
} else if (c == delimiter) {
- handleUnquoted(unquoted);
+ unquotedLen = 0;
endColumn();
- consumeSpaceCharacters(reader);
+ consumeSpaceCharacters();
} else if (c == NEW_LINE || c == CARRIAGE_RETURN) {
- if (unquoted.length() > 0) {
+ if (unquotedLen > 0) {
endColumn();
}
- handleUnquoted(unquoted);
+ unquotedLen = 0;
endRow();
- consumeNewLines(reader);
+ consumeNewLines();
} else {
- unquoted.append((char) c);
+ unquotedLen++;
}
lastC = c;
- c = read(reader);
+ c = read();
}
} catch (HitMarkLimitException e) {
hitMarkLimit = true;
@@ -194,19 +234,13 @@ class CSVSniffer {
//totally ignore
} catch (EOFException e) {
//the consume* throw this to avoid
- //having to check -1 every time and
- //having to rely on potentially wonky
- //inputstreams not consistently returning -1
- //after hitting EOF and returning the first -1.
- //Yes. That's a thing.
+ //having to check -1 every time
eof = true;
- } finally {
- r.reset();
}
//if you've hit the marklimit or an eof on a truncated file
//don't add the last row's info
if (!hitMarkLimit && !eof && lastC != NEW_LINE && lastC != CARRIAGE_RETURN) {
- handleUnquoted(unquoted);
+ unquotedLen = 0;
endColumn();
endRow();
}
@@ -222,22 +256,15 @@ class CSVSniffer {
return new CSVResult(confidence, mediaType, delimiter);
}
- private void handleUnquoted(StringBuilder unquoted) {
- if (unquoted.length() > 0) {
- unquoted(unquoted.toString());
- unquoted.setLength(0);
- }
- }
-
- void consumeSpaceCharacters(PushbackReader reader) throws IOException {
- int c = read(reader);
+ void consumeSpaceCharacters() throws IOException {
+ int c = read();
while (c == SPACE) {
- c = read(reader);
+ c = read();
}
if (c == EOF) {
throw new UnsurprisingEOF();
}
- unread(reader, c);
+ unread(c);
}
@@ -249,15 +276,15 @@ class CSVSniffer {
* @throws EOFException if the file ended in the middle of the encapsulated section
* @throws IOException on other IOExceptions
*/
- boolean consumeQuoted(PushbackReader reader, int quoteCharacter) throws IOException {
+ boolean consumeQuoted(int quoteCharacter) throws IOException {
//this currently assumes excel "escaping" of double quotes:
//'the " quick' -> "the "" quick"
//we can make this more interesting later with other
//escaping options
- int c = read(reader);
+ int c = read();
while (c != -1) {
if (c == quoteCharacter) {
- int nextC = read(reader);
+ int nextC = read();
if (nextC == EOF) {
encapsulated++;
endColumn();
@@ -265,58 +292,57 @@ class CSVSniffer {
} else if (nextC != quoteCharacter) {
encapsulated++;
endColumn();
- unread(reader, nextC);
- consumeSpaceCharacters(reader);
+ unread(nextC);
+ consumeSpaceCharacters();
//now make sure that the next character is eof, \r\n
//or a delimiter
- nextC = read(reader);
+ nextC = read();
if (nextC == EOF) {
throw new UnsurprisingEOF();
} else if (nextC == NEW_LINE || nextC == CARRIAGE_RETURN) {
- unread(reader, nextC);
+ unread(nextC);
return true;
} else if (nextC != delimiter) {
- unread(reader, nextC);
+ unread(nextC);
return false;
}
- unread(reader, nextC);
+ unread(nextC);
return true;
}
}
- c = read(reader);
+ c = read();
}
throw new EOFException();
}
- private int read(PushbackReader reader) throws IOException {
- if (charsRead >= markLimit - 1) {
+ private int read() throws IOException {
+ // pos tracks chars consumed exactly as charsRead did (unread decrements
+ // both), so the original off-by-one mark-limit semantics are preserved.
+ if (pos >= markLimit - 1) {
throw new HitMarkLimitException();
}
- int c = reader.read();
- if (c == EOF) {
+ if (pos >= len) {
return EOF;
}
- charsRead++;
- return c;
+ return buf[pos++];
}
- private void unread(PushbackReader reader, int c) throws IOException {
+ private void unread(int c) {
if (c != EOF) {
- reader.unread(c);
- charsRead--;
+ pos--;
}
}
//consume all consecutive '\r\n' in any order
- void consumeNewLines(PushbackReader reader) throws IOException {
- int c = read(reader);
+ void consumeNewLines() throws IOException {
+ int c = read();
while (c == NEW_LINE || c == CARRIAGE_RETURN) {
- c = read(reader);
+ c = read();
}
if (c == EOF) {
throw new EOFException();
}
- unread(reader, c);
+ unread(c);
}
@@ -340,11 +366,6 @@ class CSVSniffer {
rowZero = false;
}
- void unquoted(String string) {
- //TODO -- do some analysis to make sure you don't have
- //large tokens like 2,3,2,3,2,3,
- }
-
double getConfidence() {
double confidence = 0.0f;
@@ -400,14 +421,4 @@ class CSVSniffer {
}
- private static class CloseShieldReader extends ProxyReader {
- public CloseShieldReader(Reader r) {
- super(r);
- }
-
- @Override
- public void close() throws IOException {
- //do nothing
- }
- }
}
(tika) 08/13: TIKA-4868: gate rfc822 magic, cache the parser map per parse
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit be1b9cd877dcde4dc4fb9d41cb64661fbb2556d4
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 08:07:41 2026 -0400
TIKA-4868: gate rfc822 magic, cache the parser map per parse
---
CHANGES.txt | 12 +++++++++
.../java/org/apache/tika/detect/MagicDetector.java | 31 +++++++++++++++++-----
.../org/apache/tika/parser/CompositeParser.java | 30 ++++++++++++++++++++-
.../org/apache/tika/mime/tika-mimetypes.xml | 5 ++++
.../tika/server/core/resource/TikaResource.java | 31 +++++++++++++++++++++-
5 files changed, 100 insertions(+), 9 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index dee363d60b..0e6e1ad587 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,17 @@
Release 4.1.0 - unreleased
+ * More detection/dispatch savings: the message/rfc822 priority-45 magic
+ is gated behind a ':' scan of the first 30 bytes (every match provably
+ has one, so results are unchanged and non-mail text skips its ~40
+ clauses); CompositeParser caches the built type->parser map in the
+ ParseContext, so embedded documents reuse the container's map instead
+ of rebuilding it per document. Plain-text detection drops from 334us
+ (4.1.0-dev ran it twice) to 52us per document; eml requests ~-45%,
+ ppt ~-35% in tika-server. MagicDetector also precomputes a 256-entry
+ first-byte table per pattern (mask and case fold become one array
+ load per scanned position). Adds an opt-in RESOURCE_TIMING log on
+ org.apache.tika.pipes.timing.resource (TIKA-4868).
+
* WordExtractor (.doc) cleans each character run in one pass instead of
four chained replace/replaceAll copies, and tests paragraph blankness
without a regex replaceAll; ToMarkdownContentHandler collapses line
diff --git a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
index 839b722c3a..d9d6b2ad2d 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
@@ -468,6 +468,23 @@ public class MagicDetector implements Detector {
* @param endOffset the last position in the buffer to start matching (inclusive)
* @return true if a match is found, false otherwise
*/
+ // Which raw first bytes can begin a match; encodes mask[0] and the case fold so the
+ // scan loop is a single table load. Built lazily, idempotent under racing builds.
+ private transient volatile boolean[] firstByteMatches;
+
+ private boolean[] buildFirstByteTable() {
+ boolean[] table = new boolean[256];
+ int first = pattern[0];
+ for (int b = 0; b < 256; b++) {
+ int masked = ((byte) b) & mask[0];
+ if (this.isStringIgnoreCase) {
+ masked = Character.toLowerCase(masked);
+ }
+ table[b] = (masked == first);
+ }
+ return table;
+ }
+
private boolean matchesBuffer(byte[] buffer, int startOffset, int endOffset) {
if (this.isRegex) {
int bufferLen = Math.min(buffer.length - startOffset, length + (endOffset - startOffset));
@@ -497,17 +514,17 @@ public class MagicDetector implements Detector {
// degenerate empty pattern: preserves the old loop's outcome
return startOffset <= endOffset && startOffset <= buffer.length;
}
- int first = pattern[0];
- byte firstMask = mask[0];
+ // one array load per scanned position instead of mask + case-fold + compare
+ boolean[] firstMatch = firstByteMatches;
+ if (firstMatch == null) {
+ firstMatch = buildFirstByteTable();
+ firstByteMatches = firstMatch;
+ }
for (int i = startOffset; i <= endOffset; i++) {
if (i + length > buffer.length) {
break;
}
- int masked0 = buffer[i] & firstMask;
- if (this.isStringIgnoreCase) {
- masked0 = Character.toLowerCase(masked0);
- }
- if (masked0 != first) {
+ if (!firstMatch[buffer[i] & 0xFF]) {
continue;
}
boolean match = true;
diff --git a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
index fe198a75b5..4e97d96fc0 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -245,8 +246,35 @@ public class CompositeParser implements Parser {
return getParser(metadata, new ParseContext());
}
+ /**
+ * Per-parse cache of built parser maps, stored in the ParseContext so every
+ * embedded document in a parse reuses the container's map instead of
+ * rebuilding it (a full walk of every parser's supported types). Keyed by
+ * parser instance because nested composites share one context. The map is
+ * built once per (parser, context); a context entry that would change a
+ * parser's supported types mid-parse is not picked up until the next parse.
+ */
+ private static final class ParserMapCache {
+ private final Map<CompositeParser, Map<MediaType, Parser>> maps =
+ new IdentityHashMap<>();
+ }
+
+ private Map<MediaType, Parser> getParsersCached(ParseContext context) {
+ ParserMapCache cache = context.get(ParserMapCache.class);
+ if (cache == null) {
+ cache = new ParserMapCache();
+ context.set(ParserMapCache.class, cache);
+ }
+ Map<MediaType, Parser> map = cache.maps.get(this);
+ if (map == null) {
+ map = getParsers(context);
+ cache.maps.put(this, map);
+ }
+ return map;
+ }
+
protected Parser getParser(Metadata metadata, ParseContext context) {
- Map<MediaType, Parser> map = getParsers(context);
+ Map<MediaType, Parser> map = getParsersCached(context);
//check for parser override first
String contentTypeString = metadata.get(TikaCoreProperties.CONTENT_TYPE_PARSER_OVERRIDE);
if (contentTypeString == null) {
diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index 9cb4a899bc..bd98b2ae40 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -7447,6 +7447,10 @@
</match>
</magic>
<magic priority="45">
+ <!-- gate: the first branch below requires a known header at offset 0 (or after a
+ BOM), and every such header puts a ':' within the first 30 bytes, so files
+ without an early colon skip the ~40-clause evaluation entirely -->
+ <match value=":" type="string" offset="0:29">
<!-- be a bit more flexible, but require one from each of these -->
<match minShouldMatch="2">
@@ -7515,6 +7519,7 @@
<match value="\nARC-" type="string" offset="0:1024"/>
</match>
</match>
+ </match>
</magic>
<magic priority="40">
<!-- lower priority than message/news -->
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
index eca7b938c4..ab48d46e85 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
@@ -700,13 +700,18 @@ public class TikaResource {
/**
* Produces raw streaming output (text, html, xml, md) using pipes-based parsing.
*/
+ /** Per-request resource-layer latency; joins the pipes lines by adjacency at c=1. */
+ private static final org.slf4j.Logger TIMING_LOG =
+ org.slf4j.LoggerFactory.getLogger("org.apache.tika.pipes.timing.resource");
+
private Response produceRawOutput(TikaInputStream tis, Metadata metadata,
MultivaluedMap<String, String> httpHeaders,
String handlerTypeName) throws IOException {
+ long entryNanos = System.nanoTime();
fillMetadata(null, metadata, httpHeaders);
ParseContext context = createRequestContext();
setupContentHandlerFactory(context, handlerTypeName);
- return produceRawOutputWithContext(tis, metadata, context, handlerTypeName);
+ return produceRawOutputWithContext(tis, metadata, context, handlerTypeName, entryNanos);
}
/**
@@ -718,6 +723,14 @@ public class TikaResource {
private Response produceRawOutputWithContext(TikaInputStream tis, Metadata metadata,
ParseContext context,
String handlerTypeName) throws IOException {
+ return produceRawOutputWithContext(tis, metadata, context, handlerTypeName,
+ System.nanoTime());
+ }
+
+ private Response produceRawOutputWithContext(TikaInputStream tis, Metadata metadata,
+ ParseContext context,
+ String handlerTypeName, long entryNanos)
+ throws IOException {
logRequest(LOG, "/tika", metadata);
// Ensure content handler factory is set (config may have set it)
@@ -729,8 +742,10 @@ public class TikaResource {
// Parse with pipes using CONTENT_ONLY mode - the metadata filter in
// EmitHandler will strip everything except tk:content, and the content comes
// back as raw UTF-8 bytes rather than a Smile-encoded string
+ long parseStartNanos = System.nanoTime();
PipesParsingHelper.ParseOutput parsed =
parseWithPipesRaw(tis, metadata, context);
+ long parseEndNanos = System.nanoTime();
List<Metadata> metadataList = parsed.metadataList();
LOG.debug("produceRawOutput: parseWithPipes returned {} metadata objects", metadataList.size());
@@ -760,9 +775,23 @@ public class TikaResource {
// container exception should use /rmeta.
final byte[] finalContent = content == null ? new byte[0] : content;
+ final long buildEndNanos = System.nanoTime();
StreamingOutput streamingOutput = outputStream -> {
+ long writeStart = System.nanoTime();
outputStream.write(finalContent);
outputStream.flush();
+ if (TIMING_LOG.isInfoEnabled()) {
+ // pre = header/context setup before the pipes call; build = result
+ // unpacking after it; write = streaming the body (invoked later by the
+ // JAX-RS runtime, so anything left over vs the client-observed total is
+ // the HTTP stack itself)
+ TIMING_LOG.info("RESOURCE_TIMING pre_us={} pipes_us={} build_us={} write_us={} bytes={}",
+ (parseStartNanos - entryNanos) / 1000,
+ (parseEndNanos - parseStartNanos) / 1000,
+ (buildEndNanos - parseEndNanos) / 1000,
+ (System.nanoTime() - writeStart) / 1000,
+ finalContent.length);
+ }
};
return Response.status(hasException ? 422 : Response.Status.OK.getStatusCode())
.entity(streamingOutput)
(tika) 09/13: fix zip rewind re-decompression
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 967e988c59b956d923b9ffdbf1b0c4eaac86a05b
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 08:11:29 2026 -0400
fix zip rewind re-decompression
---
CHANGES.txt | 8 ++++++++
.../main/java/org/apache/tika/parser/pkg/ZipParser.java | 14 +++++++++++---
2 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 0e6e1ad587..7744bb009c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,13 @@
Release 4.1.0 - unreleased
+ * ZipParser no longer re-decompresses an entry on every rewind when the
+ entry uses a legacy compression method (implode, shrink, bzip2, ...):
+ those decode per-bit, so the rewind-by-reopen trade that works for
+ STORED/DEFLATED entries re-decoded the whole entry for each of the
+ several rewinds the embedded pipeline performs. Such entries now
+ replay from the budgeted cache. An imploded 606KB zip drops from
+ 610ms to 102ms in tika-server (TIKA-4868).
+
* More detection/dispatch savings: the message/rfc822 priority-45 magic
is gated behind a ':' scan of the first 30 bytes (every match provably
has one, so results are unchanged and non-mail text skips its ~40
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
index 58e703d79d..405bb314bb 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
@@ -497,9 +497,17 @@ public class ZipParser extends AbstractArchiveParser {
if (extractor.shouldParseEmbedded(entryMetadata, context)) {
TemporaryResources tmp = new TemporaryResources();
// Re-openable source: rewind (e.g. for digesting) re-opens the entry instead of
- // buffering/spilling it
- try (TikaInputStream tis = TikaInputStream.get(
- () -> zipFile.getInputStream(entry), tmp, entryMetadata)) {
+ // buffering/spilling it. That trade only holds when re-opening is cheap --
+ // STORED and DEFLATED. The legacy methods (implode, shrink, bzip2, ...) decode
+ // per-bit, so a re-open re-decompresses the whole entry for every rewind;
+ // those route through the budgeted cache instead (memory up to the
+ // CacheMemoryBudget, temp spill beyond it).
+ int method = entry.getMethod();
+ boolean cheapReopen = method == java.util.zip.ZipEntry.STORED
+ || method == java.util.zip.ZipEntry.DEFLATED;
+ try (TikaInputStream tis = cheapReopen
+ ? TikaInputStream.get(() -> zipFile.getInputStream(entry), tmp, entryMetadata)
+ : TikaInputStream.get(zipFile.getInputStream(entry), tmp, entryMetadata)) {
extractor.parseEmbedded(tis, xhtml, entryMetadata, context, true);
} catch (UnsupportedZipFeatureException e) {
EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
(tika) 11/13: TIKA-4868: incremental HTML probe stripping; BMP table for block lookup
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 39d5e7f44384d69501b5f8481fcbc5266102b81c
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 08:41:36 2026 -0400
TIKA-4868: incremental HTML probe stripping; BMP table for block lookup
---
CHANGES.txt | 10 ++
.../apache/tika/ml/chardetect/AdaptiveProbe.java | 12 ++-
.../tika/ml/chardetect/HtmlByteStripper.java | 115 ++++++++++++++++----
.../ml/chardetect/HtmlByteStripperResumeTest.java | 117 +++++++++++++++++++++
.../tika/ml/junkdetect/UnicodeBlockRanges.java | 20 ++++
.../tika/ml/junkdetect/UnicodeBlockRangesTest.java | 14 +++
6 files changed, 263 insertions(+), 25 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index c2b46bd0ac..f125119319 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,15 @@
Release 4.1.0 - unreleased
+ * Mojibuster's adaptive probe strips incrementally instead of
+ re-stripping the whole buffer on every read (quadratic in read count
+ on tag-heavy pages; a 293KB page's probe drops 4.5ms -> 0.8ms).
+ Byte-identical: the resumable scanner holds back at a chunk-final
+ "<!" so chunked == single-shot at every boundary (exhaustive
+ boundary tests), and detection output was unchanged across a
+ 3,034-file differential. JunkDetector's Unicode block lookup also
+ uses a precomputed BMP table (exhaustive equivalence test)
+ (TIKA-4868).
+
* Markdown rendering is another ~8x faster on large documents:
a custom Text-node renderer emits unescaped spans in bulk instead of
commonmark's per-character escape-check-and-append (9MB render
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/AdaptiveProbe.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/AdaptiveProbe.java
index e2100b0533..f9aa049890 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/AdaptiveProbe.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/AdaptiveProbe.java
@@ -61,6 +61,10 @@ public final class AdaptiveProbe {
int cap = Math.min(rawCap, contentTarget);
byte[] buf = new byte[cap];
byte[] stripDst = new byte[cap];
+ // Resumable strip: each byte is scanned once. Re-stripping from offset 0
+ // per read made this loop quadratic in read count on tag-heavy pages.
+ // Absolute cursor positions stay valid across the copyOf growth below.
+ HtmlByteStripper.Cursor cursor = new HtmlByteStripper.Cursor();
int total = 0;
while (total < rawCap) {
int want = Math.min(rawCap - total, contentTarget);
@@ -71,10 +75,10 @@ public final class AdaptiveProbe {
}
int n = IOUtils.read(tis, buf, total, want);
total += n;
- HtmlByteStripper.Result r =
- HtmlByteStripper.stripTags(buf, 0, total, stripDst, 0);
- int content = r.tagCount > 0 ? r.length : total;
- if (content >= contentTarget || n < want) {
+ boolean eof = n < want;
+ HtmlByteStripper.stripTags(buf, total, stripDst, cursor, eof);
+ int content = cursor.tagCount() > 0 ? cursor.contentLength() : total;
+ if (content >= contentTarget || eof) {
break; // enough body text, or EOF
}
}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/HtmlByteStripper.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/HtmlByteStripper.java
index 37ae5ad26b..9b2aa44937 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/HtmlByteStripper.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/HtmlByteStripper.java
@@ -184,27 +184,93 @@ public final class HtmlByteStripper {
return strip(src, srcOffset, srcLen, dst, dstOffset, false);
}
+ /**
+ * Scan state carried across chunked {@link #stripTags(byte[], int, byte[], Cursor,
+ * boolean)} calls. All positions are absolute indices into the caller's (growing)
+ * source buffer, so the buffer must retain its prefix across calls (e.g. grow with
+ * {@code Arrays.copyOf}). Tags-only: the resumable path never drops entities.
+ */
+ static final class Cursor {
+ private int state = TEXT;
+ private int nameStart;
+ private byte[] rawEnd;
+ private int rawMatch;
+ private int tagCount;
+ private int entityCount;
+ private int entityStart;
+ private int attrNameStart;
+ private boolean emitAttrValue;
+ private int written;
+ private int next;
+ // where the very first scan started; guards the comment lookbehind
+ private int origin;
+
+ int contentLength() {
+ return written;
+ }
+
+ int tagCount() {
+ return tagCount;
+ }
+ }
+
+ /**
+ * Resumable {@link #stripTags(byte[], int, int, byte[], int)}: scans
+ * {@code src[cursor.next .. end)}, appending stripped output at
+ * {@code dst[cursor.written]}. Byte-identical to a single-shot strip of the final
+ * buffer: on a non-final call the scan holds back at a {@code <!} too close to
+ * {@code end} for the comment lookahead, so the next call re-examines it with more
+ * bytes; everything else in the machine is strictly left-to-right.
+ *
+ * @param isFinal true on the last call (no more input will arrive)
+ */
+ static void stripTags(byte[] src, int end, byte[] dst, Cursor cursor, boolean isFinal) {
+ scan(src, end, dst, cursor, false, isFinal);
+ }
+
private static Result strip(byte[] src, int srcOffset, int srcLen,
byte[] dst, int dstOffset, boolean dropEntities) {
- int w = dstOffset;
- int state = TEXT;
- int nameStart = 0;
- byte[] rawEnd = null;
- int rawMatch = 0;
+ Cursor c = new Cursor();
+ c.next = srcOffset;
+ c.origin = srcOffset;
+ c.written = dstOffset;
int end = srcOffset + srcLen;
- int tagCount = 0;
- int entityCount = 0;
+ scan(src, end, dst, c, dropEntities, true);
+ int w = c.written;
+
+ // Unterminated entity at EOF: emit consumed prefix as literal text.
+ if (c.state == ENTITY || c.state == ENTITY_NAME || c.state == ENTITY_NUM
+ || c.state == ENTITY_DEC || c.state == ENTITY_HEX) {
+ for (int k = c.entityStart; k < end; k++) {
+ dst[w++] = src[k];
+ }
+ }
+
+ return new Result(w - dstOffset, c.tagCount, c.entityCount);
+ }
+
+ private static void scan(byte[] src, int end, byte[] dst, Cursor c,
+ boolean dropEntities, boolean isFinal) {
+ int w = c.written;
+ int state = c.state;
+ int nameStart = c.nameStart;
+ byte[] rawEnd = c.rawEnd;
+ int rawMatch = c.rawMatch;
+ int tagCount = c.tagCount;
+ int entityCount = c.entityCount;
// Position of the leading '&' for the in-progress entity.
// Tracked so the bailout path can emit the consumed prefix
// as literal text when the parse fails (e.g. "AT&T").
- int entityStart = 0;
- int attrNameStart = 0;
+ int entityStart = c.entityStart;
+ int attrNameStart = c.attrNameStart;
// When true, the current quoted attribute value's bytes are
// emitted to dst (attribute name matched TEXT_ATTRS). Reset
// to false when the quote closes or the tag ends.
- boolean emitAttrValue = false;
+ boolean emitAttrValue = c.emitAttrValue;
- for (int i = srcOffset; i < end; i++) {
+ int i = c.next;
+ outer:
+ for (; i < end; i++) {
byte b = src[i];
switch (state) {
case TEXT:
@@ -219,6 +285,11 @@ public final class HtmlByteStripper {
break;
case LT:
+ if (b == '!' && i + 2 >= end && !isFinal) {
+ // comment lookahead needs 2 more bytes: hold back so the next
+ // call re-examines '!' -- keeps chunked == single-shot
+ break outer;
+ }
if (b == '!' && i + 2 < end
&& src[i + 1] == '-' && src[i + 2] == '-') {
state = COMMENT;
@@ -474,7 +545,7 @@ public final class HtmlByteStripper {
break;
case COMMENT:
- if (b == '>' && i >= srcOffset + 2
+ if (b == '>' && i >= c.origin + 2
&& src[i - 1] == '-' && src[i - 2] == '-') {
state = TEXT;
}
@@ -508,15 +579,17 @@ public final class HtmlByteStripper {
}
}
- // Unterminated entity at EOF: emit consumed prefix as literal text.
- if (state == ENTITY || state == ENTITY_NAME || state == ENTITY_NUM
- || state == ENTITY_DEC || state == ENTITY_HEX) {
- for (int k = entityStart; k < end; k++) {
- dst[w++] = src[k];
- }
- }
-
- return new Result(w - dstOffset, tagCount, entityCount);
+ c.state = state;
+ c.nameStart = nameStart;
+ c.rawEnd = rawEnd;
+ c.rawMatch = rawMatch;
+ c.tagCount = tagCount;
+ c.entityCount = entityCount;
+ c.entityStart = entityStart;
+ c.attrNameStart = attrNameStart;
+ c.emitAttrValue = emitAttrValue;
+ c.written = w;
+ c.next = i;
}
/**
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/HtmlByteStripperResumeTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/HtmlByteStripperResumeTest.java
new file mode 100644
index 0000000000..77a2508a21
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/HtmlByteStripperResumeTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Chunked stripTags with a carried Cursor must be byte-identical to the single-shot
+ * strip of the final buffer, for every chunking -- including boundaries inside tags,
+ * comments (the {@code <!} lookahead hold-back), raw script/style bodies, quoted
+ * attribute values, and multibyte UTF-8 sequences.
+ */
+public class HtmlByteStripperResumeTest {
+
+ private static final String[] SAMPLES = {
+ "plain text, no markup at all — even multibyte: čšž 中文 🚀",
+ "<html><head><title>t</title></head><body>Hello <b>world</b></body></html>",
+ "before<!-- a comment with <tags> inside -->after",
+ "x<!-- unterminated comment...",
+ "a<!DOCTYPE html>b<?xml version=\"1.0\"?>c",
+ "<script>var a = '<div>'; // not a tag </script>text<style>.x{}</style>tail",
+ "<img alt=\"seen text\" src=\"nope.png\" title='also seen'>body",
+ "AT&T & friends A B ¬anentity; &unterminated",
+ "stray < less-than and << double and <3 hearts",
+ "<a href=\"x\">link</a><ul><li>item</li></ul>",
+ "<div class='q' aria-label=\"read me\">deep</div>",
+ "čšž<em>中文</em>🚀<!--中-->done",
+ "text</scripted>more<script>raw</script>end",
+ "<!",
+ "<!-",
+ "<!--",
+ "<!-->",
+ "<!---->tail",
+ };
+
+ @Test
+ public void testEveryBoundaryEqualsSingleShot() {
+ for (String sample : SAMPLES) {
+ byte[] src = sample.getBytes(StandardCharsets.UTF_8);
+ Expected want = singleShot(src);
+ // every single split point, and every pair of split points
+ for (int cut = 0; cut <= src.length; cut++) {
+ check(src, new int[]{cut}, want, sample);
+ }
+ for (int c1 = 0; c1 <= src.length; c1 += 3) {
+ for (int c2 = c1; c2 <= src.length; c2 += 3) {
+ check(src, new int[]{c1, c2}, want, sample);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testRandomChunkingsOnConcatenatedSamples() {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 50; i++) {
+ sb.append(SAMPLES[i % SAMPLES.length]);
+ }
+ byte[] src = sb.toString().getBytes(StandardCharsets.UTF_8);
+ Expected want = singleShot(src);
+ Random random = new Random(42);
+ for (int trial = 0; trial < 200; trial++) {
+ int nCuts = 1 + random.nextInt(8);
+ int[] cuts = new int[nCuts];
+ for (int i = 0; i < nCuts; i++) {
+ cuts[i] = random.nextInt(src.length + 1);
+ }
+ Arrays.sort(cuts);
+ check(src, cuts, want, "concat trial " + trial);
+ }
+ }
+
+ private record Expected(byte[] out, int tagCount) {
+ }
+
+ private static Expected singleShot(byte[] src) {
+ byte[] dst = new byte[src.length + 16];
+ HtmlByteStripper.Result r = HtmlByteStripper.stripTags(src, 0, src.length, dst, 0);
+ return new Expected(Arrays.copyOf(dst, r.length), r.tagCount);
+ }
+
+ private static void check(byte[] src, int[] cuts, Expected want, String label) {
+ byte[] dst = new byte[src.length + 16];
+ HtmlByteStripper.Cursor cursor = new HtmlByteStripper.Cursor();
+ int prev = 0;
+ for (int cut : cuts) {
+ int end = Math.max(prev, cut);
+ HtmlByteStripper.stripTags(src, end, dst, cursor, false);
+ prev = end;
+ }
+ HtmlByteStripper.stripTags(src, src.length, dst, cursor, true);
+ assertEquals(want.tagCount(), cursor.tagCount(), label + " cuts=" + Arrays.toString(cuts));
+ assertArrayEquals(want.out(), Arrays.copyOf(dst, cursor.contentLength()),
+ label + " cuts=" + Arrays.toString(cuts));
+ }
+}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/UnicodeBlockRanges.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/UnicodeBlockRanges.java
index ab7e1b00b7..6eb0dc32c3 100644
--- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/UnicodeBlockRanges.java
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/UnicodeBlockRanges.java
@@ -399,11 +399,24 @@ public final class UnicodeBlockRanges {
/** Cached start_cp array for binary search. */
private static final int[] STARTS;
+
+ /**
+ * Precomputed bucket per BMP codepoint (128KB): bucketOf runs per character of
+ * every scored text, and the binary search was a visible share of junk-detection
+ * time. Filled from {@link #searchBucketOf(int)}, so the answers are identical
+ * by construction; supplementary planes stay on the search path.
+ */
+ private static final short[] BMP_BUCKETS;
+
static {
STARTS = new int[RANGES.length];
for (int i = 0; i < RANGES.length; i++) {
STARTS[i] = RANGES[i][0];
}
+ BMP_BUCKETS = new short[0x10000];
+ for (int cp = 0; cp < 0x10000; cp++) {
+ BMP_BUCKETS[cp] = (short) searchBucketOf(cp);
+ }
}
private UnicodeBlockRanges() {
@@ -423,6 +436,13 @@ public final class UnicodeBlockRanges {
* O(log N) where N = {@value #UNASSIGNED} (the number of named blocks).
*/
public static int bucketOf(int cp) {
+ if (cp >= 0 && cp < 0x10000) {
+ return BMP_BUCKETS[cp];
+ }
+ return searchBucketOf(cp);
+ }
+
+ private static int searchBucketOf(int cp) {
// Binary search: find largest STARTS[i] <= cp
int lo = 0;
int hi = STARTS.length - 1;
diff --git a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnicodeBlockRangesTest.java b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnicodeBlockRangesTest.java
index e25cff9204..22576a1a77 100644
--- a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnicodeBlockRangesTest.java
+++ b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnicodeBlockRangesTest.java
@@ -111,4 +111,18 @@ public class UnicodeBlockRangesTest {
assertTrue(seen[b], "Bucket id " + b + " is never produced by any codepoint");
}
}
+
+ @org.junit.jupiter.api.Test
+ public void testBmpTableMatchesSearchEverywhere() throws Exception {
+ // the BMP fast table must agree with the binary search for every codepoint,
+ // including plane boundaries and a sweep of the supplementary planes
+ java.lang.reflect.Method search =
+ UnicodeBlockRanges.class.getDeclaredMethod("searchBucketOf", int.class);
+ search.setAccessible(true);
+ for (int cp = 0; cp <= 0x10FFFF; cp += (cp < 0x10000 ? 1 : 17)) {
+ org.junit.jupiter.api.Assertions.assertEquals(
+ (int) (Integer) search.invoke(null, cp),
+ UnicodeBlockRanges.bucketOf(cp), "cp=" + Integer.toHexString(cp));
+ }
+ }
}
(tika) 03/13: TIKA-4868: overlap the intermediate-result ACK with the parse; add per-stage timing logs
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 5bdf823f0e291b14df2c9d07eca0b9f4e0d6e624
Author: tallison <ta...@apache.org>
AuthorDate: Tue Sep 1 17:24:40 2026 -0400
TIKA-4868: overlap the intermediate-result ACK with the parse; add per-stage timing logs
---
CHANGES.txt | 5 ++
.../org/apache/tika/pipes/core/PipesClient.java | 69 ++++++++++++++++++++++
.../tika/pipes/core/server/ConnectionHandler.java | 61 ++++++++++++++++++-
.../apache/tika/pipes/core/server/PipesServer.java | 68 ++++++++++++++++++++-
.../apache/tika/pipes/core/server/PipesWorker.java | 48 ++++++++++++++-
.../tika/pipes/core/server/ServerProtocolIO.java | 66 ++++++++++++++++++++-
.../server/core/resource/PipesParsingHelper.java | 33 +++++++++++
7 files changed, 343 insertions(+), 7 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 8c2b56af66..ecfea4793e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,10 @@
Release 4.1.0 - unreleased
+ * Pipes workers no longer stall between pre-parse and parse waiting
+ for the client to acknowledge the intermediate-result frame; the
+ ACK round trip now overlaps the parse. Adds opt-in per-request
+ timing logs on org.apache.tika.pipes.timing.* (TIKA-4868).
+
* DefaultDetector honors CONTENT_TYPE_USER_OVERRIDE and
CONTENT_TYPE_PARSER_OVERRIDE before running magic detection, matching
CompositeDetector's contract. Removes the second full magic scan every
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesClient.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesClient.java
index 3988f9c90d..cb21f1deba 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesClient.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesClient.java
@@ -67,6 +67,10 @@ import org.apache.tika.utils.StringUtils;
public class PipesClient implements Closeable {
private static final Logger LOG = LoggerFactory.getLogger(PipesClient.class);
+ /** Per-parse latency breakdown; one key=value line per parse. Route or silence
+ * independently of {@link #LOG}. Diagnostic only -- off unless INFO is enabled. */
+ private static final Logger TIMING_LOG =
+ LoggerFactory.getLogger("org.apache.tika.pipes.timing.client");
private static final AtomicInteger CLIENT_COUNTER = new AtomicInteger(0);
public static final int SOCKET_CONNECT_TIMEOUT_MS = 60000;
public static final int SOCKET_TIMEOUT_MILLIS = 60000;
@@ -91,6 +95,17 @@ public class PipesClient implements Closeable {
private volatile long connectionGeneration = Long.MAX_VALUE;
private int filesProcessed = 0;
+ // Per-parse timing scratch. A PipesClient serves one parse at a time (the pool hands
+ // out an exclusive client), so plain fields are safe and avoid per-parse allocation.
+ private long tInitNanos;
+ private long tReqSerNanos;
+ private long tReqWriteNanos;
+ private long tFirstFrameNanos;
+ private long tFinishedNanos;
+ private long tRespDeserNanos;
+ private long tAckNanos;
+ private int tFrames;
+
/**
* Creates a PipesClient with the given server manager.
* <p>
@@ -214,8 +229,12 @@ public class PipesClient implements Closeable {
// Container object to hold latest intermediate result if the parser is doing that
IntermediateResult intermediateResult = new IntermediateResult();
PipesResult result = null;
+ long callStart = System.nanoTime();
+ resetTimings();
+ long initStart = System.nanoTime();
try {
maybeInit();
+ tInitNanos = System.nanoTime() - initStart;
} catch (InterruptedException e) {
// Same invariant as the in-flight path below: an abandoned connection,
// here possibly half-established, must not be re-queued, and an
@@ -277,9 +296,43 @@ public class PipesClient implements Closeable {
closeConnection();
return buildFatalResult(t.getId(), t.getEmitKey(), UNSPECIFIED_CRASH, intermediateResult.get());
}
+ logTiming(t.getId(), result, System.nanoTime() - callStart);
return result;
}
+ private void resetTimings() {
+ tInitNanos = 0;
+ tReqSerNanos = 0;
+ tReqWriteNanos = 0;
+ tFirstFrameNanos = 0;
+ tFinishedNanos = 0;
+ tRespDeserNanos = 0;
+ tAckNanos = 0;
+ tFrames = 0;
+ }
+
+ /**
+ * One line per parse on {@code org.apache.tika.pipes.timing.client}, microseconds.
+ * {@code finished_us} is measured from the end of the request write to the moment the
+ * FINISHED frame is fully read, so it contains the worker's own work plus everything
+ * the response leg costs; the worker logs its side under the same {@code id}.
+ */
+ private void logTiming(String id, PipesResult result, long totalNanos) {
+ if (!TIMING_LOG.isInfoEnabled()) {
+ return;
+ }
+ TIMING_LOG.info("CLIENT_TIMING client={} id={} status={} init_us={} req_ser_us={}"
+ + " req_write_us={} first_frame_us={} finished_us={} resp_deser_us={}"
+ + " ack_us={} frames={} total_us={}",
+ pipesClientId, id, result == null ? "NULL" : result.status().name(),
+ us(tInitNanos), us(tReqSerNanos), us(tReqWriteNanos), us(tFirstFrameNanos),
+ us(tFinishedNanos), us(tRespDeserNanos), us(tAckNanos), tFrames, us(totalNanos));
+ }
+
+ private static long us(long nanos) {
+ return nanos < 0 ? nanos : nanos / 1000L;
+ }
+
private void maybeInit() throws InterruptedException, ServerInitializationException {
boolean reconnect = false;
@@ -375,7 +428,9 @@ public class PipesClient implements Closeable {
throw new IOException("connection closed");
}
LOG.debug("pipesClientId={}: sending NEW_REQUEST for id={}", pipesClientId, t.getId());
+ long serStart = System.nanoTime();
byte[] bytes = JsonPipesIpc.toBytes(PipesRequest.of(t));
+ tReqSerNanos = System.nanoTime() - serStart;
// Fail fast before sending: the server would refuse the frame anyway, but only by
// dying or dropping the connection, misreported as a crash.
if (bytes.length > maxIpcPayloadBytes) {
@@ -383,7 +438,9 @@ public class PipesClient implements Closeable {
+ " is " + bytes.length + " bytes, over maxIpcPayloadBytes="
+ maxIpcPayloadBytes + "; raise maxIpcPayloadBytes or shrink the request");
}
+ long writeStart = System.nanoTime();
PipesMessage.newRequest(bytes).write(tuple.output);
+ tReqWriteNanos = System.nanoTime() - writeStart;
}
/**
@@ -420,6 +477,9 @@ public class PipesClient implements Closeable {
long clientBackstopMillis = clientBackstopMillis(limits);
// nanoTime: the backstop must be immune to wall-clock steps
long startNanos = System.nanoTime();
+ // Separate anchor for the timing log: the backstop's is conceptually the deadline
+ // clock, and conflating them would silently break if either moves.
+ final long respAnchor = startNanos;
while (true) {
if (Thread.currentThread().isInterrupted()) {
@@ -436,11 +496,17 @@ public class PipesClient implements Closeable {
}
try {
PipesMessage msg = PipesMessage.read(tuple.input, maxIpcPayloadBytes);
+ long frameReadAt = System.nanoTime() - respAnchor;
+ if (++tFrames == 1) {
+ tFirstFrameNanos = frameReadAt;
+ }
LOG.trace("clientId={}: received message type={} id={}", pipesClientId, msg.type(), t.getId());
// Send ACK only for messages that require it
if (msg.type().requiresAck()) {
+ long ackStart = System.nanoTime();
PipesMessage.ack().write(tuple.output);
+ tAckNanos += System.nanoTime() - ackStart;
}
switch (msg.type()) {
@@ -470,7 +536,10 @@ public class PipesClient implements Closeable {
// is what keeps the blocking read below from hitting SO_TIMEOUT.
break;
case FINISHED:
+ tFinishedNanos = frameReadAt;
+ long deserStart = System.nanoTime();
PipesResult result = JsonPipesIpc.fromBytes(msg.payload(), PipesResult.class);
+ tRespDeserNanos = System.nanoTime() - deserStart;
// Restore ParseContext from original FetchEmitTuple (not serialized back from server)
if (result.emitData() instanceof EmitDataImpl emitDataImpl) {
emitDataImpl.setParseContext(t.getParseContext());
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
index a1755be2ab..a211829b15 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
@@ -82,6 +82,17 @@ public class ConnectionHandler implements Runnable, Closeable {
private final long heartbeatIntervalMillis;
private final ExecutorService executorService = Executors.newSingleThreadExecutor();
+ /** Per-parse worker-side latency breakdown; joins the client line on {@code id}. */
+ private static final Logger TIMING_LOG =
+ LoggerFactory.getLogger("org.apache.tika.pipes.timing.worker");
+
+ // Per-request timing scratch. One request at a time per connection.
+ private long tReqDeserNanos = -1;
+ private long tCtxMergeNanos = -1;
+ private long tSubmitAtNanos = -1;
+ private long tHandoffNanos = -1;
+ private PipesWorker tLastWorker;
+
private final ExecutorCompletionService<PipesResult> executorCompletionService =
new ExecutorCompletionService<>(executorService);
@@ -129,6 +140,42 @@ public class ConnectionHandler implements Runnable, Closeable {
}
}
+ private void resetTimings() {
+ tReqDeserNanos = -1;
+ tCtxMergeNanos = -1;
+ tSubmitAtNanos = -1;
+ tHandoffNanos = -1;
+ tLastWorker = null;
+ protocolIO.resetLastTimings();
+ }
+
+ /**
+ * One line per parse on {@code org.apache.tika.pipes.timing.worker}, microseconds.
+ * {@code handoff_us} is the executor scheduling plus completion-poll latency around the
+ * worker; {@code resp_*} is the FINISHED frame's serialize, socket write, and the
+ * client-ACK wait that follows it.
+ */
+ private void logTiming(String id) {
+ if (!TIMING_LOG.isInfoEnabled()) {
+ return;
+ }
+ PipesWorker w = tLastWorker;
+ TIMING_LOG.info("WORKER_TIMING handler={} id={} req_deser_us={} ctx_merge_us={}"
+ + " handoff_us={} fetch_us={} parse_us={} emit_us={} worker_wall_us={}"
+ + " intermediate_us={} resp_ser_us={} resp_write_us={} resp_ack_us={}"
+ + " resp_bytes={}",
+ handlerId, id, us(tReqDeserNanos), us(tCtxMergeNanos), us(tHandoffNanos),
+ us(w == null ? -1 : w.getFetchNanos()), us(w == null ? -1 : w.getParseNanos()),
+ us(w == null ? -1 : w.getEmitNanos()), us(w == null ? -1 : w.getWallNanos()),
+ us(protocolIO.getLastIntermediateNanos()), us(protocolIO.getLastRespSerNanos()),
+ us(protocolIO.getLastRespWriteNanos()), us(protocolIO.getLastRespAckNanos()),
+ protocolIO.getLastRespBytes());
+ }
+
+ private static long us(long nanos) {
+ return nanos < 0 ? nanos : nanos / 1000L;
+ }
+
private void mainLoop() {
ArrayBlockingQueue<Metadata> intermediateResult = new ArrayBlockingQueue<>(1);
@@ -155,9 +202,12 @@ public class ConnectionHandler implements Runnable, Closeable {
PipesRequest pipesRequest;
FetchEmitTuple fetchEmitTuple;
+ resetTimings();
+ long reqDeserStart = System.nanoTime();
try {
pipesRequest = JsonPipesIpc.fromBytes(msg.payload(), PipesRequest.class);
fetchEmitTuple = pipesRequest.getTuple();
+ tReqDeserNanos = System.nanoTime() - reqDeserStart;
} catch (IOException e) {
LOG.error("handlerId={}: problem deserializing PipesRequest", handlerId, e);
handleCrash(PipesMessageType.UNSPECIFIED_CRASH, "unknown", e);
@@ -165,6 +215,7 @@ public class ConnectionHandler implements Runnable, Closeable {
}
ParseContext mergedContext = null;
try {
+ long ctxStart = System.nanoTime();
mergedContext = resources.createMergedParseContext(fetchEmitTuple.getParseContext());
ParseContextUtils.resolveAll(mergedContext, getClass().getClassLoader());
ServerProtocolIO.validateParseContext(mergedContext);
@@ -178,12 +229,16 @@ public class ConnectionHandler implements Runnable, Closeable {
// ParseTimeout.getOrCreate(mergedContext) call (inside CompositeParser)
// sees this instance rather than racing to install its own.
ParseTimeout parseTimeout = ParseTimeout.getOrCreate(mergedContext);
+ tCtxMergeNanos = System.nanoTime() - ctxStart;
PipesWorker pipesWorker = createPipesWorker(intermediateResult, fetchEmitTuple,
mergedContext, countDownLatch);
+ tLastWorker = pipesWorker;
+ tSubmitAtNanos = System.nanoTime();
executorCompletionService.submit(pipesWorker);
loopUntilDone(fetchEmitTuple, mergedContext, intermediateResult, countDownLatch, parseTimeout);
+ logTiming(fetchEmitTuple.getId());
} catch (TikaConfigException e) {
LOG.error("handlerId={}: config error processing request", handlerId, e);
handleCrash(PipesMessageType.UNSPECIFIED_CRASH, fetchEmitTuple.getId(), e);
@@ -285,7 +340,9 @@ public class ConnectionHandler implements Runnable, Closeable {
if (intermediate != null) {
if (!clientGone) {
try {
- protocolIO.writeIntermediate(intermediate);
+ // Frame-flush releases the latch so the worker parses during the
+ // ACK round trip; the countDown below is the failure-path net.
+ protocolIO.writeIntermediate(intermediate, countDownLatch::countDown);
} catch (IOException e) {
clientGone = true;
LOG.debug("handlerId={}: client gone (writing intermediate); keeping the "
@@ -300,6 +357,8 @@ public class ConnectionHandler implements Runnable, Closeable {
// Check for task completion
Future<PipesResult> future = executorCompletionService.poll(100, TimeUnit.MILLISECONDS);
if (future != null) {
+ tHandoffNanos = System.nanoTime() - tSubmitAtNanos
+ - (tLastWorker == null ? 0 : Math.max(0, tLastWorker.getWallNanos()));
PipesResult pipesResult = null;
try {
pipesResult = future.get();
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
index 173a7efc6e..dd545e38bb 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
@@ -191,6 +191,18 @@ public class PipesServer implements AutoCloseable {
private final EmitStrategy emitStrategy;
private final ServerProtocolIO protocolIO;
+ /** Per-parse worker-side latency breakdown; joins the client line on {@code id}. */
+ private static final Logger TIMING_LOG =
+ LoggerFactory.getLogger("org.apache.tika.pipes.timing.worker");
+
+ // Per-request timing scratch. A per-client fork handles one request at a time.
+ private long tReqDeserNanos = -1;
+ private long tCtxMergeNanos = -1;
+ private long tSubmitAtNanos = -1;
+ private long tHandoffNanos = -1;
+ private long tIntermediateWaitNanos = -1;
+ private PipesWorker tLastWorker;
+
public static PipesServer load(int port, Path tikaConfigPath) throws Exception {
String pipesClientId = System.getProperty("pipesClientId", "unknown");
LOG.debug("connecting to client on port={}", port);
@@ -430,9 +442,12 @@ public class PipesServer implements AutoCloseable {
PipesRequest pipesRequest;
FetchEmitTuple fetchEmitTuple;
+ resetTimings();
+ long reqDeserStart = System.nanoTime();
try {
pipesRequest = JsonPipesIpc.fromBytes(msg.payload(), PipesRequest.class);
fetchEmitTuple = pipesRequest.getTuple();
+ tReqDeserNanos = System.nanoTime() - reqDeserStart;
} catch (IOException e) {
LOG.error("problem deserializing PipesRequest", e);
handleCrash(PipesMessageType.UNSPECIFIED_CRASH, "unknown", e);
@@ -440,6 +455,7 @@ public class PipesServer implements AutoCloseable {
}
ParseContext mergedContext;
ParseTimeout parseTimeout;
+ long ctxStart = System.nanoTime();
try {
mergedContext = createMergedParseContext(fetchEmitTuple.getParseContext());
ParseContextUtils.resolveAll(mergedContext, getClass().getClassLoader());
@@ -454,6 +470,7 @@ public class PipesServer implements AutoCloseable {
// ParseTimeout.getOrCreate(mergedContext) call (inside CompositeParser)
// sees this instance rather than racing to install its own.
parseTimeout = ParseTimeout.getOrCreate(mergedContext);
+ tCtxMergeNanos = System.nanoTime() - ctxStart;
} catch (Exception e) {
// write the reason to the client instead of a bare exit code
handleCrash(PipesMessageType.UNSPECIFIED_CRASH, fetchEmitTuple.getId(), e);
@@ -461,9 +478,12 @@ public class PipesServer implements AutoCloseable {
}
PipesWorker pipesWorker = getPipesWorker(intermediateResult, fetchEmitTuple, mergedContext, countDownLatch);
+ tLastWorker = pipesWorker;
+ tSubmitAtNanos = System.nanoTime();
executorCompletionService.submit(pipesWorker);
try {
loopUntilDone(fetchEmitTuple, mergedContext, executorCompletionService, intermediateResult, countDownLatch, parseTimeout);
+ logTiming(fetchEmitTuple.getId());
} catch (Throwable t) {
if (t instanceof Error) {
// OOM or other JVM-level error: exit rather than continue in a
@@ -522,6 +542,42 @@ public class PipesServer implements AutoCloseable {
/** Steady-state slice once the intermediate result is out of the way. */
private static final long COMPLETION_POLL_MS = 100;
+ private void resetTimings() {
+ tReqDeserNanos = -1;
+ tCtxMergeNanos = -1;
+ tSubmitAtNanos = -1;
+ tHandoffNanos = -1;
+ tIntermediateWaitNanos = -1;
+ tLastWorker = null;
+ protocolIO.resetLastTimings();
+ }
+
+ /**
+ * One line per parse on {@code org.apache.tika.pipes.timing.worker}, microseconds.
+ * {@code handoff_us} is executor scheduling plus completion-poll latency around the worker;
+ * {@code resp_*} is the FINISHED frame's serialize, socket write, and the client-ACK wait.
+ */
+ private void logTiming(String id) {
+ if (!TIMING_LOG.isInfoEnabled()) {
+ return;
+ }
+ PipesWorker w = tLastWorker;
+ TIMING_LOG.info("WORKER_TIMING id={} req_deser_us={} ctx_merge_us={} intermediate_wait_us={}"
+ + " handoff_us={} fetch_us={} parse_us={} emit_us={} worker_wall_us={}"
+ + " intermediate_us={} resp_ser_us={} resp_write_us={} resp_ack_us={}"
+ + " resp_bytes={}",
+ id, us(tReqDeserNanos), us(tCtxMergeNanos), us(tIntermediateWaitNanos),
+ us(tHandoffNanos), us(w == null ? -1 : w.getFetchNanos()),
+ us(w == null ? -1 : w.getParseNanos()), us(w == null ? -1 : w.getEmitNanos()),
+ us(w == null ? -1 : w.getWallNanos()), us(protocolIO.getLastIntermediateNanos()),
+ us(protocolIO.getLastRespSerNanos()), us(protocolIO.getLastRespWriteNanos()),
+ us(protocolIO.getLastRespAckNanos()), protocolIO.getLastRespBytes());
+ }
+
+ private static long us(long nanos) {
+ return nanos < 0 ? nanos : nanos / 1000L;
+ }
+
private void loopUntilDone(FetchEmitTuple fetchEmitTuple, ParseContext mergedContext,
ExecutorCompletionService<PipesResult> executorCompletionService,
ArrayBlockingQueue<Metadata> intermediateResult, CountDownLatch countDownLatch,
@@ -539,7 +595,11 @@ public class PipesServer implements AutoCloseable {
if (!wroteIntermediateResult) {
Metadata intermediate = intermediateResult.poll(PRE_INTERMEDIATE_POLL_MS, TimeUnit.MILLISECONDS);
if (intermediate != null) {
- writeIntermediate(intermediate);
+ tIntermediateWaitNanos = System.nanoTime() - startNanos;
+ // The latch is released as soon as the frame is flushed, so the worker
+ // parses while the ACK is in flight; the extra countDown below is a
+ // no-op then, and the safety net when the write was skipped or failed.
+ writeIntermediate(intermediate, countDownLatch);
countDownLatch.countDown();
wroteIntermediateResult = true;
}
@@ -551,6 +611,8 @@ public class PipesServer implements AutoCloseable {
Future<PipesResult> future = executorCompletionService.poll(
wroteIntermediateResult ? COMPLETION_POLL_MS : 0, TimeUnit.MILLISECONDS);
if (future != null) {
+ tHandoffNanos = System.nanoTime() - tSubmitAtNanos
+ - (tLastWorker == null ? 0 : Math.max(0, tLastWorker.getWallNanos()));
PipesResult pipesResult = null;
try {
pipesResult = future.get();
@@ -775,9 +837,9 @@ public class PipesServer implements AutoCloseable {
}
}
- private void writeIntermediate(Metadata metadata) {
+ private void writeIntermediate(Metadata metadata, CountDownLatch latch) {
try {
- protocolIO.writeIntermediate(metadata);
+ protocolIO.writeIntermediate(metadata, latch::countDown);
} catch (ShutDownReceivedException e) {
handleShutDown();
} catch (IOException e) {
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
index 2f0b49eeec..8cc9962649 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
@@ -89,8 +89,40 @@ class PipesWorker implements Callable<PipesResult> {
this.defaultParseMode = defaultParseMode;
}
+ // Per-parse stage nanos, read by ConnectionHandler for the timing line after the
+ // worker completes. -1 means the stage never ran.
+ private volatile long fetchNanos = -1;
+ private volatile long parseNanos = -1;
+ private volatile long emitNanos = -1;
+ private volatile long wallNanos = -1;
+
+ long getFetchNanos() {
+ return fetchNanos;
+ }
+
+ long getParseNanos() {
+ return parseNanos;
+ }
+
+ long getEmitNanos() {
+ return emitNanos;
+ }
+
+ long getWallNanos() {
+ return wallNanos;
+ }
+
@Override
public PipesResult call() throws Exception {
+ long wallStart = System.nanoTime();
+ try {
+ return doCall();
+ } finally {
+ wallNanos = System.nanoTime() - wallStart;
+ }
+ }
+
+ private PipesResult doCall() throws Exception {
MetadataListAndEmbeddedBytes parseData = null;
TempFileUnpackHandler tempHandler = null;
FrictionlessUnpackHandler frictionlessHandler = null;
@@ -126,7 +158,12 @@ class PipesWorker implements Callable<PipesResult> {
}
}
- return emitHandler.emitParseData(fetchEmitTuple, parseData, parseContext);
+ long emitStart = System.nanoTime();
+ try {
+ return emitHandler.emitParseData(fetchEmitTuple, parseData, parseContext);
+ } finally {
+ emitNanos = System.nanoTime() - emitStart;
+ }
} finally {
// Clean up handlers if used
if (frictionlessHandler != null) {
@@ -490,13 +527,20 @@ class PipesWorker implements Callable<PipesResult> {
Metadata metadata = localContext.newMetadata();
// Carry the caller's resource name and Content-Type detection hints (see javadoc).
carryCallerHints(fetchEmitTuple.getMetadata(), metadata);
+ long fetchStart = System.nanoTime();
FetchHandler.TisOrResult tisOrResult = fetchHandler.fetch(fetchEmitTuple, metadata, localContext);
+ fetchNanos = System.nanoTime() - fetchStart;
if (tisOrResult.pipesResult() != null) {
return new ParseDataOrPipesResult(null, tisOrResult.pipesResult());
}
try (TikaInputStream tis = tisOrResult.tis()) {
- return parseHandler.parseWithStream(fetchEmitTuple, tis, metadata, localContext);
+ long parseStart = System.nanoTime();
+ try {
+ return parseHandler.parseWithStream(fetchEmitTuple, tis, metadata, localContext);
+ } finally {
+ parseNanos = System.nanoTime() - parseStart;
+ }
} catch (SecurityException e) {
LOG.error("security exception id={}", fetchEmitTuple.getId(), e);
throw e;
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
index f8512bc6bf..1e8eb30ee6 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
@@ -121,9 +121,12 @@ public class ServerProtocolIO {
*/
public void writeFinished(PipesResult pipesResult) throws IOException {
BoundedOutputStream bos = new BoundedOutputStream(maxIpcPayloadBytes);
+ long serStart = System.nanoTime();
try {
JsonPipesIpc.toStream(pipesResult, bos);
+ lastRespSerNanos = System.nanoTime() - serStart;
} catch (IOException e) {
+ lastRespSerNanos = System.nanoTime() - serStart;
if (!bos.overflowed()) {
throw e;
}
@@ -153,8 +156,52 @@ public class ServerProtocolIO {
doWritePayloadLimitExceeded();
return;
}
- PipesMessage.finished(bos.toByteArray()).write(output);
+ byte[] payload = bos.toByteArray();
+ lastRespBytes = payload.length;
+ long writeStart = System.nanoTime();
+ PipesMessage.finished(payload).write(output);
+ long ackStart = System.nanoTime();
+ lastRespWriteNanos = ackStart - writeStart;
awaitAck();
+ lastRespAckNanos = System.nanoTime() - ackStart;
+ }
+
+ // Last FINISHED-frame costs, read by ConnectionHandler for its per-parse timing line.
+ // One request at a time per connection, so plain fields suffice.
+ private long lastRespSerNanos = -1;
+ private long lastRespWriteNanos = -1;
+ private long lastRespAckNanos = -1;
+ private int lastRespBytes = -1;
+
+ long getLastRespSerNanos() {
+ return lastRespSerNanos;
+ }
+
+ long getLastRespWriteNanos() {
+ return lastRespWriteNanos;
+ }
+
+ long getLastRespAckNanos() {
+ return lastRespAckNanos;
+ }
+
+ int getLastRespBytes() {
+ return lastRespBytes;
+ }
+
+ /** Write+ACK cost of the INTERMEDIATE_RESULT frame; -1 when none was sent. */
+ private long lastIntermediateNanos = -1;
+
+ long getLastIntermediateNanos() {
+ return lastIntermediateNanos;
+ }
+
+ void resetLastTimings() {
+ lastRespSerNanos = -1;
+ lastRespWriteNanos = -1;
+ lastRespAckNanos = -1;
+ lastRespBytes = -1;
+ lastIntermediateNanos = -1;
}
/**
@@ -184,6 +231,18 @@ public class ServerProtocolIO {
* @throws IOException on serialization or I/O errors
*/
public void writeIntermediate(Metadata metadata) throws IOException {
+ writeIntermediate(metadata, null);
+ }
+
+ /**
+ * Like {@link #writeIntermediate(Metadata)}, but runs {@code afterFrameWritten} once the
+ * frame is flushed to the socket, before waiting for the client's ACK. Lets the caller
+ * unblock the parse worker while the ACK is still in flight -- the ACK round trip would
+ * otherwise sit between pre-parse and parse on every request. Not invoked when the
+ * oversized intermediate is skipped or the write fails; callers must handle those paths
+ * themselves.
+ */
+ public void writeIntermediate(Metadata metadata, Runnable afterFrameWritten) throws IOException {
BoundedOutputStream bos = new BoundedOutputStream(maxIpcPayloadBytes);
try {
JsonPipesIpc.toStream(metadata, bos);
@@ -195,7 +254,12 @@ public class ServerProtocolIO {
}
throw e;
}
+ long interStart = System.nanoTime();
PipesMessage.intermediateResult(bos.toByteArray()).write(output);
+ if (afterFrameWritten != null) {
+ afterFrameWritten.run();
+ }
+ lastIntermediateNanos = System.nanoTime() - interStart;
awaitAck();
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
index 9c1c1765b8..b88cdc0bd9 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
@@ -67,6 +67,9 @@ import org.apache.tika.server.core.TikaServerParseException;
public class PipesParsingHelper {
private static final Logger LOG = LoggerFactory.getLogger(PipesParsingHelper.class);
+ /** Per-request server-layer latency breakdown; joins the pipes lines on {@code id}. */
+ private static final Logger TIMING_LOG =
+ LoggerFactory.getLogger("org.apache.tika.pipes.timing.server");
/**
* The fetcher ID used for reading temp files.
@@ -179,10 +182,15 @@ public class PipesParsingHelper {
String requestId = UUID.randomUUID().toString();
PayloadRouter.Routed routed = null;
String callerSuppliedName = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
+ long entryNanos = System.nanoTime();
+ long routeNanos = -1;
+ long pipesNanos = -1;
+ long postNanos = -1;
try {
routed = PayloadRouter.route(tis, maxInlineBytes,
() -> Files.createTempFile(inputTempDirectory, "tika-", getSuffix(metadata)));
+ routeNanos = System.nanoTime() - entryNanos;
String relativeName = null;
FetchKey fetchKey;
@@ -217,13 +225,19 @@ public class PipesParsingHelper {
);
// Execute parse via pipes - results will be passed back through socket
+ long pipesStart = System.nanoTime();
PipesResult result = pipesParser.parse(tuple);
+ pipesNanos = System.nanoTime() - pipesStart;
// Process result
+ long postStart = System.nanoTime();
List<Metadata> metadataList = processResult(result);
if (relativeName != null) {
stripSpoolIdentity(metadataList, relativeName, callerSuppliedName);
}
+ postNanos = System.nanoTime() - postStart;
+ logTiming(requestId, routed.route().name(), routeNanos, pipesNanos, postNanos,
+ System.nanoTime() - entryNanos);
return metadataList;
} catch (InterruptedException e) {
@@ -241,6 +255,25 @@ public class PipesParsingHelper {
}
}
+ /**
+ * One line per request on {@code org.apache.tika.pipes.timing.server}, microseconds.
+ * {@code route_us} covers reading the request body and deciding inline-vs-spool;
+ * {@code pipes_us} is the whole pipes round trip; {@code post_us} is result unpacking.
+ * The HTTP/JAX-RS layer outside this method is measured from the client.
+ */
+ private static void logTiming(String id, String route, long routeNanos, long pipesNanos,
+ long postNanos, long totalNanos) {
+ if (!TIMING_LOG.isInfoEnabled()) {
+ return;
+ }
+ TIMING_LOG.info("SERVER_TIMING id={} route={} route_us={} pipes_us={} post_us={} total_us={}",
+ id, route, us(routeNanos), us(pipesNanos), us(postNanos), us(totalNanos));
+ }
+
+ private static long us(long nanos) {
+ return nanos < 0 ? nanos : nanos / 1000L;
+ }
+
/** Longest suffix carried over from a client filename; keeps well clear of NAME_MAX. */
private static final int MAX_SUFFIX_LENGTH = 20;
(tika) 07/13: TIKA-4868: single-pass text cleanup in WordExtractor and markdown handler
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 953b23251a3606958d1657544bdf802962074ef8
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 06:28:57 2026 -0400
TIKA-4868: single-pass text cleanup in WordExtractor and markdown handler
---
CHANGES.txt | 6 +++
.../apache/tika/sax/ToMarkdownContentHandler.java | 13 ++++-
.../tika/parser/microsoft/WordExtractor.java | 56 +++++++++++++++++-----
3 files changed, 61 insertions(+), 14 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index d5beba927b..dee363d60b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,11 @@
Release 4.1.0 - unreleased
+ * WordExtractor (.doc) cleans each character run in one pass instead of
+ four chained replace/replaceAll copies, and tests paragraph blankness
+ without a regex replaceAll; ToMarkdownContentHandler collapses line
+ breaks in one pass with no copy for clean runs. ~39% off parsing a
+ text-heavy 2MB .doc (TIKA-4868).
+
* tika-server's raw-output endpoints (/tika, /tika/text, ...) carry the
extracted content as raw UTF-8 bytes from the pipes worker to the HTTP
response instead of a Smile-encoded string: one encode in the worker
diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
index 5903a6d818..0c28ef40c2 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
@@ -659,7 +659,18 @@ public class ToMarkdownContentHandler extends DefaultHandler {
}
private static String collapseLineBreaks(String s) {
- return s.replace('\r', ' ').replace('\n', ' ');
+ // single pass, and no copy at all for the common clean run
+ char[] chars = null;
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ if (c == '\r' || c == '\n') {
+ if (chars == null) {
+ chars = s.toCharArray();
+ }
+ chars[i] = ' ';
+ }
+ }
+ return chars == null ? s : new String(chars);
}
private static String withTrailingNewline(String s) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
index 09a2b82767..25ad90d49f 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
@@ -110,6 +110,42 @@ public class WordExtractor extends AbstractPOIFSExtractor {
* Given a style name, return what tag should be used, and
* what style should be applied to it.
*/
+ // matches the old regex [\r\n\s]+ ( \s == [ \t\n\x0B\f\r] ), without the
+ // full replaceAll copy it used to make per paragraph
+ private static boolean isBlankParagraph(String text) {
+ for (int i = 0; i < text.length(); i++) {
+ char c = text.charAt(i);
+ if (c != '\r' && c != '\n' && c != ' ' && c != '\t' && c != 0x0B && c != '\f') {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static String cleanControlCharacters(String text) {
+ char[] chars = null;
+ for (int i = 0; i < text.length(); i++) {
+ char c = text.charAt(i);
+ char r;
+ if (c == 30) {
+ r = UNICODECHAR_NONBREAKING_HYPHEN;
+ } else if (c == 31) {
+ r = UNICODECHAR_ZERO_WIDTH_SPACE;
+ } else if (c < 0x20) {
+ r = '\n';
+ } else {
+ r = c;
+ }
+ if (r != c) {
+ if (chars == null) {
+ chars = text.toCharArray();
+ }
+ chars[i] = r;
+ }
+ }
+ return chars == null ? text : new String(chars);
+ }
+
public static TagAndStyle buildParagraphTagAndStyle(String styleName, boolean isTable) {
if (styleName == null || styleName.length() < 2) {
@@ -354,7 +390,7 @@ public class WordExtractor extends AbstractPOIFSExtractor {
}
String text = p.text();
- if (text.replaceAll("[\\r\\n\\s]+", "").isEmpty()) {
+ if (isBlankParagraph(text)) {
// Skip empty paragraphs
return 0;
}
@@ -456,22 +492,16 @@ public class WordExtractor extends AbstractPOIFSExtractor {
// Clean up the text
String text = cr.text();
- text = text.replace('\r', '\n');
if (text.endsWith("\u0007")) {
// Strip the table cell end marker
text = text.substring(0, text.length() - 1);
}
-
- // Copied from POI's org/apache/poi/hwpf/converter/AbstractWordConverter.processCharacters:
-
- // Non-breaking hyphens are returned as char 30
- text = text.replace((char) 30, UNICODECHAR_NONBREAKING_HYPHEN);
-
- // Non-required hyphens to zero-width space
- text = text.replace((char) 31, UNICODECHAR_ZERO_WIDTH_SPACE);
-
- // Control characters as line break
- text = text.replaceAll("[\u0000-\u001f]", "\n");
+ // One pass over the run replaces what used to be four chained
+ // replace/replaceAll copies (this runs once per character run):
+ // \r and every other control char become line breaks, POI's char 30
+ // (non-breaking hyphen) and char 31 (non-required hyphen) become
+ // their Unicode equivalents.
+ text = cleanControlCharacters(text);
xhtml.characters(text);
}
(tika) 06/13: side channel content bytes for fewer encode/decode in tika-server
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 25a2f6059e1cc532cc9091e5941606e8f6c0e21d
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 05:51:26 2026 -0400
side channel content bytes for fewer encode/decode in tika-server
---
CHANGES.txt | 9 ++++
.../apache/tika/pipes/api/emitter/EmitData.java | 9 ++++
.../apache/tika/pipes/core/ContentBytesConfig.java | 44 ++++++++++++++++
.../tika/pipes/core/emitter/EmitDataImpl.java | 22 +++++++-
.../core/serialization/EmitDataDeserializer.java | 7 ++-
.../core/serialization/EmitDataSerializer.java | 6 +++
.../apache/tika/pipes/core/server/EmitHandler.java | 32 ++++++++++++
.../serialization/EmitDataContentBytesTest.java | 58 ++++++++++++++++++++++
.../server/core/resource/PipesParsingHelper.java | 32 +++++++++++-
.../tika/server/core/resource/TikaResource.java | 44 ++++++++++------
10 files changed, 244 insertions(+), 19 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 8ef82bedf1..d5beba927b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,14 @@
Release 4.1.0 - unreleased
+ * tika-server's raw-output endpoints (/tika, /tika/text, ...) carry the
+ extracted content as raw UTF-8 bytes from the pipes worker to the HTTP
+ response instead of a Smile-encoded string: one encode in the worker
+ replaces a string transcode on both sides of the IPC plus a re-encode
+ at the HTTP layer (9MB text: 115ms -> 75ms end-to-end). Opt-in via
+ the new content-bytes-config parse-context component, which moves
+ CONTENT_ONLY passback content out of tk:content into
+ EmitData.getContentBytes() (TIKA-4868).
+
* Detection hot-path cleanups: MagicMatch resolves its detector via
double-checked locking instead of a synchronized method per eval;
glob patterns are compiled once at registration instead of per
diff --git a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/EmitData.java b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/EmitData.java
index ec6266ed23..0a337360ba 100644
--- a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/EmitData.java
+++ b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/EmitData.java
@@ -28,6 +28,15 @@ public interface EmitData {
String getContainerStackTrace();
+ /**
+ * Raw UTF-8 content bytes, set only when the request opted in via
+ * {@code content-bytes-config} under {@code ParseMode.CONTENT_ONLY};
+ * the metadata then no longer carries {@code TIKA_CONTENT}. Null otherwise.
+ */
+ default byte[] getContentBytes() {
+ return null;
+ }
+
long getEstimatedSizeBytes();
/**
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
new file mode 100644
index 0000000000..5d00bbd6ff
--- /dev/null
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.core;
+
+import java.io.Serializable;
+
+import org.apache.tika.annotation.TikaComponent;
+
+/**
+ * Opt-in for CONTENT_ONLY passback: the worker moves the extracted content out of
+ * {@code TIKA_CONTENT} into raw UTF-8 bytes on the {@code EmitData}, which travel
+ * as binary over the IPC instead of a Smile-encoded string. A caller who sets this
+ * must read {@code EmitData.getContentBytes()}; the metadata no longer carries the
+ * content. tika-server sets it for the raw-output endpoints.
+ */
+@TikaComponent(name = "content-bytes-config")
+public class ContentBytesConfig implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private boolean enabled = true;
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java
index 5fec421209..ebe53de666 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java
@@ -31,6 +31,9 @@ public class EmitDataImpl implements EmitData {
// ParseContext is not serialized - it's set by PipesClient after deserialization
private ParseContext parseContext;
+ // Raw UTF-8 content under content-bytes-config; rides the IPC as binary
+ private byte[] contentBytes;
+
public EmitDataImpl(String emitKey, List<Metadata> metadataList) {
this(emitKey, metadataList, StringUtils.EMPTY);
}
@@ -54,8 +57,21 @@ public class EmitDataImpl implements EmitData {
return containerStackTrace;
}
+ @Override
+ public byte[] getContentBytes() {
+ return contentBytes;
+ }
+
+ public void setContentBytes(byte[] contentBytes) {
+ this.contentBytes = contentBytes;
+ }
+
public long getEstimatedSizeBytes() {
- return estimateSizeInBytes(getEmitKey(), getMetadataList(), containerStackTrace);
+ long sz = estimateSizeInBytes(getEmitKey(), getMetadataList(), containerStackTrace);
+ if (contentBytes != null) {
+ sz += 36 + contentBytes.length;
+ }
+ return sz;
}
/**
@@ -97,6 +113,8 @@ public class EmitDataImpl implements EmitData {
@Override
public String toString() {
return "EmitData{" + "emitKey=" + emitKey + ", metadataList=" + metadataList +
- ", containerStackTrace='" + containerStackTrace + '\'' + '}';
+ ", containerStackTrace='" + containerStackTrace + '\'' +
+ ", contentBytes.length=" + (contentBytes == null ? "null" : contentBytes.length) +
+ '}';
}
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataDeserializer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataDeserializer.java
index 8d8c4e303d..1f63fa9325 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataDeserializer.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataDeserializer.java
@@ -46,7 +46,12 @@ public class EmitDataDeserializer extends JsonDeserializer<EmitDataImpl> {
String containerStackTrace = readString(CONTAINER_STACK_TRACE, root, StringUtils.EMPTY, false);
// ParseContext is NOT deserialized - it's restored by PipesClient from the original FetchEmitTuple
- return new EmitDataImpl(emitKey, metadataList, containerStackTrace);
+ EmitDataImpl emitData = new EmitDataImpl(emitKey, metadataList, containerStackTrace);
+ JsonNode contentBytesNode = root.get(EmitDataSerializer.CONTENT_BYTES);
+ if (contentBytesNode != null && !contentBytesNode.isNull()) {
+ emitData.setContentBytes(contentBytesNode.binaryValue());
+ }
+ return emitData;
}
private static List<Metadata> readMetadataList(JsonNode root, ObjectMapper mapper) throws IOException {
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataSerializer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataSerializer.java
index 5687088e55..5f7b87b397 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataSerializer.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/EmitDataSerializer.java
@@ -30,6 +30,7 @@ public class EmitDataSerializer extends JsonSerializer<EmitData> {
public static final String EMIT_KEY = "emitKey";
public static final String METADATA_LIST = "metadataList";
public static final String CONTAINER_STACK_TRACE = "containerStackTrace";
+ public static final String CONTENT_BYTES = "contentBytes";
@Override
public void serialize(EmitData emitData, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
@@ -39,6 +40,11 @@ public class EmitDataSerializer extends JsonSerializer<EmitData> {
if (!StringUtils.isBlank(emitData.getContainerStackTrace())) {
jsonGenerator.writeStringField(CONTAINER_STACK_TRACE, emitData.getContainerStackTrace());
}
+ if (emitData.getContentBytes() != null) {
+ // Smile writes this as raw binary (7-bit encoding is disabled), so multi-MB
+ // content costs a copy, not a string transcode
+ jsonGenerator.writeBinaryField(CONTENT_BYTES, emitData.getContentBytes());
+ }
// ParseContext is NOT serialized - it's restored by PipesClient from the original FetchEmitTuple
jsonGenerator.writeEndObject();
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
index f1fe8a7a8a..57bac19e86 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
@@ -43,6 +43,7 @@ import org.apache.tika.pipes.api.PipesResult;
import org.apache.tika.pipes.api.emitter.EmitKey;
import org.apache.tika.pipes.api.emitter.Emitter;
import org.apache.tika.pipes.api.emitter.StreamEmitter;
+import org.apache.tika.pipes.core.ContentBytesConfig;
import org.apache.tika.pipes.core.EmitStrategy;
import org.apache.tika.pipes.core.EmitStrategyConfig;
import org.apache.tika.pipes.core.PassbackFilter;
@@ -133,6 +134,9 @@ class EmitHandler {
&& emitterId != null
&& emitterManager.getAllIds().contains(emitterId);
boolean willEmit = forceEmit || shouldEmit(parseMode, parseData, emitDataTuple, parseContext);
+ if (!willEmit) {
+ maybeMoveContentToBytes(parseMode, emitDataTuple, parseContext);
+ }
if (willEmit) {
return emit(t.getId(), emitKey, parseMode == ParseMode.UNPACK,
parseData, stack, parseContext);
@@ -148,6 +152,34 @@ class EmitHandler {
}
}
+ /**
+ * Under {@code content-bytes-config} + CONTENT_ONLY, moves the extracted content out of
+ * {@code TIKA_CONTENT} into raw UTF-8 bytes on the passback {@code EmitData}. One encode
+ * here replaces a Smile string encode, the client-side string decode, and the consumer's
+ * re-encode. Passback only -- the direct-emit path streams the string itself.
+ */
+ private static void maybeMoveContentToBytes(ParseMode parseMode, EmitDataImpl emitData,
+ ParseContext parseContext) {
+ if (parseMode != ParseMode.CONTENT_ONLY) {
+ return;
+ }
+ ContentBytesConfig config = parseContext.get(ContentBytesConfig.class);
+ if (config == null || !config.isEnabled()) {
+ return;
+ }
+ List<Metadata> metadataList = emitData.getMetadataList();
+ if (metadataList == null || metadataList.isEmpty()) {
+ return;
+ }
+ Metadata m = metadataList.get(0);
+ String content = m.get(TikaCoreProperties.TIKA_CONTENT);
+ if (content == null) {
+ return;
+ }
+ emitData.setContentBytes(content.getBytes(StandardCharsets.UTF_8));
+ m.remove(TikaCoreProperties.TIKA_CONTENT);
+ }
+
private PipesResult emit(String taskId, EmitKey emitKey,
boolean isExtractEmbeddedBytes, MetadataListAndEmbeddedBytes parseData,
String parseExceptionStack, ParseContext parseContext) {
diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/EmitDataContentBytesTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/EmitDataContentBytesTest.java
new file mode 100644
index 0000000000..1acb0bb035
--- /dev/null
+++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/EmitDataContentBytesTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.core.serialization;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.api.PipesResult;
+import org.apache.tika.pipes.core.emitter.EmitDataImpl;
+
+public class EmitDataContentBytesTest {
+
+ @Test
+ public void testContentBytesRoundTrip() throws Exception {
+ Metadata m = new Metadata();
+ m.set("k", "v");
+ EmitDataImpl emitData = new EmitDataImpl("key", List.of(m));
+ byte[] content = "the quick UTF-8 café 中文".getBytes(StandardCharsets.UTF_8);
+ emitData.setContentBytes(content);
+ PipesResult result = new PipesResult(PipesResult.RESULT_STATUS.PARSE_SUCCESS, emitData);
+
+ byte[] wire = JsonPipesIpc.toBytes(result);
+ PipesResult back = JsonPipesIpc.fromBytes(wire, PipesResult.class);
+
+ assertEquals(PipesResult.RESULT_STATUS.PARSE_SUCCESS, back.status());
+ assertArrayEquals(content, back.emitData().getContentBytes());
+ assertEquals("v", back.emitData().getMetadataList().get(0).get("k"));
+ }
+
+ @Test
+ public void testAbsentContentBytesStaysNull() throws Exception {
+ EmitDataImpl emitData = new EmitDataImpl("key", List.of(new Metadata()));
+ PipesResult result = new PipesResult(PipesResult.RESULT_STATUS.PARSE_SUCCESS, emitData);
+ PipesResult back = JsonPipesIpc.fromBytes(JsonPipesIpc.toBytes(result), PipesResult.class);
+ assertNull(back.emitData().getContentBytes());
+ }
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
index b88cdc0bd9..967aabbca7 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
@@ -45,6 +45,7 @@ import org.apache.tika.pipes.api.PipesResult;
import org.apache.tika.pipes.api.emitter.EmitData;
import org.apache.tika.pipes.api.emitter.EmitKey;
import org.apache.tika.pipes.api.fetcher.FetchKey;
+import org.apache.tika.pipes.core.ContentBytesConfig;
import org.apache.tika.pipes.core.EmitStrategy;
import org.apache.tika.pipes.core.EmitStrategyConfig;
import org.apache.tika.pipes.core.PipesConfig;
@@ -179,6 +180,30 @@ public class PipesParsingHelper {
*/
public List<Metadata> parse(TikaInputStream tis, Metadata metadata,
ParseContext parseContext, ParseMode parseMode) throws IOException {
+ return parseInternal(tis, metadata, parseContext, parseMode, false).metadataList();
+ }
+
+ /**
+ * The metadata plus, when requested via {@code content-bytes-config}, the extracted
+ * content as raw UTF-8 -- {@code TIKA_CONTENT} is then absent from the metadata.
+ */
+ public record ParseOutput(List<Metadata> metadataList, byte[] contentBytes) {
+ }
+
+ /**
+ * Like {@link #parse} but asks the worker for the content as raw UTF-8 bytes, which
+ * travel as binary over the IPC instead of a Smile-encoded string -- the win is one
+ * UTF-8 encode in the worker instead of a string transcode on both sides plus a
+ * re-encode at the HTTP layer. CONTENT_ONLY only.
+ */
+ public ParseOutput parseContentOnlyToBytes(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext) throws IOException {
+ return parseInternal(tis, metadata, parseContext, ParseMode.CONTENT_ONLY, true);
+ }
+
+ private ParseOutput parseInternal(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext, ParseMode parseMode,
+ boolean contentAsBytes) throws IOException {
String requestId = UUID.randomUUID().toString();
PayloadRouter.Routed routed = null;
String callerSuppliedName = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
@@ -209,6 +234,9 @@ public class PipesParsingHelper {
// Set parse mode in context
parseContext.set(ParseMode.class, parseMode);
+ if (contentAsBytes) {
+ parseContext.set(ContentBytesConfig.class, new ContentBytesConfig());
+ }
// This parser is shared with /pipes, whose own default is EMIT_ALL. No
// emitter is configured for /tika/rmeta/unpack requests (EmitKey.NO_EMIT
@@ -235,10 +263,12 @@ public class PipesParsingHelper {
if (relativeName != null) {
stripSpoolIdentity(metadataList, relativeName, callerSuppliedName);
}
+ byte[] contentBytes = (contentAsBytes && result.emitData() != null)
+ ? result.emitData().getContentBytes() : null;
postNanos = System.nanoTime() - postStart;
logTiming(requestId, routed.route().name(), routeNanos, pipesNanos, postNanos,
System.nanoTime() - entryNanos);
- return metadataList;
+ return new ParseOutput(metadataList, contentBytes);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
index c590a183d6..eca7b938c4 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
@@ -22,8 +22,6 @@ import static org.apache.tika.server.core.resource.RecursiveMetadataResource.HAN
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
@@ -404,6 +402,20 @@ public class TikaResource {
* @return list of metadata objects from parsing
* @throws IOException if parsing fails
*/
+ private PipesParsingHelper.ParseOutput parseWithPipesRaw(TikaInputStream tis,
+ Metadata metadata, ParseContext parseContext) throws IOException {
+ if (pipesParsingHelper == null) {
+ throw new IllegalStateException("Pipes-based parsing is not enabled");
+ }
+ String fileName = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
+ long taskId = serverStatus.start(ServerStatus.TASK.PARSE, fileName);
+ try {
+ return pipesParsingHelper.parseContentOnlyToBytes(tis, metadata, parseContext);
+ } finally {
+ serverStatus.complete(taskId);
+ }
+ }
+
public List<Metadata> parseWithPipes(TikaInputStream tis, Metadata metadata,
ParseContext parseContext, ParseMode parseMode)
throws IOException {
@@ -715,22 +727,26 @@ public class TikaResource {
handlerTypeName, context.get(ContentHandlerFactory.class));
// Parse with pipes using CONTENT_ONLY mode - the metadata filter in
- // EmitHandler will strip everything except tk:content
- List<Metadata> metadataList =
- parseWithPipes(tis, metadata, context, ParseMode.CONTENT_ONLY);
+ // EmitHandler will strip everything except tk:content, and the content comes
+ // back as raw UTF-8 bytes rather than a Smile-encoded string
+ PipesParsingHelper.ParseOutput parsed =
+ parseWithPipesRaw(tis, metadata, context);
+ List<Metadata> metadataList = parsed.metadataList();
LOG.debug("produceRawOutput: parseWithPipes returned {} metadata objects", metadataList.size());
// Extract content before checking for an exception -- content must not be
// discarded just because a container-level exception also occurred.
- String content = "";
+ byte[] content = parsed.contentBytes();
boolean hasException = false;
String exceptionMessage = null;
if (!metadataList.isEmpty()) {
- String extracted = metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT);
- LOG.debug("produceRawOutput: TIKA_CONTENT length={}", extracted != null ? extracted.length() : 0);
- if (extracted != null) {
- content = extracted;
+ if (content == null) {
+ // fallback: results built without the byte path (crash/error metadata)
+ String extracted = metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT);
+ if (extracted != null) {
+ content = extracted.getBytes(UTF_8);
+ }
}
exceptionMessage = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
hasException = exceptionMessage != null && !exceptionMessage.isEmpty();
@@ -742,13 +758,11 @@ public class TikaResource {
// 422 status signals the partial parse and the body carries the extracted content
// only -- never the server-side exception/stack trace. Clients that need the
// container exception should use /rmeta.
- final String finalContent = content;
+ final byte[] finalContent = content == null ? new byte[0] : content;
StreamingOutput streamingOutput = outputStream -> {
- try (Writer writer = new OutputStreamWriter(outputStream, UTF_8)) {
- writer.write(finalContent);
- writer.flush();
- }
+ outputStream.write(finalContent);
+ outputStream.flush();
};
return Response.status(hasException ? 422 : Response.Status.OK.getStatusCode())
.entity(streamingOutput)
(tika) 01/13: TIKA-4868: improve md
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 92c53d17a551557cb9560bf9878ca0112d5f5818
Author: tallison <ta...@apache.org>
AuthorDate: Tue Sep 1 15:16:43 2026 -0400
TIKA-4868: improve md
---
CHANGES.txt | 5 ++
.../apache/tika/sax/ToMarkdownContentHandler.java | 75 +++++++++++++++++++++-
2 files changed, 78 insertions(+), 2 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 80e898aa72..397b2915b8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,10 @@
Release 4.1.0 - unreleased
+ * Markdown output is ~4x faster on large documents:
+ ToMarkdownContentHandler now buffers the commonmark renderer's
+ per-character writes instead of paying the synchronized
+ Writer.write(int) cost for every character (TIKA-4868).
+
* Raster previews for the vector thumbnails of Office documents: the new
poi-metafile-renderer draws EMF and WMF images through POI (a PNG of
a configurable width; Word's bitmap-in-WMF thumbnails from the bitmap
diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
index 33f35c9718..5903a6d818 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
@@ -417,11 +417,82 @@ public class ToMarkdownContentHandler extends DefaultHandler {
if (finished) {
return;
}
- renderer.render(document, writer);
- writer.flush();
+ // MarkdownRenderer emits mostly one char at a time; Writer.write(int) is
+ // synchronized and allocating in every stock Writer, which made rendering
+ // ~4x slower than the render logic itself on multi-MB documents. The
+ // unsynchronized buffer turns those calls into array stores.
+ RenderBuffer buffered = new RenderBuffer(writer);
+ renderer.render(document, buffered);
+ buffered.flush();
finished = true;
}
+ /** Unsynchronized bulk buffer between the renderer and the target writer. */
+ private static final class RenderBuffer extends Writer {
+ private final Writer out;
+ private final char[] buf = new char[8192];
+ private int n;
+
+ RenderBuffer(Writer out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int c) throws IOException {
+ if (n == buf.length) {
+ drain();
+ }
+ buf[n++] = (char) c;
+ }
+
+ @Override
+ public void write(char[] c, int off, int len) throws IOException {
+ if (len >= buf.length) {
+ drain();
+ out.write(c, off, len);
+ return;
+ }
+ if (n + len > buf.length) {
+ drain();
+ }
+ System.arraycopy(c, off, buf, n, len);
+ n += len;
+ }
+
+ @Override
+ public void write(String s, int off, int len) throws IOException {
+ if (len >= buf.length) {
+ drain();
+ out.write(s, off, len);
+ return;
+ }
+ if (n + len > buf.length) {
+ drain();
+ }
+ s.getChars(off, off + len, buf, n);
+ n += len;
+ }
+
+ private void drain() throws IOException {
+ if (n > 0) {
+ out.write(buf, 0, n);
+ n = 0;
+ }
+ }
+
+ /** Flushes through to the target; the target's lifecycle stays the caller's. */
+ @Override
+ public void flush() throws IOException {
+ drain();
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ flush();
+ }
+ }
+
@Override
public String toString() {
if (finished) {
(tika) 13/13: TIKA-4868: tersify comments; CHANGES entries trimmed
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit a1e91c362aa797e4adb9ac0896616951cb9afd91
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 10:31:09 2026 -0400
TIKA-4868: tersify comments; CHANGES entries trimmed
---
CHANGES.txt | 102 +++++++++------------
.../org/apache/tika/detect/DefaultDetector.java | 5 +-
.../java/org/apache/tika/detect/MagicDetector.java | 5 +-
.../main/java/org/apache/tika/mime/Patterns.java | 7 +-
.../org/apache/tika/parser/CompositeParser.java | 8 +-
.../apache/tika/sax/ToMarkdownContentHandler.java | 6 +-
.../tika/parser/microsoft/WordExtractor.java | 8 +-
.../org/apache/tika/parser/csv/CSVSniffer.java | 15 +--
.../apache/tika/pipes/core/ContentBytesConfig.java | 7 +-
.../serdes/ParseContextSerializer.java | 5 +-
10 files changed, 66 insertions(+), 102 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index f0a5cf99d3..54b847e3b5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,80 +1,66 @@
Release 4.1.0 - unreleased
+ * TikaInputStream.hasReliableLength() distinguishes measured lengths
+ from declared Content-Length hints, and one-shot streams now carry a
+ declared length without spooling; detection sizes its magic read only
+ from measured lengths, so a lying declared length can no longer
+ truncate it (TIKA-4868).
+
+ * ParseContext entries holding per-parse runtime state (ParseRecord,
+ ParseTimeout, the parser-map cache) are skipped during serialization
+ instead of failing as unregistered components (TIKA-4868).
+
* Mojibuster's adaptive probe strips incrementally instead of
- re-stripping the whole buffer on every read (quadratic in read count
- on tag-heavy pages; a 293KB page's probe drops 4.5ms -> 0.8ms).
- Byte-identical: the resumable scanner holds back at a chunk-final
- "<!" so chunked == single-shot at every boundary (exhaustive
- boundary tests), and detection output was unchanged across a
- 3,034-file differential. JunkDetector's Unicode block lookup also
- uses a precomputed BMP table (exhaustive equivalence test)
- (TIKA-4868).
+ re-stripping the whole buffer on every read (quadratic on tag-heavy
+ pages); JunkDetector's Unicode block lookup uses a precomputed BMP
+ table. Output unchanged (TIKA-4868).
- * Markdown rendering is another ~8x faster on large documents:
- a custom Text-node renderer emits unescaped spans in bulk instead of
- commonmark's per-character escape-check-and-append (9MB render
- 42ms -> 4ms; byte-identical output on a 1,900-document differential
- across the test corpus and real-world HTML) (TIKA-4868).
+ * Markdown rendering is another ~8x faster on large documents: a custom
+ Text-node renderer emits unescaped spans in bulk instead of
+ commonmark's per-character escape-check-and-append. Byte-identical
+ output, guarded by a fast-vs-stock differential test (TIKA-4868).
* ZipParser no longer re-decompresses an entry on every rewind when the
entry uses a legacy compression method (implode, shrink, bzip2, ...):
- those decode per-bit, so the rewind-by-reopen trade that works for
- STORED/DEFLATED entries re-decoded the whole entry for each of the
- several rewinds the embedded pipeline performs. Such entries now
- replay from the budgeted cache. An imploded 606KB zip drops from
- 610ms to 102ms in tika-server (TIKA-4868).
+ such entries replay from the budgeted cache instead of re-opening.
+ An imploded 606KB zip drops from 610ms to 102ms (TIKA-4868).
* More detection/dispatch savings: the message/rfc822 priority-45 magic
- is gated behind a ':' scan of the first 30 bytes (every match provably
- has one, so results are unchanged and non-mail text skips its ~40
- clauses); CompositeParser caches the built type->parser map in the
- ParseContext, so embedded documents reuse the container's map instead
- of rebuilding it per document. Plain-text detection drops from 334us
- (4.1.0-dev ran it twice) to 52us per document; eml requests ~-45%,
- ppt ~-35% in tika-server. MagicDetector also precomputes a 256-entry
- first-byte table per pattern (mask and case fold become one array
- load per scanned position). Adds a RESOURCE_TIMING log on
- org.apache.tika.pipes.timing.resource, silenced by default in the
- shipped log4j2 configs (TIKA-4868).
-
- * WordExtractor (.doc) cleans each character run in one pass instead of
- four chained replace/replaceAll copies, and tests paragraph blankness
- without a regex replaceAll; ToMarkdownContentHandler collapses line
- breaks in one pass with no copy for clean runs. ~39% off parsing a
- text-heavy 2MB .doc (TIKA-4868).
+ is gated behind a lossless ':' scan of the first 30 bytes;
+ CompositeParser caches the built type->parser map in the ParseContext
+ so embedded documents reuse the container's map; MagicDetector
+ precomputes a per-pattern first-byte table. Adds a RESOURCE_TIMING
+ log, silenced by default in the shipped log4j2 configs (TIKA-4868).
+
+ * WordExtractor (.doc) cleans each character run and tests paragraph
+ blankness in single passes; ToMarkdownContentHandler collapses line
+ breaks copy-free for clean runs. ~39% off a text-heavy 2MB .doc
+ (TIKA-4868).
* tika-server's raw-output endpoints (/tika, /tika/text, ...) carry the
extracted content as raw UTF-8 bytes from the pipes worker to the HTTP
- response instead of a Smile-encoded string: one encode in the worker
- replaces a string transcode on both sides of the IPC plus a re-encode
- at the HTTP layer (9MB text: 115ms -> 75ms end-to-end). Opt-in via
- the new content-bytes-config parse-context component, which moves
- CONTENT_ONLY passback content out of tk:content into
- EmitData.getContentBytes() (TIKA-4868).
+ response instead of a Smile-encoded string (9MB text: 115ms -> 75ms).
+ Opt-in via the new content-bytes-config parse-context component, which
+ moves CONTENT_ONLY passback content out of tk:content into
+ EmitData.getContentBytes(); results routed to a regular Emitter get
+ the content restored to the metadata (TIKA-4868).
* Detection hot-path cleanups: MagicMatch resolves its detector via
- double-checked locking instead of a synchronized method per eval;
- glob patterns are compiled once at registration instead of per
- lookup; MimeTypes.forName reads a ConcurrentHashMap (fixing an
- unsynchronized-read race) and indexes the normalized key so
- parameterized aliases stop re-locking per call; resource names
+ double-checked locking; glob patterns are compiled once at
+ registration; MimeTypes.forName reads a ConcurrentHashMap (fixing an
+ unsynchronized-read race) and indexes normalized keys; resource names
containing spaces skip the URI-parse-by-exception; the magic-header
- buffer is sized by the stream's known length instead of a fixed
- 64KB per detection; and the Adobe Illustrator ranged regex is
- gated behind a literal scan (detection results unchanged,
- ~28us off every non-AI detection) (TIKA-4868).
+ buffer is sized by the stream's measured length instead of a fixed
+ 64KB; the Adobe Illustrator ranged regex is gated behind a literal
+ scan. Detection results unchanged (TIKA-4868).
* Magic detection is ~35% faster on unmatched (e.g. plain-text) input:
- MagicDetector range scans find first-byte candidates with a tight
- scan before running the full masked/case-folded compare, instead of
- paying it at every offset in the range (TIKA-4868).
+ range scans find first-byte candidates before running the full
+ masked/case-folded compare (TIKA-4868).
* CSVSniffer reads its detection window once into a shared buffer and
- runs every delimiter hypothesis against it, instead of re-reading the
- stream through a pushback/mark-reset stack per delimiter; windows with
- no delimiter and no quote skip the scan outright. Identical results
- on a 1,125-file differential; ~28% off small-text parse time in
- tika-server (TIKA-4868).
+ runs every delimiter hypothesis against it; windows with no delimiter
+ and no quote skip the scan outright. Results unchanged (TIKA-4868).
* Pipes workers no longer stall between pre-parse and parse waiting
for the client to acknowledge the intermediate-result frame; the
diff --git a/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java b/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
index bda7f5e6f9..200c3eec9a 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
@@ -135,10 +135,7 @@ public class DefaultDetector extends CompositeDetector {
@Override
public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext)
throws IOException {
- // 0. An override short-circuits detection entirely, as in CompositeDetector.
- // Without this check the magic scan below runs even when a prior detect pass
- // recorded its result as CONTENT_TYPE_PARSER_OVERRIDE -- doubling detection
- // cost on every pipes parse.
+ // An override short-circuits detection entirely, as in CompositeDetector.
MediaType override = detectOverrides(metadata);
if (override != null) {
return override;
diff --git a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
index 2af36ee54f..e5476204a4 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
@@ -507,9 +507,8 @@ public class MagicDetector implements Detector {
}
}
} else {
- // Range scans (e.g. "\nHeader:" over 0:1024) dominate detection time, so
- // find first-byte candidates with a tight scan and run the full compare
- // only there, instead of paying the masked/case machinery at every offset.
+ // Range scans (e.g. "\nHeader:" over 0:1024) dominate detection cost:
+ // scan for first-byte candidates, run the full compare only there.
if (length == 0) {
// degenerate empty pattern: preserves the old loop's outcome
return startOffset <= endOffset && startOffset <= buffer.length;
diff --git a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
index f5f8c234e0..46fe77e0d5 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
@@ -52,10 +52,9 @@ class Patterns implements Serializable {
new TreeMap<>(new LengthComparator());
/**
- * Compiled forms of {@link #globs}' keys. Matching recompiled every glob regex
- * per lookup before; for names that miss the name/extension indexes that was
- * a Pattern.compile per glob per call. Transient with a lazy rebuild so a
- * serialized form from a build without this field still deserializes.
+ * Compiled forms of {@link #globs}' keys, so a lookup does not
+ * Pattern.compile per glob. Transient with a lazy rebuild so a serialized
+ * form from a build without this field still deserializes.
*/
private transient Map<String, Pattern> compiledGlobs = new HashMap<>();
private int minExtensionLength = Integer.MAX_VALUE;
diff --git a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
index 7d6a46e0b3..c4de97085a 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
@@ -249,11 +249,9 @@ public class CompositeParser implements Parser {
/**
* Cache of built parser maps, stored in the ParseContext so every embedded
- * document in a parse reuses the container's map instead of rebuilding it
- * (a full walk of every parser's supported types). Keyed by parser instance
- * because nested composites share one context. The map is built once per
- * (parser, context) and lives as long as the context: a context entry that
- * would change a parser's supported types is not picked up mid-parse, nor on
+ * document in a parse reuses the container's map. Keyed by parser instance
+ * because nested composites share one context. Lives as long as the context:
+ * a change to a parser's supported types is not picked up mid-parse, nor on
* later parses that reuse the same ParseContext instance.
*/
private static final class ParserMapCache implements TransientParseState {
diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
index 6a958b1a7d..8be96fd00a 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
@@ -420,10 +420,8 @@ public class ToMarkdownContentHandler extends DefaultHandler {
if (finished) {
return;
}
- // MarkdownRenderer emits mostly one char at a time; Writer.write(int) is
- // synchronized and allocating in every stock Writer, which made rendering
- // ~4x slower than the render logic itself on multi-MB documents. The
- // unsynchronized buffer turns those calls into array stores.
+ // The renderer writes mostly one char at a time; stock Writers take a
+ // synchronized, allocating write(int) for each. The buffer makes those array stores.
RenderBuffer buffered = new RenderBuffer(writer);
renderer.render(document, buffered);
buffered.flush();
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
index 18efa9bcd2..346de85dcc 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
@@ -106,8 +106,7 @@ public class WordExtractor extends AbstractPOIFSExtractor {
return count;
}
- // matches the old regex [\r\n\s]+ ( \s == [ \t\n\x0B\f\r] ), without the
- // full replaceAll copy it used to make per paragraph
+ // blank == matches [\r\n\s]+ ( \s == [ \t\n\x0B\f\r] )
private static boolean isBlankParagraph(String text) {
for (int i = 0; i < text.length(); i++) {
char c = text.charAt(i);
@@ -492,11 +491,6 @@ public class WordExtractor extends AbstractPOIFSExtractor {
// Strip the table cell end marker
text = text.substring(0, text.length() - 1);
}
- // One pass over the run replaces what used to be four chained
- // replace/replaceAll copies (this runs once per character run):
- // \r and every other control char become line breaks, POI's char 30
- // (non-breaking hyphen) and char 31 (non-required hyphen) become
- // their Unicode equivalents.
text = cleanControlCharacters(text);
xhtml.characters(text);
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
index 0ef0cd53cf..f5e7c2a405 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
@@ -57,10 +57,8 @@ class CSVSniffer {
if (!reader.markSupported()) {
reader = new BufferedReader(reader);
}
- // Every snifflet examines the same window, so read it once into a buffer
- // instead of once per delimiter through a mark/reset + pushback stack.
- // Grown on demand: markLimit is user-configurable and most inputs are
- // far smaller, so an eager char[markLimit] would be waste per parse.
+ // Every snifflet examines the same window, so read it once. Grown on
+ // demand: markLimit is user-configurable and usually far exceeds the input.
char[] buf = new char[Math.min(markLimit, 8192)];
reader.mark(markLimit);
int len = 0;
@@ -172,8 +170,7 @@ class CSVSniffer {
//hardcode this for now
private final char quoteCharacter = '"';
- // The shared window read once by sniff(Reader); pos is the cursor, so
- // "unread" is a decrement and the mark-limit check is a bounds check.
+ // the shared window from sniff(Reader); pos is the cursor
private final char[] buf;
private final int len;
private int pos = 0;
@@ -184,8 +181,7 @@ class CSVSniffer {
boolean rowZeroEmpty = false;
int encapsulated = 0; //number of cells that are encapsulated in dquotes (for now)
boolean parseException = false;
- // Cell content is never analyzed (see the unquoted() TODO that was here);
- // only whether the current cell is non-empty matters.
+ // only cell non-emptiness matters; content is never analyzed
private int unquotedLen = 0;
public Snifflet(char delimiter, char[] buf, int len) {
@@ -322,8 +318,7 @@ class CSVSniffer {
}
private int read() throws IOException {
- // pos tracks chars consumed exactly as charsRead did (unread decrements
- // both), so the original off-by-one mark-limit semantics are preserved.
+ // markLimit - 1, not markLimit: HitMarkLimit must win over EOF at the boundary
if (pos >= markLimit - 1) {
throw new HitMarkLimitException();
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
index a8f8d2c5e3..2e8b3a2b42 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
@@ -25,10 +25,9 @@ import org.apache.tika.annotation.TikaComponent;
* {@code TIKA_CONTENT} into raw UTF-8 bytes on the {@code EmitData}, which travel
* as binary over the IPC instead of a Smile-encoded string. A caller who sets this
* must read {@code EmitData.getContentBytes()}; the metadata no longer carries the
- * content. tika-server sets it for the raw-output endpoints. Results routed to a
- * regular Emitter get the content restored into the metadata first
- * ({@code EmitDataImpl#restoreContentFromBytes()}), so the move only sticks for
- * consumers on the passback path.
+ * content. tika-server sets it for the raw-output endpoints. Results routed to
+ * a regular Emitter get the content restored first (see
+ * {@code EmitDataImpl#restoreContentFromBytes}).
*/
@TikaComponent(name = "content-bytes-config")
public class ContentBytesConfig implements Serializable {
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
index 658dcaaf51..bff48304cb 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
@@ -80,9 +80,8 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> {
continue;
}
- // Per-parse runtime state (ParseRecord, ParseTimeout, parser-map cache)
- // is never configuration; a context that has been through a parse must
- // still serialize.
+ // per-parse runtime state (ParseRecord, ParseTimeout, ...) is never
+ // configuration; a context that has been through a parse must still serialize
if (value instanceof TransientParseState) {
continue;
}
(tika) 02/13: don't double detect
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 17e67b4d311d83ae298c3cdefbc19d287ef7e889
Author: tallison <ta...@apache.org>
AuthorDate: Tue Sep 1 17:11:18 2026 -0400
don't double detect
---
CHANGES.txt | 7 +++++++
.../src/main/java/org/apache/tika/detect/CompositeDetector.java | 2 +-
.../src/main/java/org/apache/tika/detect/DefaultDetector.java | 9 +++++++++
3 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 397b2915b8..8c2b56af66 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,12 @@
Release 4.1.0 - unreleased
+ * DefaultDetector honors CONTENT_TYPE_USER_OVERRIDE and
+ CONTENT_TYPE_PARSER_OVERRIDE before running magic detection, matching
+ CompositeDetector's contract. Removes the second full magic scan every
+ pipes parse paid per document. Compat note: with a user override set,
+ DefaultDetector no longer lets a more specific magic result overrule
+ the override (TIKA-4868).
+
* Markdown output is ~4x faster on large documents:
ToMarkdownContentHandler now buffers the commonmark renderer's
per-character writes instead of paying the synchronized
diff --git a/tika-core/src/main/java/org/apache/tika/detect/CompositeDetector.java b/tika-core/src/main/java/org/apache/tika/detect/CompositeDetector.java
index 027b6f4c6d..158dbdeaec 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/CompositeDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/CompositeDetector.java
@@ -96,7 +96,7 @@ public class CompositeDetector implements Detector {
* @param metadata
* @return mediaType if a parseable mediatype was sent in via user or parser overrides
*/
- private static MediaType detectOverrides(Metadata metadata) {
+ protected static MediaType detectOverrides(Metadata metadata) {
String override = metadata.get(TikaCoreProperties.CONTENT_TYPE_USER_OVERRIDE);
if (!StringUtils.isBlank(override)) {
MediaType mt = MediaType.parse(override);
diff --git a/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java b/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
index 6b19b87867..bda7f5e6f9 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
@@ -135,6 +135,15 @@ public class DefaultDetector extends CompositeDetector {
@Override
public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext)
throws IOException {
+ // 0. An override short-circuits detection entirely, as in CompositeDetector.
+ // Without this check the magic scan below runs even when a prior detect pass
+ // recorded its result as CONTENT_TYPE_PARSER_OVERRIDE -- doubling detection
+ // cost on every pipes parse.
+ MediaType override = detectOverrides(metadata);
+ if (override != null) {
+ return override;
+ }
+
// 1. Magic detection via MimeTypes
MediaType magicType = mimeTypes.detect(tis, metadata, parseContext);
metadata.set(TikaCoreProperties.CONTENT_TYPE_MAGIC_DETECTED, magicType.toString());
(tika) 05/13: improve detector efficiency
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6ad41e0f200e6b47a7f4aa136ecef67eae55104f
Author: tallison <ta...@apache.org>
AuthorDate: Tue Sep 1 21:27:33 2026 -0400
improve detector efficiency
---
CHANGES.txt | 17 ++++++
.../java/org/apache/tika/detect/MagicDetector.java | 22 ++++++--
.../main/java/org/apache/tika/mime/MagicMatch.java | 19 +++++--
.../main/java/org/apache/tika/mime/MimeTypes.java | 61 ++++++++++++++++------
.../main/java/org/apache/tika/mime/Patterns.java | 14 ++++-
.../org/apache/tika/mime/tika-mimetypes.xml | 6 ++-
6 files changed, 112 insertions(+), 27 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index db2f14e11f..8ef82bedf1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,22 @@
Release 4.1.0 - unreleased
+ * Detection hot-path cleanups: MagicMatch resolves its detector via
+ double-checked locking instead of a synchronized method per eval;
+ glob patterns are compiled once at registration instead of per
+ lookup; MimeTypes.forName reads a ConcurrentHashMap (fixing an
+ unsynchronized-read race) and indexes the normalized key so
+ parameterized aliases stop re-locking per call; resource names
+ containing spaces skip the URI-parse-by-exception; the magic-header
+ buffer is sized by the stream's known length instead of a fixed
+ 64KB per detection; and the Adobe Illustrator ranged regex is
+ gated behind a literal scan (detection results unchanged,
+ ~28us off every non-AI detection) (TIKA-4868).
+
+ * Magic detection is ~35% faster on unmatched (e.g. plain-text) input:
+ MagicDetector range scans find first-byte candidates with a tight
+ scan before running the full masked/case-folded compare, instead of
+ paying it at every offset in the range (TIKA-4868).
+
* CSVSniffer reads its detection window once into a shared buffer and
runs every delimiter hypothesis against it, instead of re-reading the
stream through a pushback/mark-reset stack per delimiter; windows with
diff --git a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
index 80a7725453..839b722c3a 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
@@ -490,15 +490,29 @@ public class MagicDetector implements Detector {
}
}
} else {
- // Loop until we've covered the entire offset range
+ // Range scans (e.g. "\nHeader:" over 0:1024) dominate detection time, so
+ // find first-byte candidates with a tight scan and run the full compare
+ // only there, instead of paying the masked/case machinery at every offset.
+ if (length == 0) {
+ // degenerate empty pattern: preserves the old loop's outcome
+ return startOffset <= endOffset && startOffset <= buffer.length;
+ }
+ int first = pattern[0];
+ byte firstMask = mask[0];
for (int i = startOffset; i <= endOffset; i++) {
if (i + length > buffer.length) {
break;
}
+ int masked0 = buffer[i] & firstMask;
+ if (this.isStringIgnoreCase) {
+ masked0 = Character.toLowerCase(masked0);
+ }
+ if (masked0 != first) {
+ continue;
+ }
boolean match = true;
- int masked;
- for (int j = 0; match && j < length; j++) {
- masked = (buffer[i + j] & mask[j]);
+ for (int j = 1; match && j < length; j++) {
+ int masked = (buffer[i + j] & mask[j]);
if (this.isStringIgnoreCase) {
masked = Character.toLowerCase(masked);
}
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java b/tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java
index 835d4b90c4..b4ed3a9733 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java
@@ -33,7 +33,7 @@ class MagicMatch implements Clause {
private final String mask;
- private MagicDetector detector = null;
+ private volatile MagicDetector detector = null;
MagicMatch(MediaType mediaType, String type, String offset, String value, String mask) {
this.mediaType = mediaType;
@@ -43,11 +43,20 @@ class MagicMatch implements Clause {
this.mask = mask;
}
- private synchronized MagicDetector getDetector() {
- if (detector == null) {
- detector = MagicDetector.parse(mediaType, type, offset, value, mask);
+ private MagicDetector getDetector() {
+ // Double-checked: eval() runs per magic per detect call, so a synchronized
+ // method here means thousands of monitor acquisitions per detection.
+ MagicDetector d = detector;
+ if (d == null) {
+ synchronized (this) {
+ d = detector;
+ if (d == null) {
+ d = MagicDetector.parse(mediaType, type, offset, value, mask);
+ detector = d;
+ }
+ }
}
- return detector;
+ return d;
}
public boolean eval(byte[] data) {
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
index c178729cde..eb0657c3a3 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
@@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import javax.xml.namespace.QName;
import org.apache.tika.Tika;
@@ -103,7 +104,8 @@ public final class MimeTypes implements Detector, Serializable {
/**
* All the registered MimeTypes indexed on their canonical names
*/
- private final Map<MediaType, MimeType> types = new HashMap<>();
+ // ConcurrentHashMap: forName reads this outside the lock that mutates it
+ private final Map<MediaType, MimeType> types = new ConcurrentHashMap<>();
/**
* The patterns matcher
*/
@@ -318,11 +320,23 @@ public final class MimeTypes implements Detector, Serializable {
* @throws IOException if the stream can not be read
*/
byte[] readMagicHeader(InputStream stream) throws IOException {
+ return readMagicHeader(stream, getMinLength());
+ }
+
+ /**
+ * Like {@link #readMagicHeader(InputStream)} but sized by {@code maxBytes} when the
+ * caller knows the stream is shorter than {@link #getMinLength()} -- avoids a fresh
+ * 64KB allocation per detection for every small (e.g. embedded) document.
+ */
+ byte[] readMagicHeader(InputStream stream, int maxBytes) throws IOException {
if (stream == null) {
throw new IllegalArgumentException("InputStream is missing");
}
+ if (maxBytes == 0) {
+ return new byte[0];
+ }
- byte[] bytes = new byte[getMinLength()];
+ byte[] bytes = new byte[maxBytes];
int totalRead = 0;
int lastRead = stream.read(bytes);
@@ -364,7 +378,9 @@ public final class MimeTypes implements Detector, Serializable {
if (mime == null) {
mime = new MimeType(type);
add(mime);
- types.put(type, mime);
+ // add() indexed the raw type; index the normalized key too, or an
+ // aliased parameterized name would miss (and re-lock) on every call
+ types.putIfAbsent(normalisedType, mime);
}
}
}
@@ -525,9 +541,17 @@ public final class MimeTypes implements Detector, Serializable {
// Get type based on magic prefix
if (tis != null) {
+ int toRead = getMinLength();
+ // hasLength() is non-forcing; getLength() would only spool when unknown
+ if (tis.hasLength()) {
+ long known = tis.getLength() - tis.getPosition();
+ if (known >= 0 && known < toRead) {
+ toRead = (int) known;
+ }
+ }
tis.mark(getMinLength());
try {
- byte[] prefix = readMagicHeader(tis);
+ byte[] prefix = readMagicHeader(tis, toRead);
possibleTypes = getMimeType(prefix);
} finally {
tis.reset();
@@ -541,21 +565,26 @@ public final class MimeTypes implements Detector, Serializable {
boolean isHttp = false;
// Deal with a URI or a path name in as the resource name
- try {
- URI uri = new URI(resourceName);
- String scheme = uri.getScheme();
- isHttp = scheme != null && scheme.startsWith("http"); // http or https
- String path = uri.getPath();
- if (path != null) {
- int slash = path.lastIndexOf('/');
- if (slash + 1 < path.length()) {
- name = path.substring(slash + 1);
+ // A space guarantees URISyntaxException; skip the constructor (and its
+ // exception fill-in) for that common embedded-resource case.
+ if (resourceName.indexOf(' ') >= 0) {
+ name = resourceName;
+ } else {
+ try {
+ URI uri = new URI(resourceName);
+ String scheme = uri.getScheme();
+ isHttp = scheme != null && scheme.startsWith("http"); // http or https
+ String path = uri.getPath();
+ if (path != null) {
+ int slash = path.lastIndexOf('/');
+ if (slash + 1 < path.length()) {
+ name = path.substring(slash + 1);
+ }
}
+ } catch (URISyntaxException e) {
+ name = resourceName;
}
- } catch (URISyntaxException e) {
- name = resourceName;
}
-
if (name != null) {
MimeType hint = getMimeType(name);
diff --git a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
index 48c0329f06..8cc8ce9728 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.regex.Pattern;
/**
* Defines a MimeType pattern.
@@ -49,6 +50,13 @@ class Patterns implements Serializable {
*/
private final SortedMap<String, MimeType> globs =
new TreeMap<>(new LengthComparator());
+
+ /**
+ * Compiled forms of {@link #globs}' keys. Matching recompiled every glob regex
+ * per lookup before; for names that miss the name/extension indexes that was
+ * a Pattern.compile per glob per call.
+ */
+ private final Map<String, Pattern> compiledGlobs = new HashMap<>();
private int minExtensionLength = Integer.MAX_VALUE;
private int maxExtensionLength = 0;
@@ -116,6 +124,7 @@ class Patterns implements Serializable {
MimeType previous = globs.get(glob);
if (previous == null || registry.isSpecializationOf(previous.getType(), type.getType())) {
globs.put(glob, type);
+ compiledGlobs.put(glob, Pattern.compile(glob));
} else if (previous == type ||
registry.isSpecializationOf(type.getType(), previous.getType())) {
// do nothing
@@ -159,7 +168,10 @@ class Patterns implements Serializable {
// And finally, try complex regexp matching
for (Map.Entry<String, MimeType> entry : globs.entrySet()) {
- if (name.matches(entry.getKey())) {
+ Pattern glob = compiledGlobs.get(entry.getKey());
+ boolean matched = glob != null ? glob.matcher(name).matches()
+ : name.matches(entry.getKey());
+ if (matched) {
return entry.getValue();
}
}
diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index 62a18b7840..9cb4a899bc 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -312,7 +312,11 @@
<!-- we can do more specific versions by focusing on the integer in the regex below -->
<tika:link>http://justsolve.archiveteam.org/wiki/Adobe_Illustrator_Artwork</tika:link>
<magic priority="60">
- <match value="[\r\n]%AI5_FileFormat [1-4][\r\n]" type="regex" offset="0:8192"/>
+ <!-- literal gate first: the ranged regex only runs when the marker exists,
+ so non-AI files pay a cheap string scan instead of 8K regex attempts -->
+ <match value="%AI5_FileFormat " type="string" offset="0:8192">
+ <match value="[\r\n]%AI5_FileFormat [1-4][\r\n]" type="regex" offset="0:8192"/>
+ </match>
</magic>
<sub-class-of type="application/postscript"/>
</mime-type>
(tika) 12/13: TIKA-4868 fixes/reviewer feedback
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit a864bc897e203f2c94dd814db7ab2c058e9dfc19
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 09:38:01 2026 -0400
TIKA-4868 fixes/reviewer feedback
---
CHANGES.txt | 17 +-
tika-app/src/main/resources/log4j2.xml | 2 +
.../java/org/apache/tika/config/ParseTimeout.java | 2 +-
.../apache/tika/config/TransientParseState.java | 26 +++
.../java/org/apache/tika/detect/MagicDetector.java | 18 +-
.../java/org/apache/tika/io/ByteArraySource.java | 5 +
.../java/org/apache/tika/io/CachingSource.java | 9 +
.../main/java/org/apache/tika/io/FileSource.java | 5 +
.../java/org/apache/tika/io/ReopenableSource.java | 9 +
.../java/org/apache/tika/io/TikaInputSource.java | 6 +
.../java/org/apache/tika/io/TikaInputStream.java | 46 +++--
.../main/java/org/apache/tika/mime/MimeTypes.java | 5 +-
.../main/java/org/apache/tika/mime/Patterns.java | 22 ++-
.../org/apache/tika/parser/CompositeParser.java | 20 +-
.../java/org/apache/tika/parser/ParseRecord.java | 3 +-
.../apache/tika/sax/FastMarkdownTextRenderer.java | 9 +-
.../apache/tika/sax/ToMarkdownContentHandler.java | 3 +-
.../org/apache/tika/io/DeclaredLengthTest.java | 111 ++++++++++++
.../org/apache/tika/mime/MimeDetectionTest.java | 14 ++
.../java/org/apache/tika/mime/PatternsTest.java | 16 ++
.../tika/sax/FastMarkdownTextRendererTest.java | 201 +++++++++++++++++++++
.../tika/sax/ToMarkdownContentHandlerTest.java | 18 ++
.../tika/parser/microsoft/WordExtractor.java | 4 -
.../org/apache/tika/parser/csv/CSVSniffer.java | 12 +-
.../apache/tika/pipes/core/ContentBytesConfig.java | 5 +-
.../tika/pipes/core/async/AsyncProcessor.java | 6 +
.../tika/pipes/core/emitter/EmitDataImpl.java | 20 ++
.../apache/tika/pipes/core/server/EmitHandler.java | 7 +-
.../serdes/ParseContextSerializer.java | 8 +
.../TestParseContextSerialization.java | 15 ++
.../tika/server/core/resource/TikaResource.java | 6 +-
.../tika-server-core/src/main/resources/log4j2.xml | 2 +
.../src/main/resources/log4j2.xml | 2 +
33 files changed, 591 insertions(+), 63 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index f125119319..f0a5cf99d3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -33,8 +33,9 @@ Release 4.1.0 - unreleased
(4.1.0-dev ran it twice) to 52us per document; eml requests ~-45%,
ppt ~-35% in tika-server. MagicDetector also precomputes a 256-entry
first-byte table per pattern (mask and case fold become one array
- load per scanned position). Adds an opt-in RESOURCE_TIMING log on
- org.apache.tika.pipes.timing.resource (TIKA-4868).
+ load per scanned position). Adds a RESOURCE_TIMING log on
+ org.apache.tika.pipes.timing.resource, silenced by default in the
+ shipped log4j2 configs (TIKA-4868).
* WordExtractor (.doc) cleans each character run in one pass instead of
four chained replace/replaceAll copies, and tests paragraph blankness
@@ -77,15 +78,19 @@ Release 4.1.0 - unreleased
* Pipes workers no longer stall between pre-parse and parse waiting
for the client to acknowledge the intermediate-result frame; the
- ACK round trip now overlaps the parse. Adds opt-in per-request
- timing logs on org.apache.tika.pipes.timing.* (TIKA-4868).
+ ACK round trip now overlaps the parse. Adds per-request timing logs
+ on org.apache.tika.pipes.timing.*, silenced by default in the shipped
+ log4j2 configs; raise that logger to info to enable (TIKA-4868).
* DefaultDetector honors CONTENT_TYPE_USER_OVERRIDE and
CONTENT_TYPE_PARSER_OVERRIDE before running magic detection, matching
CompositeDetector's contract. Removes the second full magic scan every
- pipes parse paid per document. Compat note: with a user override set,
+ pipes parse paid per document. Compat note: with either override set,
DefaultDetector no longer lets a more specific magic result overrule
- the override (TIKA-4868).
+ it; parts whose parser declares a type from container headers (e.g.
+ inline text/* mail parts) now report the declared type, and
+ CONTENT_TYPE_MAGIC_DETECTED is not recorded when an override short
+ circuits detection (TIKA-4868).
* Markdown output is ~4x faster on large documents:
ToMarkdownContentHandler now buffers the commonmark renderer's
diff --git a/tika-app/src/main/resources/log4j2.xml b/tika-app/src/main/resources/log4j2.xml
index c88e66e99e..d22d891023 100644
--- a/tika-app/src/main/resources/log4j2.xml
+++ b/tika-app/src/main/resources/log4j2.xml
@@ -25,6 +25,8 @@
</Console>
</Appenders>
<Loggers>
+ <!-- opt-in per-request timing lines (TIKA-4868): raise to info to enable -->
+ <Logger name="org.apache.tika.pipes.timing" level="warn"/>
<Root level="info">
<AppenderRef ref="Console"/>
</Root>
diff --git a/tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java b/tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java
index 4f9d385907..df95726f46 100644
--- a/tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java
+++ b/tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java
@@ -49,7 +49,7 @@ import org.apache.tika.parser.ParseContext;
*
* @since Apache Tika 4.0
*/
-public class ParseTimeout {
+public class ParseTimeout implements TransientParseState {
private static final Logger LOG = LoggerFactory.getLogger(ParseTimeout.class);
diff --git a/tika-core/src/main/java/org/apache/tika/config/TransientParseState.java b/tika-core/src/main/java/org/apache/tika/config/TransientParseState.java
new file mode 100644
index 0000000000..cf3d0ef3f2
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/config/TransientParseState.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.config;
+
+/**
+ * Marker for per-parse runtime state that parsers store in a
+ * {@link org.apache.tika.parser.ParseContext} (e.g. {@code ParseRecord},
+ * {@code ParseTimeout}). Such entries are never configuration: serializers
+ * skip them instead of failing on an unregistered component.
+ */
+public interface TransientParseState {
+}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
index d9d6b2ad2d..2af36ee54f 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java
@@ -459,15 +459,6 @@ public class MagicDetector implements Detector {
}
}
- /**
- * Core matching logic that checks if the pattern matches anywhere in the buffer
- * within the specified offset range.
- *
- * @param buffer the byte array to search in
- * @param startOffset the first position in the buffer to start matching (inclusive)
- * @param endOffset the last position in the buffer to start matching (inclusive)
- * @return true if a match is found, false otherwise
- */
// Which raw first bytes can begin a match; encodes mask[0] and the case fold so the
// scan loop is a single table load. Built lazily, idempotent under racing builds.
private transient volatile boolean[] firstByteMatches;
@@ -485,6 +476,15 @@ public class MagicDetector implements Detector {
return table;
}
+ /**
+ * Core matching logic that checks if the pattern matches anywhere in the buffer
+ * within the specified offset range.
+ *
+ * @param buffer the byte array to search in
+ * @param startOffset the first position in the buffer to start matching (inclusive)
+ * @param endOffset the last position in the buffer to start matching (inclusive)
+ * @return true if a match is found, false otherwise
+ */
private boolean matchesBuffer(byte[] buffer, int startOffset, int endOffset) {
if (this.isRegex) {
int bufferLen = Math.min(buffer.length - startOffset, length + (endOffset - startOffset));
diff --git a/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java b/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java
index d5eee0b07d..76a3842ce7 100644
--- a/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java
+++ b/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java
@@ -123,6 +123,11 @@ class ByteArraySource extends InputStream implements TikaInputSource {
return spilledPath;
}
+ @Override
+ public boolean hasReliableLength() {
+ return true;
+ }
+
@Override
public long getLength() {
return length;
diff --git a/tika-core/src/main/java/org/apache/tika/io/CachingSource.java b/tika-core/src/main/java/org/apache/tika/io/CachingSource.java
index 8b424696c7..47255ccfa4 100644
--- a/tika-core/src/main/java/org/apache/tika/io/CachingSource.java
+++ b/tika-core/src/main/java/org/apache/tika/io/CachingSource.java
@@ -49,6 +49,8 @@ class CachingSource extends InputStream implements TikaInputSource {
// temp-file suffix for threshold spills, which precede any getPath(suffix) call
private final String suffix;
private long length;
+ // getLength() is a declared hint until a drain/spill measures it
+ private boolean lengthMeasured;
// Passthrough mode: just a BufferedInputStream
private BufferedInputStream passthroughStream;
@@ -224,6 +226,7 @@ class CachingSource extends InputStream implements TikaInputSource {
// Record the drained length like getPath() does (feeds SecureContentHandler's
// zip-bomb ratio)
length = channel.size();
+ lengthMeasured = true;
if (metadata != null &&
StringUtils.isBlank(metadata.get(HttpHeaders.CONTENT_LENGTH))) {
metadata.set(HttpHeaders.CONTENT_LENGTH, Long.toString(length));
@@ -309,6 +312,7 @@ class CachingSource extends InputStream implements TikaInputSource {
// The spooled size is ground truth, even when it is 0 and a
// Content-Length claimed otherwise
length = Files.size(spilledPath);
+ lengthMeasured = true;
// Update metadata if not already set
if (metadata != null &&
@@ -326,6 +330,11 @@ class CachingSource extends InputStream implements TikaInputSource {
return length;
}
+ @Override
+ public boolean hasReliableLength() {
+ return lengthMeasured;
+ }
+
// seekTo() reopens fileStream, so the registered resource must close whichever
// handle is current rather than the one open at registration time.
private void closeFileStream() throws IOException {
diff --git a/tika-core/src/main/java/org/apache/tika/io/FileSource.java b/tika-core/src/main/java/org/apache/tika/io/FileSource.java
index 257cbacbc3..a57abf7ba9 100644
--- a/tika-core/src/main/java/org/apache/tika/io/FileSource.java
+++ b/tika-core/src/main/java/org/apache/tika/io/FileSource.java
@@ -115,6 +115,11 @@ class FileSource extends InputStream implements TikaInputSource {
return path;
}
+ @Override
+ public boolean hasReliableLength() {
+ return true;
+ }
+
@Override
public long getLength() {
return length;
diff --git a/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java b/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java
index 332e56eba3..8cbaa66881 100644
--- a/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java
+++ b/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java
@@ -53,6 +53,8 @@ class ReopenableSource extends InputStream implements TikaInputSource {
private final TemporaryResources tmp;
private final String suffix;
private long length;
+ // getLength() is a declared hint until a drain/spill measures it
+ private boolean lengthMeasured;
private InputStream currentStream; // lazily opened
private long position;
@@ -160,6 +162,7 @@ class ReopenableSource extends InputStream implements TikaInputSource {
spilledPath = p;
// The spooled size is ground truth, even over a lying declared length
length = Files.size(p);
+ lengthMeasured = true;
}
return spilledPath;
}
@@ -169,6 +172,11 @@ class ReopenableSource extends InputStream implements TikaInputSource {
return length;
}
+ @Override
+ public boolean hasReliableLength() {
+ return lengthMeasured;
+ }
+
@Override
public void enableRewind(CacheMemoryBudget budget) throws IOException {
if (position != 0) {
@@ -291,6 +299,7 @@ class ReopenableSource extends InputStream implements TikaInputSource {
retainedReservation = reservedHere;
// The full read is ground truth, even over a lying declared length
length = total;
+ lengthMeasured = true;
return true;
}
diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java b/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java
index 9e7b4a8d30..9f3dc96146 100644
--- a/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java
+++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java
@@ -63,6 +63,12 @@ interface TikaInputSource extends Closeable {
*/
long getLength();
+ /**
+ * True when {@link #getLength()} is ground truth (file size, byte array,
+ * fully-drained cache) rather than a caller-declared hint, which may lie.
+ */
+ boolean hasReliableLength();
+
/**
* Enables full rewind capability.
* <p>
diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java
index b58cd548db..ae62397b9e 100644
--- a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java
+++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java
@@ -100,7 +100,8 @@ public class TikaInputStream extends TaggedInputStream {
return (TikaInputStream) stream;
}
String ext = getExtension(metadata);
- TikaInputSource inputSource = new CachingSource(stream, tmp, -1, metadata, ext);
+ TikaInputSource inputSource =
+ new CachingSource(stream, tmp, declaredLength(metadata), metadata, ext);
return new TikaInputStream(inputSource, tmp, ext);
}
@@ -123,18 +124,8 @@ public class TikaInputStream extends TaggedInputStream {
throw new NullPointerException("The opener must not be null");
}
String ext = getExtension(metadata);
- long length = -1;
- if (metadata != null) {
- String cl = metadata.get(HttpHeaders.CONTENT_LENGTH);
- if (cl != null) {
- try {
- length = Long.parseLong(cl);
- } catch (NumberFormatException e) {
- length = -1;
- }
- }
- }
- TikaInputSource inputSource = new ReopenableSource(opener, tmp, length, ext);
+ TikaInputSource inputSource =
+ new ReopenableSource(opener, tmp, declaredLength(metadata), ext);
return new TikaInputStream(inputSource, tmp, ext);
}
@@ -284,6 +275,21 @@ public class TikaInputStream extends TaggedInputStream {
return tis;
}
+ private static long declaredLength(Metadata metadata) {
+ if (metadata == null) {
+ return -1;
+ }
+ String cl = metadata.get(HttpHeaders.CONTENT_LENGTH);
+ if (cl == null) {
+ return -1;
+ }
+ try {
+ return Long.parseLong(cl);
+ } catch (NumberFormatException e) {
+ return -1;
+ }
+ }
+
private static String getExtension(Metadata metadata) {
if (metadata == null) {
return StringUtils.EMPTY;
@@ -440,6 +446,20 @@ public class TikaInputStream extends TaggedInputStream {
return source != null && source.getLength() != -1;
}
+ /**
+ * True when {@link #getLength()} would return a measured, ground-truth length
+ * (file, byte array, fully-drained cache, explicit override) without forcing a
+ * spool. False when the only length available is a caller-declared hint
+ * (Content-Length metadata, HTTP header, archive central directory), which may lie.
+ */
+ public boolean hasReliableLength() {
+ if (overrideLength >= 0) {
+ return true;
+ }
+ TikaInputSource source = inputSource();
+ return source != null && source.hasReliableLength() && source.getLength() != -1;
+ }
+
/**
* The stream length. For a stream-backed instance with no declared length this
* spools the entire remaining stream to a temporary file to measure it.
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
index eb0657c3a3..d2e9f082ff 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
@@ -542,8 +542,9 @@ public final class MimeTypes implements Detector, Serializable {
// Get type based on magic prefix
if (tis != null) {
int toRead = getMinLength();
- // hasLength() is non-forcing; getLength() would only spool when unknown
- if (tis.hasLength()) {
+ // Only a measured length may shrink the read: a lying declared
+ // Content-Length would silently truncate the magic prefix.
+ if (tis.hasReliableLength()) {
long known = tis.getLength() - tis.getPosition();
if (known >= 0 && known < toRead) {
toRead = (int) known;
diff --git a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
index 8cc8ce9728..f5f8c234e0 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java
@@ -54,9 +54,10 @@ class Patterns implements Serializable {
/**
* Compiled forms of {@link #globs}' keys. Matching recompiled every glob regex
* per lookup before; for names that miss the name/extension indexes that was
- * a Pattern.compile per glob per call.
+ * a Pattern.compile per glob per call. Transient with a lazy rebuild so a
+ * serialized form from a build without this field still deserializes.
*/
- private final Map<String, Pattern> compiledGlobs = new HashMap<>();
+ private transient Map<String, Pattern> compiledGlobs = new HashMap<>();
private int minExtensionLength = Integer.MAX_VALUE;
private int maxExtensionLength = 0;
@@ -120,11 +121,23 @@ class Patterns implements Serializable {
}
}
+ private Map<String, Pattern> compiledGlobs() {
+ Map<String, Pattern> compiled = compiledGlobs;
+ if (compiled == null) {
+ compiled = new HashMap<>();
+ for (String glob : globs.keySet()) {
+ compiled.put(glob, Pattern.compile(glob));
+ }
+ compiledGlobs = compiled;
+ }
+ return compiled;
+ }
+
private void addGlob(String glob, MimeType type) throws MimeTypeException {
MimeType previous = globs.get(glob);
if (previous == null || registry.isSpecializationOf(previous.getType(), type.getType())) {
globs.put(glob, type);
- compiledGlobs.put(glob, Pattern.compile(glob));
+ compiledGlobs().put(glob, Pattern.compile(glob));
} else if (previous == type ||
registry.isSpecializationOf(type.getType(), previous.getType())) {
// do nothing
@@ -167,8 +180,9 @@ class Patterns implements Serializable {
}
// And finally, try complex regexp matching
+ Map<String, Pattern> compiled = compiledGlobs();
for (Map.Entry<String, MimeType> entry : globs.entrySet()) {
- Pattern glob = compiledGlobs.get(entry.getKey());
+ Pattern glob = compiled.get(entry.getKey());
boolean matched = glob != null ? glob.matcher(name).matches()
: name.matches(entry.getKey());
if (matched) {
diff --git a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
index 4e97d96fc0..7d6a46e0b3 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
@@ -31,6 +31,7 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.apache.tika.config.ParseTimeout;
+import org.apache.tika.config.TransientParseState;
import org.apache.tika.exception.EmbeddedLimitReachedException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.WriteLimitReachedException;
@@ -247,16 +248,19 @@ public class CompositeParser implements Parser {
}
/**
- * Per-parse cache of built parser maps, stored in the ParseContext so every
- * embedded document in a parse reuses the container's map instead of
- * rebuilding it (a full walk of every parser's supported types). Keyed by
- * parser instance because nested composites share one context. The map is
- * built once per (parser, context); a context entry that would change a
- * parser's supported types mid-parse is not picked up until the next parse.
+ * Cache of built parser maps, stored in the ParseContext so every embedded
+ * document in a parse reuses the container's map instead of rebuilding it
+ * (a full walk of every parser's supported types). Keyed by parser instance
+ * because nested composites share one context. The map is built once per
+ * (parser, context) and lives as long as the context: a context entry that
+ * would change a parser's supported types is not picked up mid-parse, nor on
+ * later parses that reuse the same ParseContext instance.
*/
- private static final class ParserMapCache {
+ private static final class ParserMapCache implements TransientParseState {
+ // synchronized: contexts are occasionally shared across threads, and a
+ // concurrent put into a bare IdentityHashMap can corrupt it
private final Map<CompositeParser, Map<MediaType, Parser>> maps =
- new IdentityHashMap<>();
+ Collections.synchronizedMap(new IdentityHashMap<>());
}
private Map<MediaType, Parser> getParsersCached(ParseContext context) {
diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java b/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java
index 88100d532e..84ce139b2d 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java
@@ -23,6 +23,7 @@ import java.util.Set;
import org.apache.tika.config.EmbeddedLimits;
import org.apache.tika.config.TimeoutLimits;
+import org.apache.tika.config.TransientParseState;
import org.apache.tika.metadata.Metadata;
/**
@@ -34,7 +35,7 @@ import org.apache.tika.metadata.Metadata;
* which can be configured via {@link #setMaxEmbeddedDepth(int)} and
* {@link #setMaxEmbeddedCount(int)}.
*/
-public class ParseRecord {
+public class ParseRecord implements TransientParseState {
//hard limits so that specially crafted files
//don't cause an OOM
diff --git a/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java b/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
index ad68db7261..e821ff35c9 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
@@ -40,10 +40,11 @@ import org.commonmark.text.AsciiMatcher;
* {@code raw()} call.
* <p>
* The escaping semantics replicate {@code CoreMarkdownNodeRenderer#visit(Text)} for
- * commonmark 0.27.x exactly: the line-start disambiguation cases, the heading escape
- * variant, the {@code !}-before-link case, and the {@code \n} numeric reference. The
- * differential test in {@code ToMarkdownContentHandlerTest} and the corpus differential
- * guard the equivalence.
+ * the commonmark version pinned in tika-parent exactly: the line-start disambiguation
+ * cases, the heading escape variant, the {@code !}-before-link case, and the
+ * {@code \n} numeric reference. {@code FastMarkdownTextRendererTest} renders ASTs
+ * through this renderer and the stock one and requires byte-identical output; a
+ * commonmark upgrade that changes escaping fails there and must be re-synced here.
* <p>
* Text inside a table cell renders through the stock per-char path: the tables extension
* pushes a raw-escape for {@code |} onto the writer there, and hand-emitted escapes would
diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
index a6614bffae..6a958b1a7d 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
@@ -90,7 +90,8 @@ public class ToMarkdownContentHandler extends DefaultHandler {
private final Writer writer;
private final MarkdownRenderer renderer =
MarkdownRenderer.builder().extensions(EXTENSIONS)
- // registered first, so it wins Text rendering from the core renderer
+ // registered before the core fallback, which is the only other
+ // renderer claiming Text, so first-wins registration picks this one
.nodeRendererFactory(FastMarkdownTextRenderer.FACTORY).build();
private final Document document = new Document();
diff --git a/tika-core/src/test/java/org/apache/tika/io/DeclaredLengthTest.java b/tika-core/src/test/java/org/apache/tika/io/DeclaredLengthTest.java
new file mode 100644
index 0000000000..1a9a6f5c01
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/io/DeclaredLengthTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.io;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * Declared-vs-measured length semantics of {@link TikaInputStream}: a
+ * Content-Length hint is served without a spool but is never "reliable";
+ * a measured length (byte array, file, spool) is.
+ */
+public class DeclaredLengthTest {
+
+ @TempDir
+ Path tmpDir;
+
+ @Test
+ public void testStreamHonorsDeclaredLength() throws Exception {
+ byte[] data = "0123456789".getBytes(UTF_8);
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_LENGTH, "7");
+ try (TikaInputStream tis = TikaInputStream.get(
+ new ByteArrayInputStream(data), new TemporaryResources(), metadata)) {
+ assertTrue(tis.hasLength());
+ assertFalse(tis.hasReliableLength(), "declared length is a hint, not ground truth");
+ // declared value served without forcing a spool
+ assertEquals(7, tis.getLength());
+ assertFalse(tis.hasFile());
+ }
+ }
+
+ @Test
+ public void testSpoolMeasuresOverDeclaredLength() throws Exception {
+ byte[] data = "0123456789".getBytes(UTF_8);
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_LENGTH, "7");
+ try (TikaInputStream tis = TikaInputStream.get(
+ new ByteArrayInputStream(data), new TemporaryResources(), metadata)) {
+ tis.getPath();
+ assertTrue(tis.hasReliableLength());
+ assertEquals(data.length, tis.getLength(), "spooled size wins over the lying hint");
+ }
+ }
+
+ @Test
+ public void testStreamWithNoDeclaredLengthStillSpoolsToMeasure() throws Exception {
+ byte[] data = "0123456789".getBytes(UTF_8);
+ try (TikaInputStream tis = TikaInputStream.get(
+ new ByteArrayInputStream(data), new TemporaryResources(), new Metadata())) {
+ assertFalse(tis.hasLength());
+ assertEquals(data.length, tis.getLength());
+ assertTrue(tis.hasReliableLength());
+ }
+ }
+
+ @Test
+ public void testByteArrayAndFileAreReliable() throws Exception {
+ try (TikaInputStream tis = TikaInputStream.get("abc".getBytes(UTF_8))) {
+ assertTrue(tis.hasReliableLength());
+ assertEquals(3, tis.getLength());
+ }
+ Path f = tmpDir.resolve("len.bin");
+ java.nio.file.Files.write(f, "abcd".getBytes(UTF_8));
+ try (TikaInputStream tis = TikaInputStream.get(f)) {
+ assertTrue(tis.hasReliableLength());
+ assertEquals(4, tis.getLength());
+ }
+ }
+
+ @Test
+ public void testReopenableDeclaredLengthNotReliableUntilSpool() throws Exception {
+ byte[] data = "0123456789".getBytes(UTF_8);
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_LENGTH, "3");
+ try (TikaInputStream tis = TikaInputStream.get(
+ () -> new ByteArrayInputStream(data), new TemporaryResources(), metadata)) {
+ assertTrue(tis.hasLength());
+ assertFalse(tis.hasReliableLength());
+ assertEquals(3, tis.getLength());
+ tis.getPath();
+ assertTrue(tis.hasReliableLength());
+ assertEquals(data.length, tis.getLength());
+ }
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
index 5748f185c1..a3794f3184 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
@@ -306,4 +306,18 @@ public class MimeDetectionTest {
public void testPNGWithSomeEmlHeaders() throws IOException {
testFile("image/png", "test-pngNotEml.bin");
}
+
+ @Test
+ public void testLyingDeclaredLengthDoesNotTruncateMagic() throws Exception {
+ byte[] pdf = "%PDF-1.4\nsome pdf content".getBytes(UTF_8);
+ Metadata metadata = new Metadata();
+ // an under-declared Content-Length must not shrink the magic read
+ metadata.set(HttpHeaders.CONTENT_LENGTH, "2");
+ try (TikaInputStream tis = TikaInputStream.get(
+ new java.io.ByteArrayInputStream(pdf),
+ new org.apache.tika.io.TemporaryResources(), metadata)) {
+ assertEquals(MediaType.application("pdf"),
+ MIME_TYPES.detect(tis, metadata, new ParseContext()));
+ }
+ }
}
diff --git a/tika-core/src/test/java/org/apache/tika/mime/PatternsTest.java b/tika-core/src/test/java/org/apache/tika/mime/PatternsTest.java
index 25721b15fd..9d3c7f3aa9 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/PatternsTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/PatternsTest.java
@@ -97,4 +97,20 @@ public class PatternsTest {
assertTrue(extensions.contains(".jpeg"));
}
+
+ @Test
+ public void testSerializationRoundTrip() throws Exception {
+ patterns.add("*ile*.txt", text);
+ java.io.ByteArrayOutputStream bos = new java.io.ByteArrayOutputStream();
+ try (java.io.ObjectOutputStream oos = new java.io.ObjectOutputStream(bos)) {
+ oos.writeObject(patterns);
+ }
+ Patterns copy;
+ try (java.io.ObjectInputStream ois = new java.io.ObjectInputStream(
+ new java.io.ByteArrayInputStream(bos.toByteArray()))) {
+ copy = (Patterns) ois.readObject();
+ }
+ // compiledGlobs is transient; the glob path must rebuild it lazily
+ assertEquals(text, copy.matches("file7.txt"));
+ }
}
diff --git a/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java b/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java
new file mode 100644
index 0000000000..9122e84269
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.sax;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.commonmark.Extension;
+import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension;
+import org.commonmark.ext.gfm.tables.TablesExtension;
+import org.commonmark.node.Document;
+import org.commonmark.node.Heading;
+import org.commonmark.node.Link;
+import org.commonmark.node.Node;
+import org.commonmark.node.Paragraph;
+import org.commonmark.node.Text;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Differential guard for {@link FastMarkdownTextRenderer}: every AST must render
+ * byte-identically through the fast renderer and through commonmark's stock
+ * renderer. If a commonmark upgrade changes {@code CoreMarkdownNodeRenderer}'s
+ * escaping, these tests fail and the fast renderer must be re-synced.
+ */
+public class FastMarkdownTextRendererTest {
+
+ private static final List<Extension> EXTENSIONS =
+ Arrays.asList(TablesExtension.create(), StrikethroughExtension.create());
+
+ private static final MarkdownRenderer FAST = MarkdownRenderer.builder()
+ .extensions(EXTENSIONS)
+ .nodeRendererFactory(FastMarkdownTextRenderer.FACTORY).build();
+
+ private static final MarkdownRenderer STOCK =
+ MarkdownRenderer.builder().extensions(EXTENSIONS).build();
+
+ private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build();
+
+ private static void assertSameRendering(Node document, String label) {
+ assertEquals(STOCK.render(document), FAST.render(document), label);
+ }
+
+ private static void assertSameRendering(String markdown) {
+ assertSameRendering(PARSER.parse(markdown), "diverged on: " + markdown);
+ }
+
+ private static void assertSameRendering(Node document) {
+ assertSameRendering(document, "diverged on hand-built AST");
+ }
+
+ private static Node paragraphs(String... literals) {
+ Document doc = new Document();
+ Paragraph p = new Paragraph();
+ for (String literal : literals) {
+ p.appendChild(new Text(literal));
+ }
+ doc.appendChild(p);
+ return doc;
+ }
+
+ @Test
+ public void testLineStartEscapes() {
+ // Literals whose first char would parse as block structure at line start
+ for (String literal : new String[]{
+ "- not a list", "-not a list", "-", "+ plus", "* star",
+ "# not a heading", "#hash", "#",
+ "= not setext", "=",
+ "1. not ordered", "12. also not", "123456789. limit", "1) paren",
+ "1234567890. ten digits is not a marker", "12x. no",
+ "> not a quote",
+ " leading space", "\tleading tab", " two spaces"}) {
+ assertSameRendering(paragraphs(literal), "diverged on literal: " + literal);
+ }
+ }
+
+ @Test
+ public void testSecondTextNodeNotAtLineStart() {
+ // The line-start branches must not fire mid-line
+ assertSameRendering(paragraphs("before ", "- mid", "# mid", "= mid", "12. mid"));
+ }
+
+ @Test
+ public void testEscapableCharacters() {
+ for (String literal : new String[]{
+ "a[b]c", "a<b>c", "a`b`c", "a*b*c", "a_b_c", "a&b;c", "a\\b",
+ "pipe | in text", "tilde ~~x~~", "", "plain text, no specials.",
+ "text\nwith\nnewlines", "\n", "trailing newline\n", "*", "\\"}) {
+ assertSameRendering(paragraphs(literal), "diverged on literal: " + literal);
+ }
+ }
+
+ @Test
+ public void testBangBeforeLink() {
+ Document doc = new Document();
+ Paragraph p = new Paragraph();
+ p.appendChild(new Text("see!"));
+ Link link = new Link("http://example.com", null);
+ link.appendChild(new Text("here"));
+ p.appendChild(link);
+ doc.appendChild(p);
+ assertSameRendering(doc, "bang before link");
+
+ // bang NOT followed by a link needs no escape
+ assertSameRendering(paragraphs("no link here!"));
+ }
+
+ @Test
+ public void testHeadingEscapeSet() {
+ for (String literal : new String[]{"plain", "with # hash", "with\nnewline", "a`b"}) {
+ Document doc = new Document();
+ Heading h = new Heading();
+ h.setLevel(2);
+ h.appendChild(new Text(literal));
+ doc.appendChild(h);
+ assertSameRendering(doc, "heading literal: " + literal);
+ }
+ }
+
+ @Test
+ public void testParsedDocuments() {
+ // Round-trips through the parser: escaped specials come back as raw Text
+ for (String markdown : new String[]{
+ "\\- not a list\n",
+ "\\# not a heading\n",
+ "12\\. not ordered\n",
+ "para one\n\npara two with \\*stars\\* and \\[brackets\\]\n",
+ "**bold** and _em_ and ~~strike~~\n",
+ "a paragraph\nwith a soft break\n",
+ "| a | b\\|c |\n|---|---|\n| d | *e* |\n",
+ "# heading `code` and *em*\n",
+ "> quoted \\> text\n",
+ "[link](http://example.com \"ti\\\"tle\") and \n"}) {
+ assertSameRendering(markdown);
+ }
+ }
+
+ @Test
+ public void testRandomizedDifferential() {
+ char[] alphabet = ("abc XYZ \t\n-#=!*_[]<>&`\\|~.)0129" +
+ "é中😀").toCharArray();
+ for (long seed = 0; seed < 50; seed++) {
+ Random random = new Random(seed);
+ StringBuilder sb = new StringBuilder();
+ int len = 1 + random.nextInt(80);
+ for (int i = 0; i < len; i++) {
+ sb.append(alphabet[random.nextInt(alphabet.length)]);
+ }
+ String literal = sb.toString();
+ // the alphabet splits astral chars into chars; skip broken pairs
+ if (!isWellFormed(literal)) {
+ continue;
+ }
+ assertSameRendering(paragraphs(literal), "seed " + seed + " literal: " + literal);
+ }
+ }
+
+ private static boolean isWellFormed(String s) {
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ if (Character.isHighSurrogate(c)) {
+ if (i + 1 >= s.length() || !Character.isLowSurrogate(s.charAt(i + 1))) {
+ return false;
+ }
+ i++;
+ } else if (Character.isLowSurrogate(c)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Test
+ public void testTableCellKeepsStockPath() {
+ // '|' inside a cell must come out escaped once, not twice
+ Node doc = PARSER.parse("| a\\|b | *c* |\n|---|---|\n| \\|start | end\\| |\n");
+ String stock = STOCK.render(doc);
+ String fast = FAST.render(doc);
+ assertEquals(stock, fast, "table cell rendering");
+ assertTrue(fast.contains("a\\|b"), "single-escaped pipe expected: " + fast);
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
index 26c8a3d6e5..2b060e504d 100644
--- a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
+++ b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
@@ -1520,4 +1520,22 @@ public class ToMarkdownContentHandlerTest {
handler.writePartialContentIfUnfinished();
assertEquals(afterEnd, writer.toString(), "writePartial after endDocument must be a no-op");
}
+
+ @Test
+ public void testRenderBufferBoundaries() throws Exception {
+ // Spans sized around the internal 8 KB render buffer: exact fit, one-over,
+ // buffer-sized single write (direct path), and a multi-drain run.
+ for (int size : new int[]{8191, 8192, 8193, 16384, 40000}) {
+ StringWriter writer = new StringWriter();
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler(writer);
+ String run = "a".repeat(size);
+ handler.startDocument();
+ startElement(handler, "p");
+ chars(handler, run);
+ endElement(handler, "p");
+ handler.endDocument();
+ assertTrue(writer.toString().contains(run),
+ "clean " + size + "-char run must survive buffering intact");
+ }
+ }
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
index 25ad90d49f..18efa9bcd2 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java
@@ -106,10 +106,6 @@ public class WordExtractor extends AbstractPOIFSExtractor {
return count;
}
- /**
- * Given a style name, return what tag should be used, and
- * what style should be applied to it.
- */
// matches the old regex [\r\n\s]+ ( \s == [ \t\n\x0B\f\r] ), without the
// full replaceAll copy it used to make per paragraph
private static boolean isBlankParagraph(String text) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
index 164eacfd15..0ef0cd53cf 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
@@ -21,6 +21,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -58,13 +59,18 @@ class CSVSniffer {
}
// Every snifflet examines the same window, so read it once into a buffer
// instead of once per delimiter through a mark/reset + pushback stack.
- char[] buf = new char[markLimit];
+ // Grown on demand: markLimit is user-configurable and most inputs are
+ // far smaller, so an eager char[markLimit] would be waste per parse.
+ char[] buf = new char[Math.min(markLimit, 8192)];
reader.mark(markLimit);
int len = 0;
try {
while (len < markLimit) {
- int n = reader.read(buf, len, markLimit - len);
- if (n == -1) {
+ if (len == buf.length) {
+ buf = Arrays.copyOf(buf, (int) Math.min((long) buf.length * 2, markLimit));
+ }
+ int n = reader.read(buf, len, buf.length - len);
+ if (n <= 0) {
break;
}
len += n;
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
index 5d00bbd6ff..a8f8d2c5e3 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
@@ -25,7 +25,10 @@ import org.apache.tika.annotation.TikaComponent;
* {@code TIKA_CONTENT} into raw UTF-8 bytes on the {@code EmitData}, which travel
* as binary over the IPC instead of a Smile-encoded string. A caller who sets this
* must read {@code EmitData.getContentBytes()}; the metadata no longer carries the
- * content. tika-server sets it for the raw-output endpoints.
+ * content. tika-server sets it for the raw-output endpoints. Results routed to a
+ * regular Emitter get the content restored into the metadata first
+ * ({@code EmitDataImpl#restoreContentFromBytes()}), so the move only sticks for
+ * consumers on the passback path.
*/
@TikaComponent(name = "content-bytes-config")
public class ContentBytesConfig implements Serializable {
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java
index 70849fa8c7..1a6b472bbc 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java
@@ -54,6 +54,7 @@ import org.apache.tika.pipes.core.PipesResults;
import org.apache.tika.pipes.core.RestartReason;
import org.apache.tika.pipes.core.ServerManager;
import org.apache.tika.pipes.core.SharedServerManager;
+import org.apache.tika.pipes.core.emitter.EmitDataImpl;
import org.apache.tika.pipes.core.emitter.EmitterManager;
import org.apache.tika.pipes.core.reporter.ReporterManager;
import org.apache.tika.plugins.TikaPluginManager;
@@ -484,6 +485,11 @@ public class AsyncProcessor implements Closeable {
long offerStart = System.currentTimeMillis();
if (shouldEmit(result)) {
+ if (result.emitData() instanceof EmitDataImpl emitDataImpl) {
+ // a client-side Emitter reads only the metadata list;
+ // undo any content-bytes move so content is not lost
+ emitDataImpl.restoreContentFromBytes();
+ }
LOG.trace("adding result to emitter queue: " + result.emitData());
boolean offered = emitDataTupleQueue.offer(
new EmitDataPair(t.getEmitKey().getEmitterId(), result.emitData()), MAX_OFFER_WAIT_MS,
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java
index ebe53de666..d7683b5e76 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java
@@ -16,9 +16,11 @@
*/
package org.apache.tika.pipes.core.emitter;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.api.emitter.EmitData;
import org.apache.tika.utils.StringUtils;
@@ -66,6 +68,24 @@ public class EmitDataImpl implements EmitData {
this.contentBytes = contentBytes;
}
+ /**
+ * Inverse of the content-bytes move: puts the content back in
+ * {@code TIKA_CONTENT} for consumers that only read the metadata list
+ * (e.g. a regular Emitter). No-op when there are no content bytes or
+ * the metadata already carries content.
+ */
+ public void restoreContentFromBytes() {
+ if (contentBytes == null || metadataList == null || metadataList.isEmpty()) {
+ return;
+ }
+ Metadata m = metadataList.get(0);
+ if (m.get(TikaCoreProperties.TIKA_CONTENT) == null) {
+ m.set(TikaCoreProperties.TIKA_CONTENT,
+ new String(contentBytes, StandardCharsets.UTF_8));
+ }
+ contentBytes = null;
+ }
+
public long getEstimatedSizeBytes() {
long sz = estimateSizeInBytes(getEmitKey(), getMetadataList(), containerStackTrace);
if (contentBytes != null) {
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
index 57bac19e86..7a73feb4a7 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
@@ -172,11 +172,12 @@ class EmitHandler {
return;
}
Metadata m = metadataList.get(0);
- String content = m.get(TikaCoreProperties.TIKA_CONTENT);
- if (content == null) {
+ String[] content = m.getValues(TikaCoreProperties.TIKA_CONTENT);
+ // remove() drops every value; only move when that loses nothing
+ if (content.length != 1) {
return;
}
- emitData.setContentBytes(content.getBytes(StandardCharsets.UTF_8));
+ emitData.setContentBytes(content[0].getBytes(StandardCharsets.UTF_8));
m.remove(TikaCoreProperties.TIKA_CONTENT);
}
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
index 592aaed460..658dcaaf51 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
@@ -29,6 +29,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import org.apache.tika.config.JsonConfig;
+import org.apache.tika.config.TransientParseState;
import org.apache.tika.config.loader.TikaObjectMapperFactory;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.serialization.ComponentNameResolver;
@@ -79,6 +80,13 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> {
continue;
}
+ // Per-parse runtime state (ParseRecord, ParseTimeout, parser-map cache)
+ // is never configuration; a context that has been through a parse must
+ // still serialize.
+ if (value instanceof TransientParseState) {
+ continue;
+ }
+
// Find the friendly component name — all serializable components must be registered
String keyName = ComponentNameResolver.getFriendlyName(value.getClass());
if (keyName == null) {
diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java b/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java
index d93e5924b1..5a5526b941 100644
--- a/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java
+++ b/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java
@@ -591,4 +591,19 @@ public class TestParseContextSerialization {
"the message must name the offending key: " + e.getMessage());
}
+
+ @Test
+ public void testTransientParseStateSkipped() throws Exception {
+ // A context that has been through a parse carries per-parse runtime state
+ // (ParseRecord, ParseTimeout); serialization must skip it, not throw.
+ ParseContext pc = new ParseContext();
+ pc.set(org.apache.tika.parser.ParseRecord.class,
+ org.apache.tika.parser.ParseRecord.newInstance(pc));
+ org.apache.tika.config.ParseTimeout.getOrCreate(pc);
+
+ String json = serializeParseContext(pc);
+ ObjectMapper mapper = createMapper();
+ JsonNode root = mapper.readTree(json);
+ assertEquals(0, root.size(), "transient parse state must not serialize: " + json);
+ }
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
index ab48d46e85..b955bde571 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
@@ -697,13 +697,13 @@ public class TikaResource {
// ==================== Internal methods ====================
- /**
- * Produces raw streaming output (text, html, xml, md) using pipes-based parsing.
- */
/** Per-request resource-layer latency; joins the pipes lines by adjacency at c=1. */
private static final org.slf4j.Logger TIMING_LOG =
org.slf4j.LoggerFactory.getLogger("org.apache.tika.pipes.timing.resource");
+ /**
+ * Produces raw streaming output (text, html, xml, md) using pipes-based parsing.
+ */
private Response produceRawOutput(TikaInputStream tis, Metadata metadata,
MultivaluedMap<String, String> httpHeaders,
String handlerTypeName) throws IOException {
diff --git a/tika-server/tika-server-core/src/main/resources/log4j2.xml b/tika-server/tika-server-core/src/main/resources/log4j2.xml
index c88e66e99e..d22d891023 100644
--- a/tika-server/tika-server-core/src/main/resources/log4j2.xml
+++ b/tika-server/tika-server-core/src/main/resources/log4j2.xml
@@ -25,6 +25,8 @@
</Console>
</Appenders>
<Loggers>
+ <!-- opt-in per-request timing lines (TIKA-4868): raise to info to enable -->
+ <Logger name="org.apache.tika.pipes.timing" level="warn"/>
<Root level="info">
<AppenderRef ref="Console"/>
</Root>
diff --git a/tika-server/tika-server-standard/src/main/resources/log4j2.xml b/tika-server/tika-server-standard/src/main/resources/log4j2.xml
index c88e66e99e..d22d891023 100644
--- a/tika-server/tika-server-standard/src/main/resources/log4j2.xml
+++ b/tika-server/tika-server-standard/src/main/resources/log4j2.xml
@@ -25,6 +25,8 @@
</Console>
</Appenders>
<Loggers>
+ <!-- opt-in per-request timing lines (TIKA-4868): raise to info to enable -->
+ <Logger name="org.apache.tika.pipes.timing" level="warn"/>
<Root level="info">
<AppenderRef ref="Console"/>
</Root>
(tika) 10/13: TIKA-4868: bulk-span markdown text rendering
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4868-performance-improvements
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 683e8fe9be8b5f2e8a54569aeed81383077a146a
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 08:23:14 2026 -0400
TIKA-4868: bulk-span markdown text rendering
---
CHANGES.txt | 6 +
.../apache/tika/sax/FastMarkdownTextRenderer.java | 204 +++++++++++++++++++++
.../apache/tika/sax/ToMarkdownContentHandler.java | 4 +-
3 files changed, 213 insertions(+), 1 deletion(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 7744bb009c..c2b46bd0ac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,11 @@
Release 4.1.0 - unreleased
+ * Markdown rendering is another ~8x faster on large documents:
+ a custom Text-node renderer emits unescaped spans in bulk instead of
+ commonmark's per-character escape-check-and-append (9MB render
+ 42ms -> 4ms; byte-identical output on a 1,900-document differential
+ across the test corpus and real-world HTML) (TIKA-4868).
+
* ZipParser no longer re-decompresses an entry on every rewind when the
entry uses a legacy compression method (implode, shrink, bzip2, ...):
those decode per-bit, so the rewind-by-reopen trade that works for
diff --git a/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java b/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
new file mode 100644
index 0000000000..ad68db7261
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.sax;
+
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.commonmark.ext.gfm.tables.TableCell;
+import org.commonmark.node.Heading;
+import org.commonmark.node.Link;
+import org.commonmark.node.Node;
+import org.commonmark.node.Text;
+import org.commonmark.renderer.NodeRenderer;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererFactory;
+import org.commonmark.renderer.markdown.MarkdownWriter;
+import org.commonmark.text.AsciiMatcher;
+
+/**
+ * Replacement for commonmark's {@code Text} node rendering that emits unescaped spans in
+ * bulk instead of one character at a time. commonmark's {@code MarkdownWriter} routes every
+ * character of a text literal through an escape check and a per-char {@code Appendable}
+ * append; on multi-megabyte documents that is most of the render cost. This renderer scans
+ * for the next escape-needing character and writes the clean span between them with one
+ * {@code raw()} call.
+ * <p>
+ * The escaping semantics replicate {@code CoreMarkdownNodeRenderer#visit(Text)} for
+ * commonmark 0.27.x exactly: the line-start disambiguation cases, the heading escape
+ * variant, the {@code !}-before-link case, and the {@code \n} numeric reference. The
+ * differential test in {@code ToMarkdownContentHandlerTest} and the corpus differential
+ * guard the equivalence.
+ * <p>
+ * Text inside a table cell renders through the stock per-char path: the tables extension
+ * pushes a raw-escape for {@code |} onto the writer there, and hand-emitted escapes would
+ * be escaped a second time by {@code raw()}. That is the only push site in the extensions
+ * this handler registers, so outside a cell {@code raw()} is a verbatim bulk append.
+ */
+class FastMarkdownTextRenderer implements NodeRenderer {
+
+ static final MarkdownNodeRendererFactory FACTORY = new MarkdownNodeRendererFactory() {
+ @Override
+ public NodeRenderer create(MarkdownNodeRendererContext context) {
+ return new FastMarkdownTextRenderer(context);
+ }
+
+ @Override
+ public Set<Character> getSpecialCharacters() {
+ return Set.of();
+ }
+ };
+
+ // mirrors CoreMarkdownNodeRenderer's orderedListMarkerPattern
+ private static final Pattern ORDERED_LIST_MARKER = Pattern.compile("^([0-9]{1,9})([.)])");
+
+ private static final String TEXT_ESCAPE_CHARS = "[]<>`*_&\n\\";
+
+ private final MarkdownNodeRendererContext context;
+ // AsciiMatcher semantics: characters >= 128 never escape
+ private final boolean[] textEscape = new boolean[128];
+ private final boolean[] headingEscape = new boolean[128];
+ private final AsciiMatcher textEscapeMatcher;
+ private final AsciiMatcher headingEscapeMatcher;
+
+ FastMarkdownTextRenderer(MarkdownNodeRendererContext context) {
+ this.context = context;
+ textEscapeMatcher = AsciiMatcher.builder()
+ .anyOf(TEXT_ESCAPE_CHARS)
+ .anyOf(context.getSpecialCharacters())
+ .build();
+ headingEscapeMatcher = AsciiMatcher.builder(textEscapeMatcher).anyOf("#").build();
+ for (int c = 0; c < 128; c++) {
+ textEscape[c] = textEscapeMatcher.matches((char) c);
+ headingEscape[c] = headingEscapeMatcher.matches((char) c);
+ }
+ }
+
+ @Override
+ public Set<Class<? extends Node>> getNodeTypes() {
+ return Set.of(Text.class);
+ }
+
+ @Override
+ public void render(Node node) {
+ Text text = (Text) node;
+ MarkdownWriter writer = context.getWriter();
+ String literal = text.getLiteral();
+
+ // line-start disambiguation, identical to CoreMarkdownNodeRenderer
+ if (writer.isAtLineStart() && !literal.isEmpty()) {
+ char c = literal.charAt(0);
+ switch (c) {
+ case '-': {
+ writer.raw("\\-");
+ literal = literal.substring(1);
+ break;
+ }
+ case '#': {
+ writer.raw("\\#");
+ literal = literal.substring(1);
+ break;
+ }
+ case '=': {
+ if (text.getPrevious() != null) {
+ writer.raw("\\=");
+ literal = literal.substring(1);
+ }
+ break;
+ }
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': {
+ Matcher m = ORDERED_LIST_MARKER.matcher(literal);
+ if (m.find()) {
+ writer.raw(m.group(1));
+ writer.raw("\\" + m.group(2));
+ literal = literal.substring(m.end());
+ }
+ break;
+ }
+ case '\t': {
+ writer.raw("	");
+ literal = literal.substring(1);
+ break;
+ }
+ case ' ': {
+ writer.raw(" ");
+ literal = literal.substring(1);
+ break;
+ }
+ default:
+ break;
+ }
+ }
+
+ boolean inHeading = text.getParent() instanceof Heading;
+ boolean bangBeforeLink = literal.endsWith("!") && text.getNext() instanceof Link;
+ String body = bangBeforeLink ? literal.substring(0, literal.length() - 1) : literal;
+
+ if (inTableCell(text)) {
+ // stock path: the writer carries a raw-escape here, so hand-emitted
+ // escapes would be double-escaped by raw()
+ writer.text(body, inHeading ? headingEscapeMatcher : textEscapeMatcher);
+ } else {
+ writeSpanned(writer, body, inHeading ? headingEscape : textEscape);
+ }
+ if (bangBeforeLink) {
+ writer.raw("\\!");
+ }
+ }
+
+ private static boolean inTableCell(Node node) {
+ for (Node p = node.getParent(); p != null; p = p.getParent()) {
+ if (p instanceof TableCell) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static void writeSpanned(MarkdownWriter writer, String s, boolean[] escape) {
+ if (s.isEmpty()) {
+ // writer.text() is a no-op on empty input; match it
+ return;
+ }
+ int n = s.length();
+ int start = 0;
+ for (int i = 0; i < n; i++) {
+ char c = s.charAt(i);
+ if (c < 128 && escape[c]) {
+ if (i > start) {
+ writer.raw(s.substring(start, i));
+ }
+ // same replacements append(char, matcher) makes
+ writer.raw(c == '\n' ? " " : "\\" + c);
+ start = i + 1;
+ }
+ }
+ if (start < n) {
+ writer.raw(start == 0 ? s : s.substring(start));
+ }
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
index 0c28ef40c2..a6614bffae 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
@@ -89,7 +89,9 @@ public class ToMarkdownContentHandler extends DefaultHandler {
private final Writer writer;
private final MarkdownRenderer renderer =
- MarkdownRenderer.builder().extensions(EXTENSIONS).build();
+ MarkdownRenderer.builder().extensions(EXTENSIONS)
+ // registered first, so it wins Text rendering from the core renderer
+ .nodeRendererFactory(FastMarkdownTextRenderer.FACTORY).build();
private final Document document = new Document();
private final Deque<Node> stack = new ArrayDeque<>();