You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/09/03 09:37:07 UTC
(tika) branch TIKA-4875 created (now 7d02b1f1a2)
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a change to branch TIKA-4875
in repository https://gitbox.apache.org/repos/asf/tika.git
at 7d02b1f1a2 TIKA-4875: improve tika-eval performance
This branch includes the following new commits:
new 7d02b1f1a2 TIKA-4875: improve tika-eval performance
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tika) 01/01: TIKA-4875: improve tika-eval performance
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4875
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 7d02b1f1a268a8c619241c4788c2147b02be3883
Author: tallison <ta...@apache.org>
AuthorDate: Thu Sep 3 05:36:38 2026 -0400
TIKA-4875: improve tika-eval performance
---
CHANGES.txt | 7 +
.../tika/eval/app/ExtractComparerRunner.java | 4 +-
.../apache/tika/eval/app/ExtractProfileRunner.java | 4 +-
.../org/apache/tika/eval/app/StatusReporter.java | 18 ++-
.../charsoup/core/CharSoupFeatureExtractor.java | 151 ++++++++++++++++++---
.../core/CharSoupFeatureExtractorTest.java | 86 ++++++++++++
.../tika/langdetect/opennlp/OpenNLPDetector.java | 6 +-
7 files changed, 248 insertions(+), 28 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 0d7c4604cd..f37fb7e5c5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,12 @@
Release 4.1.0 - unreleased
+ * tika-eval Profile/Compare speedups: single-pass URL/mail stripping
+ replaces the bounded regexes in langdetect preprocessing (same output,
+ 17-290x faster on web text), the default H2 db URL drops MVStore chunk
+ retention and raises the page cache, and the status log adds a
+ last-interval docs-per-sec rate next to the cumulative average
+ (TIKA-4875).
+
* AVIF images are parsed rather than only detected: HeifParser accepts
image/avif, which is the same ISO-BMFF container, so dimensions, EXIF
and XMP come out of it the way they do for HEIC (TIKA-4870).
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java
index 6a2c2fdea5..191839ac64 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java
@@ -183,8 +183,8 @@ public class ExtractComparerRunner {
}
//default to h2
Path p = Paths.get(dbPath);
- return "jdbc:h2:file:" + p.toAbsolutePath();
-
+ // drop the 45s MVStore chunk retention (bloat + growing compaction cost); CACHE_SIZE is KB
+ return "jdbc:h2:file:" + p.toAbsolutePath() + ";RETENTION_TIME=0;CACHE_SIZE=1048576";
}
private static void execute(Path inputDir, Path extractsA, Path extractsB, String dbPath, EvalConfig evalConfig, PipesReport pipesReportA,
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
index daa2ddc81d..ba76f484a5 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
@@ -124,8 +124,8 @@ public class ExtractProfileRunner {
}
//default to h2
Path p = Paths.get(dbPath);
- return "jdbc:h2:file:" + p.toAbsolutePath();
-
+ // drop the 45s MVStore chunk retention (bloat + growing compaction cost); CACHE_SIZE is KB
+ return "jdbc:h2:file:" + p.toAbsolutePath() + ";RETENTION_TIME=0;CACHE_SIZE=1048576";
}
private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig, PipesReport pipesReport,
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
index 3f81f17758..484f327fc8 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
@@ -38,6 +38,8 @@ public class StatusReporter implements Callable<Long> {
private final AtomicBoolean crawlerIsActive;
private final long start;
private final NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ROOT);
+ private int lastCnt = 0;
+ private long lastReportMillis;
public StatusReporter(CallablePipesIterator pipesIterator, AtomicInteger filesProcessed, AtomicInteger activeWorkers, AtomicBoolean crawlerIsActive) {
@@ -46,6 +48,7 @@ public class StatusReporter implements Callable<Long> {
this.activeWorkers = activeWorkers;
this.crawlerIsActive = crawlerIsActive;
this.start = System.currentTimeMillis();
+ this.lastReportMillis = this.start;
}
@Override
@@ -68,13 +71,22 @@ public class StatusReporter implements Callable<Long> {
}
private void report() {
+ long now = System.currentTimeMillis();
int cnt = filesProcessed.get();
- long elapsed = System.currentTimeMillis() - start;
+ long elapsed = now - start;
double elapsedSecs = (double) elapsed / (double) 1000;
int avg = (elapsedSecs > 5 || cnt > 100) ? (int) ((double) cnt / elapsedSecs) : -1;
- String elapsedString = DurationFormatUtils.formatMillis(System.currentTimeMillis() - start);
- String docsPerSec = avg > -1 ? String.format(Locale.ROOT, " (%s docs per sec)", numberFormat.format(avg)) : "";
+ // the cumulative average declines by construction and masks cliffs
+ double windowSecs = (double) (now - lastReportMillis) / (double) 1000;
+ int windowRate = windowSecs > 0 ? (int) ((double) (cnt - lastCnt) / windowSecs) : avg;
+ lastCnt = cnt;
+ lastReportMillis = now;
+
+ String elapsedString = DurationFormatUtils.formatMillis(elapsed);
+ String docsPerSec = avg > -1 ?
+ String.format(Locale.ROOT, " (%s docs per sec overall; %s in the last interval)",
+ numberFormat.format(avg), numberFormat.format(windowRate)) : "";
String msg = String.format(Locale.ROOT, "Processed %s documents in %s%s.", numberFormat.format(cnt), elapsedString, docsPerSec);
LOGGER.info(msg);
diff --git a/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java b/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java
index faae8cb85b..dd02bf273d 100644
--- a/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java
+++ b/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java
@@ -17,7 +17,6 @@
package org.apache.tika.langdetect.charsoup.core;
import java.text.Normalizer;
-import java.util.regex.Pattern;
/**
* Extracts character n-gram features from text using the hashing trick (FNV-1a).
@@ -36,7 +35,7 @@ import java.util.regex.Pattern;
* <h3>Pipeline</h3>
* <ol>
* <li>Truncate input at {@link #MAX_TEXT_LENGTH} chars</li>
- * <li>Strip URLs and emails (TIKA-2777 bounded patterns)</li>
+ * <li>Strip URLs and emails</li>
* <li>NFC normalize</li>
* <li>Iterate codepoints (surrogate-safe)</li>
* <li>Skip transparent characters (see {@link #isTransparent(int)})</li>
@@ -73,11 +72,26 @@ public class CharSoupFeatureExtractor {
/** Underscore sentinel codepoint used for word boundary bigrams. */
static final int SENTINEL = '_';
- // TIKA-2777: bounded regexes to avoid catastrophic backtracking
- private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}");
- private static final Pattern MAIL_REGEX =
- Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}");
+ // char classes for the URL/mail scanners below (TIKA-4875)
+ private static final boolean[] URL_CHARS = new boolean[128];
+ private static final boolean[] MAIL_CHARS = new boolean[128]; // domain head: no '.'
+ private static final boolean[] MAIL_DOT_CHARS = new boolean[128]; // local part / domain tail
+
+ static {
+ for (int c = '0'; c <= '9'; c++) {
+ URL_CHARS[c] = MAIL_CHARS[c] = MAIL_DOT_CHARS[c] = true;
+ }
+ for (int c = 'A'; c <= 'Z'; c++) {
+ URL_CHARS[c] = MAIL_CHARS[c] = MAIL_DOT_CHARS[c] = true;
+ int l = c + ('a' - 'A');
+ URL_CHARS[l] = MAIL_CHARS[l] = MAIL_DOT_CHARS[l] = true;
+ }
+ for (char c : "-_.?&~;+=/#".toCharArray()) {
+ URL_CHARS[c] = true;
+ }
+ MAIL_CHARS['-'] = MAIL_CHARS['_'] = true;
+ MAIL_DOT_CHARS['-'] = MAIL_DOT_CHARS['_'] = MAIL_DOT_CHARS['.'] = true;
+ }
/** Arabic Tatweel (kashida) — a typographic stretching character (U+0640). */
private static final int TATWEEL = 0x0640;
@@ -244,17 +258,9 @@ public class CharSoupFeatureExtractor {
* @return cleaned, NFC-normalized text
*/
public static String preprocessNoTruncate(String rawText) {
- // Strip URLs and emails. Both regexes scan the entire input on every call;
- // skip each unless its required marker is present ("://" for URL_REGEX, "@"
- // for MAIL_REGEX). This is a no-op for the common (markerless) case — the
- // output is identical — but avoids a full-buffer regex scan + Matcher alloc.
- String text = rawText;
- if (text.indexOf("://") >= 0) {
- text = URL_REGEX.matcher(text).replaceAll(" ");
- }
- if (text.indexOf('@') >= 0) {
- text = MAIL_REGEX.matcher(text).replaceAll(" ");
- }
+ // order matters: the URL replacement is a barrier ("http://aaaaaaaaaa@bb" is not a mail match)
+ String text = stripUrls(rawText);
+ text = stripEmails(text);
// NFC normalize
if (!Normalizer.isNormalized(text, Normalizer.Form.NFC)) {
@@ -264,6 +270,115 @@ public class CharSoupFeatureExtractor {
return text;
}
+ /**
+ * Equivalent to {@code "https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}"} replaced with " ".
+ * Matches cannot overlap (':' is outside the char class), so consuming them left to
+ * right reproduces the regex's leftmost order.
+ */
+ private static String stripUrls(String text) {
+ int n = text.length();
+ StringBuilder sb = null;
+ int emitted = 0;
+ int i = 0;
+ while (i < n) {
+ if (text.charAt(i) == 'h') {
+ int afterScheme = -1;
+ if (text.startsWith("http://", i)) {
+ afterScheme = i + 7;
+ } else if (text.startsWith("https://", i)) {
+ afterScheme = i + 8;
+ }
+ if (afterScheme > 0) {
+ int max = Math.min(n, afterScheme + 10000);
+ int k = afterScheme;
+ while (k < max && text.charAt(k) < 128 && URL_CHARS[text.charAt(k)]) {
+ k++;
+ }
+ if (k - afterScheme >= 10) {
+ if (sb == null) {
+ sb = new StringBuilder(n);
+ }
+ sb.append(text, emitted, i).append(' ');
+ emitted = k;
+ i = k;
+ continue;
+ }
+ }
+ }
+ i++;
+ }
+ if (sb == null) {
+ return text;
+ }
+ return sb.append(text, emitted, n).toString();
+ }
+
+ /**
+ * Equivalent to
+ * {@code "[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}"} replaced with " ".
+ * Rebuilt around each '@': the local part scans back (max 100, not past the previous
+ * match, where find() resumes); ahead, the dotless head runs greedily (max 100) and
+ * yields exactly one char to the tail when the tail cannot otherwise start -- the
+ * only backtrack the regex can take ("a@bb" matches, "a@b" does not).
+ * No match spans a second '@', so left to right is leftmost order.
+ */
+ private static String stripEmails(String text) {
+ int n = text.length();
+ StringBuilder sb = null;
+ int emitted = 0;
+ int floor = 0;
+ int i = 0;
+ while (i < n) {
+ if (text.charAt(i) == '@') {
+ int s = i;
+ while (s > floor && i - s < 100) {
+ char p = text.charAt(s - 1);
+ if (p < 128 && MAIL_DOT_CHARS[p]) {
+ s--;
+ } else {
+ break;
+ }
+ }
+ if (s < i) {
+ int m = i + 1;
+ int cap2 = Math.min(n, m + 100);
+ int j = m;
+ while (j < cap2 && text.charAt(j) < 128 && MAIL_CHARS[text.charAt(j)]) {
+ j++;
+ }
+ int end = -1;
+ if (j > m) {
+ if (j < n && text.charAt(j) < 128 && MAIL_DOT_CHARS[text.charAt(j)]) {
+ int cap3 = Math.min(n, j + 100);
+ int k = j;
+ while (k < cap3 && text.charAt(k) < 128 && MAIL_DOT_CHARS[text.charAt(k)]) {
+ k++;
+ }
+ end = k;
+ } else if (j - m >= 2) {
+ end = j;
+ }
+ }
+ if (end >= 0) {
+ if (sb == null) {
+ sb = new StringBuilder(n);
+ }
+ sb.append(text, emitted, s).append(' ');
+ emitted = end;
+ floor = end;
+ i = end;
+ continue;
+ }
+ }
+ }
+ i++;
+ }
+ if (sb == null) {
+ return text;
+ }
+ return sb.append(text, emitted, n).toString();
+ }
+
/**
* Determine whether a codepoint should be treated as transparent (skipped)
* during bigram extraction and word tokenization.
diff --git a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java b/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java
index 9a557b8e86..e00570c956 100644
--- a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java
+++ b/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java
@@ -21,6 +21,11 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.regex.Pattern;
+
import org.junit.jupiter.api.Test;
@@ -93,6 +98,87 @@ public class CharSoupFeatureExtractorTest {
assertArrayEquals(countsC, countsD);
}
+ @Test
+ public void testUrlMailStrippingMatchesGreedyRegexReference() {
+ // TIKA-4875: the scanners must stay byte-identical to the regexes they replaced --
+ // the langdetect and junkdetect models were trained on this exact preprocessing.
+ // Inputs are NFC-stable, so preprocessNoTruncate's NFC step is an identity here.
+ Pattern greedyUrl = Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}");
+ Pattern greedyMail = Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}");
+
+ List<String> cases = new ArrayList<>();
+ // middle mail repeat must backtrack to feed the dot-class tail
+ cases.add("a@bb");
+ cases.add("a@b");
+ cases.add("a@.b");
+ cases.add("a@b..c");
+ cases.add("a@" + "b".repeat(250));
+ cases.add("a".repeat(150) + "@x.y");
+ cases.add("a".repeat(200) + "@");
+ cases.add("local@local@local");
+ cases.add("x@y.z a@bb user@example.com");
+ cases.add("@@@@@@");
+ cases.add("a@a@a@a@a@a@");
+ // mail caps: head {1,100} then tail {1,100} over one long run
+ cases.add("a@" + "b".repeat(100) + "." + "c".repeat(150));
+ cases.add("a@" + "b".repeat(99) + "." + "c".repeat(99));
+ cases.add("a@" + "b".repeat(300));
+ // find() resumes after a match: leftover run chars are not a fresh local part
+ cases.add("aa@bb cc@dd ee@ff");
+ cases.add("a@bb@cc@dd");
+ // URL length boundaries: min 10 after scheme, cap 10000
+ cases.add("http://" + "a".repeat(9));
+ cases.add("http://" + "a".repeat(10));
+ cases.add("https://" + "a".repeat(10000));
+ cases.add("https://" + "a".repeat(10001));
+ cases.add("https://" + "a".repeat(10005) + "@bb");
+ cases.add("http://http://aaaaaaaaaa");
+ cases.add("http://aaaahttp://bbbbbbbbbb");
+ cases.add("hhttp://aaaaaaaaaaa");
+ cases.add("http:/notaurl http//nope https:/x");
+ // the URL pass runs first; its replacement is a barrier for the mail pass
+ cases.add("http://aaaaaaaaaa@bb");
+ cases.add("a@http://aaaaaaaaaa");
+ cases.add("a@httpx.y");
+ cases.add("user@example.com/http://foobarbazqux");
+ // non-ASCII neighbors exercise the < 128 guards
+ cases.add("é@bb");
+ cases.add("aé@bb");
+ cases.add("a@büc.d");
+ cases.add("http://aéaaaaaaaaaa");
+ cases.add("see http://example.com/a/b?q=1#f and mail bob.smith@sub-domain_x.example.org.");
+
+ long seed = new Random().nextLong();
+ Random random = new Random(seed);
+ String[] atoms = {"a", "B", "9", ".", "-", "_", "@", ":", "/", "#", "?", "=", " ",
+ "http://", "https://", "http", "://", "@a.", "é", "aaaaaaaaaa"};
+ for (int i = 0; i < 5000; i++) {
+ int len = 1 + random.nextInt(60);
+ StringBuilder sb = new StringBuilder();
+ for (int j = 0; j < len; j++) {
+ sb.append(atoms[random.nextInt(atoms.length)]);
+ }
+ cases.add(sb.toString());
+ }
+ // long-run shapes that hit the 100/10000 caps
+ for (int i = 0; i < 50; i++) {
+ StringBuilder sb = new StringBuilder();
+ while (sb.length() < 3000) {
+ sb.append("a".repeat(1 + random.nextInt(400)));
+ sb.append(atoms[random.nextInt(atoms.length)]);
+ }
+ cases.add(sb.toString());
+ }
+
+ for (String text : cases) {
+ String expected = greedyMail
+ .matcher(greedyUrl.matcher(text).replaceAll(" "))
+ .replaceAll(" ");
+ String actual = CharSoupFeatureExtractor.preprocessNoTruncate(text);
+ assertEquals(expected, actual, "seed=" + seed + " input=" + text);
+ }
+ }
+
@Test
public void testURLStripping() {
CharSoupFeatureExtractor ext = new CharSoupFeatureExtractor(NUM_BUCKETS);
diff --git a/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java b/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java
index 243976104c..5bae71b4d4 100644
--- a/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java
+++ b/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java
@@ -220,11 +220,11 @@ public class OpenNLPDetector extends LanguageDetector {
private static class TikaUrlCharSequenceNormalizer implements CharSequenceNormalizer {
//use this custom copy/paste of opennlp to avoid long, long hang with mail_regex
- //TIKA-2777
+ //TIKA-2777. TIKA-4875: possessive where match-equivalent; the middle repeat must stay greedy ("a@bb")
private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}");
+ Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}+");
private static final Pattern MAIL_REGEX =
- Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}");
+ Pattern.compile("[-_.0-9A-Za-z]{1,100}+@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}+");
private static final TikaUrlCharSequenceNormalizer INSTANCE =
new TikaUrlCharSequenceNormalizer();