You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2026/09/21 16:51:22 UTC
(stormcrawler) branch main updated: #2097 Tika ParserBolt: limit the extracted text with parser.tika.text.maxlength (#2174)
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new c746b72b #2097 Tika ParserBolt: limit the extracted text with parser.tika.text.maxlength (#2174)
c746b72b is described below
commit c746b72bc8d2cd048f6bc9ecf1bc3a4cf5f0c2a2
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Mon Sep 21 18:51:17 2026 +0200
#2097 Tika ParserBolt: limit the extracted text with parser.tika.text.maxlength (#2174)
ParserBolt built its text handler with BodyContentHandler(-1), which
removes the limit Tika applies by default, and there was no way to set
one. The whole text of a document was held as a single String next to
the original bytes until the tuple was emitted, so a large body reaching
the bolt could take a lot of the worker heap.
The new key parser.tika.text.maxlength sets the maximum number of
characters of text kept per document. When the limit is reached Tika
stops the parse with a WriteLimitReachedException; the bolt now keeps
the text and outlinks extracted up to that point, sets
parse.text.trimmed to true in the metadata, counts it under
text_trimmed and emits the document instead of failing it with a parse
error. The default stays -1 so the behaviour does not change unless the
key is set; any negative value means no limit, since Tika itself only
accepts exactly -1 and fails every document on other negative values.
The parse timeout described in the issue is left for a follow-up.
---
docs/src/main/asciidoc/configuration.adoc | 1 +
external/tika/README.md | 2 +
.../org/apache/stormcrawler/tika/ParserBolt.java | 38 +++++-
.../stormcrawler/tika/ParserBoltTextLimitTest.java | 129 +++++++++++++++++++++
4 files changed, 167 insertions(+), 3 deletions(-)
diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index 19df7288..ba4cd55c 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -561,6 +561,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/tika[tika
| parser.mimetype.whitelist | - | Regex list of allowed MIME types. If set, only matching types are parsed by the Tika ParserBolt. The pattern is matched against the byte-detected MIME type, not the server-declared HTTP `Content-Type` header.
| parser.tika.config.file | tika-config.json | Name of the classpath resource holding the Tika configuration (JSON format since Tika 4).
| parser.extract.embedded | false | Whether to parse embedded documents. Since Tika 4 embedded documents are no longer parsed unless this is set to `true`.
+| parser.tika.text.maxlength | -1 | Maximum number of characters of text extracted from a document, -1 (or any negative value) for no limit. When the limit is reached the parse stops, the text and outlinks extracted so far are kept and the metadata `parse.text.trimmed` is set to `true`.
|===
NOTE: When using the Tika `ParserBolt` alongside `JSoupParserBolt`, set `jsoup.treat.non.html.as.error` to `false` so that non-HTML content is passed through to the Tika parser rather than being treated as an error.
diff --git a/external/tika/README.md b/external/tika/README.md
index 2f034434..b04ae1dd 100644
--- a/external/tika/README.md
+++ b/external/tika/README.md
@@ -37,4 +37,6 @@ Note that a configuration which is present on the classpath but invalid is treat
Embedded documents are only parsed when `parser.extract.embedded` is set to `true` (default `false`).
+The length of the text extracted from a document can be limited with `parser.tika.text.maxlength` (number of characters, default `-1`, any negative value means no limit). When the limit is reached the parse stops, the text and outlinks extracted so far are kept and the document is emitted with the metadata `parse.text.trimmed` set to `true`.
+
Since Tika 4, Tika metadata keys use namespaced names, which surface as renamed `parse.*` keys, e.g. `parse.resourceName` is now `parse.tk:resource-name`.
diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
index e90f4263..5c96aeb0 100644
--- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
@@ -63,6 +63,7 @@ import org.apache.stormcrawler.util.URLUtil;
import org.apache.tika.Tika;
import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.WriteLimitReachedException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.EmptyParser;
@@ -83,6 +84,18 @@ import org.xml.sax.ContentHandler;
/** Uses Tika to parse the output of a fetch and extract text + metadata. */
public class ParserBolt extends BaseRichBolt {
+ /**
+ * Configuration key for the maximum number of characters of text extracted from a document, a
+ * negative value for no limit.
+ */
+ public static final String TEXT_MAX_LENGTH_PARAM = "parser.tika.text.maxlength";
+
+ /**
+ * Metadata key set to "true" when the extracted text has been cut at {@link
+ * #TEXT_MAX_LENGTH_PARAM}.
+ */
+ public static final String TEXT_TRIMMED_KEY = "parse.text.trimmed";
+
private Tika tika;
/** ParseContext configured from the "parse-context" section of the Tika configuration. */
@@ -110,6 +123,8 @@ public class ParserBolt extends BaseRichBolt {
private String protocolMDprefix;
+ private int textMaxLength = -1;
+
@Override
public void prepare(
@NotNull Map<String, Object> conf,
@@ -151,6 +166,10 @@ public class ParserBolt extends BaseRichBolt {
protocolMDprefix = ConfUtils.getString(conf, ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "");
+ // Tika only treats exactly -1 as unlimited and fails on other negative values
+ int maxLength = ConfUtils.getInt(conf, TEXT_MAX_LENGTH_PARAM, -1);
+ textMaxLength = maxLength < 0 ? -1 : maxLength;
+
tika = instantiateTika(conf);
this.collector = collector;
@@ -258,7 +277,7 @@ public class ParserBolt extends BaseRichBolt {
}
LinkContentHandler linkHandler = new LinkContentHandler();
- ContentHandler textHandler = new BodyContentHandler(-1);
+ ContentHandler textHandler = new BodyContentHandler(textMaxLength);
TeeContentHandler teeHandler = new TeeContentHandler(linkHandler, textHandler);
// seed the context with the components configured in the
// "parse-context" section of the Tika configuration
@@ -293,12 +312,21 @@ public class ParserBolt extends BaseRichBolt {
// parse
String text;
+ boolean textTrimmed = false;
try (TikaInputStream tis = TikaInputStream.get(content)) {
tika.getParser().parse(tis, teeHandler, md, parseContext);
text = textHandler.toString();
} catch (Throwable e) {
- handleException(url, e, metadata, tuple, "parse error");
- return;
+ if (!WriteLimitReachedException.isWriteLimitReached(e)) {
+ handleException(url, e, metadata, tuple, "parse error");
+ return;
+ }
+ // the parse stopped at the text limit: keep the text and the
+ // links extracted so far instead of failing the document
+ text = textHandler.toString();
+ textTrimmed = true;
+ LOG.info("Text of {} trimmed to {} characters", url, textMaxLength);
+ eventCounter.scope("text_trimmed").incrBy(1);
}
// add parse md to metadata
@@ -307,6 +335,10 @@ public class ParserBolt extends BaseRichBolt {
metadata.setValues("parse." + k, values);
}
+ if (textTrimmed) {
+ metadata.setValue(TEXT_TRIMMED_KEY, "true");
+ }
+
long duration = System.currentTimeMillis() - start;
LOG.info("Parsed {} in {} msec", url, duration);
diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTextLimitTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTextLimitTest.java
new file mode 100644
index 00000000..3fc2c168
--- /dev/null
+++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltTextLimitTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.tika;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.storm.task.OutputCollector;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestUtil;
+import org.apache.stormcrawler.parse.ParsingTester;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Checks that the text extracted by the ParserBolt is limited by parser.tika.text.maxlength.
+ *
+ * @see <a href="https://github.com/apache/stormcrawler/issues/2097">#2097</a>
+ */
+class ParserBoltTextLimitTest extends ParsingTester {
+
+ private static final byte[] CONTENT = "word ".repeat(10_000).getBytes(StandardCharsets.UTF_8);
+
+ @BeforeEach
+ void setupParserBolt() {
+ bolt = new ParserBolt();
+ setupParserBolt(bolt);
+ }
+
+ private void prepare(Integer maxLength) {
+ Map<String, Object> conf = new HashMap<>();
+ if (maxLength != null) {
+ conf.put(ParserBolt.TEXT_MAX_LENGTH_PARAM, maxLength);
+ }
+ bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+ }
+
+ @Test
+ void textIsTrimmedAtLimit() throws IOException {
+ prepare(100);
+ parse("https://example.org/big.txt", CONTENT, new Metadata());
+
+ Assertions.assertTrue(output.getEmitted(Constants.StatusStreamName).isEmpty());
+ List<List<Object>> emitted = output.getEmitted();
+ Assertions.assertEquals(1, emitted.size());
+ String text = emitted.get(0).get(3).toString();
+ Assertions.assertEquals(100, text.length());
+ Assertions.assertTrue(text.startsWith("word word"));
+ Metadata metadata = (Metadata) emitted.get(0).get(2);
+ Assertions.assertEquals("true", metadata.getFirstValue(ParserBolt.TEXT_TRIMMED_KEY));
+ }
+
+ @Test
+ void linksBeforeLimitAreKept() throws IOException {
+ prepare(100);
+ String html =
+ "<html><body><a href=\"https://example.org/first\">first</a><p>"
+ + "word ".repeat(10_000)
+ + "</p></body></html>";
+ parse(
+ "https://example.org/big.html",
+ html.getBytes(StandardCharsets.UTF_8),
+ new Metadata());
+
+ List<List<Object>> emitted = output.getEmitted();
+ Assertions.assertEquals(1, emitted.size());
+ Metadata metadata = (Metadata) emitted.get(0).get(2);
+ Assertions.assertEquals("true", metadata.getFirstValue(ParserBolt.TEXT_TRIMMED_KEY));
+ List<List<Object>> status = output.getEmitted(Constants.StatusStreamName);
+ Assertions.assertEquals(1, status.size());
+ Assertions.assertEquals("https://example.org/first", status.get(0).get(0));
+ }
+
+ @Test
+ void noLimitByDefault() throws IOException {
+ prepare(null);
+ parse("https://example.org/big.txt", CONTENT, new Metadata());
+
+ List<List<Object>> emitted = output.getEmitted();
+ Assertions.assertEquals(1, emitted.size());
+ Assertions.assertTrue(emitted.get(0).get(3).toString().length() >= CONTENT.length - 1);
+ Metadata metadata = (Metadata) emitted.get(0).get(2);
+ Assertions.assertNull(metadata.getFirstValue(ParserBolt.TEXT_TRIMMED_KEY));
+ }
+
+ @Test
+ void noLimitWithMinusOne() throws IOException {
+ prepare(-1);
+ parse("https://example.org/big.txt", CONTENT, new Metadata());
+
+ List<List<Object>> emitted = output.getEmitted();
+ Assertions.assertEquals(1, emitted.size());
+ Assertions.assertTrue(emitted.get(0).get(3).toString().length() >= CONTENT.length - 1);
+ Metadata metadata = (Metadata) emitted.get(0).get(2);
+ Assertions.assertNull(metadata.getFirstValue(ParserBolt.TEXT_TRIMMED_KEY));
+ }
+
+ @Test
+ void noLimitWithOtherNegativeValue() throws IOException {
+ prepare(-2);
+ parse("https://example.org/big.txt", CONTENT, new Metadata());
+
+ Assertions.assertTrue(output.getEmitted(Constants.StatusStreamName).isEmpty());
+ List<List<Object>> emitted = output.getEmitted();
+ Assertions.assertEquals(1, emitted.size());
+ Assertions.assertTrue(emitted.get(0).get(3).toString().length() >= CONTENT.length - 1);
+ Metadata metadata = (Metadata) emitted.get(0).get(2);
+ Assertions.assertNull(metadata.getFirstValue(ParserBolt.TEXT_TRIMMED_KEY));
+ }
+}