You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/09/02 20:04:31 UTC
(tika) 01/04: TIKA-4871: ocr/inference bug fixes
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch TIKA-4872
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6cc2ca70ba236ac180e3c51da74e25972d1665b9
Author: tallison <ta...@apache.org>
AuthorDate: Wed Sep 2 10:39:08 2026 -0400
TIKA-4871: ocr/inference bug fixes
---
CHANGES.txt | 11 ++++
.../ROOT/pages/configuration/metadata-filters.adoc | 5 ++
.../pages/configuration/parsers/vlm-parsers.adoc | 12 ++--
.../tika/inference/AbstractEmbeddingFilter.java | 68 +++++++++++++++++-----
.../apache/tika/inference/JinaEmbeddingFilter.java | 5 ++
.../tika/inference/OpenAIEmbeddingFilter.java | 12 ++++
.../tika/inference/OpenAIEmbeddingFilterTest.java | 57 ++++++++++++++++++
.../apache/tika/parser/vlm/OpenAIVLMParser.java | 3 +-
.../tika/parser/vlm/OpenAIVLMParserTest.java | 17 ++++++
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 31 +++++++++-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 52 +++++++++++++++++
11 files changed, 248 insertions(+), 25 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 80e898aa72..b632080e80 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,16 @@
Release 4.1.0 - unreleased
+ * Inference/OCR hardening (TIKA-4871): OpenAIVLMParser no longer
+ auto-registers via SPI, matching its Claude/Gemini siblings; select
+ it by name ("openai-vlm-parser") in config. Per-request parse-context
+ config for the embedding filters now works and is validated:
+ {"openai-embedding-filter": {"skipEmbedding": true}} (likewise
+ "jina-embedding-filter") merges over the server config, and
+ baseUrl/apiKey/model may not be changed at runtime. The embedding
+ filters release their HTTP client resources on close(). Inline PDF
+ page OCR now accumulates tk:chunks from every OCR'd page onto the
+ parent document instead of keeping only the first page's.
+
* Raster previews for the vector thumbnails of Office documents: the new
poi-metafile-renderer draws EMF and WMF images through POI (a PNG of
a configurable width; Word's bitmap-in-WMF thumbnails from the bitmap
diff --git a/docs/modules/ROOT/pages/configuration/metadata-filters.adoc b/docs/modules/ROOT/pages/configuration/metadata-filters.adoc
index c7b1eeae30..262fee13a9 100644
--- a/docs/modules/ROOT/pages/configuration/metadata-filters.adoc
+++ b/docs/modules/ROOT/pages/configuration/metadata-filters.adoc
@@ -233,6 +233,11 @@ around.
`parse-context` can carry a per-request `metadata-filters` list, which *replaces* the configured
one for that request rather than adding to it.
+Since 4.1.0, the embedding filters also accept per-request config under their own name --
+e.g. `{"parse-context": {"openai-embedding-filter": {"skipEmbedding": true}}}` (likewise
+`jina-embedding-filter`) -- which is merged over the server-side config for that request.
+Endpoint fields (`baseUrl`, `apiKey`, `model`) cannot be changed per request.
+
NOTE: In `CONTENT_ONLY` xref:pipes/parse-modes.adoc[parse mode], Tika applies an
`include-field-metadata-filter` for `tk:content` and `tk:exception:container-exception` when you have
configured no filter of your own. Only a filter that reaches the `parse-context` replaces it —
diff --git a/docs/modules/ROOT/pages/configuration/parsers/vlm-parsers.adoc b/docs/modules/ROOT/pages/configuration/parsers/vlm-parsers.adoc
index 895c162529..8c68b7e7d4 100644
--- a/docs/modules/ROOT/pages/configuration/parsers/vlm-parsers.adoc
+++ b/docs/modules/ROOT/pages/configuration/parsers/vlm-parsers.adoc
@@ -22,27 +22,25 @@ to remote Vision-Language Model (VLM) endpoints. These parsers send images
(or PDFs) to an external API and convert the model's markdown response into
structured XHTML.
-Three implementations are provided out of the box. Only `openai-vlm-parser` is auto-loaded via SPI;
-the other two must be named explicitly in your configuration.
+Three implementations are provided out of the box. None is auto-loaded: each must be
+named explicitly in your configuration. (Changed in 4.1.0: `openai-vlm-parser` previously
+auto-registered via SPI.)
-[cols="1,2,1,1"]
+[cols="1,2,1"]
|===
-|Parser |Endpoint |Config key |Auto-loaded
+|Parser |Endpoint |Config key
|`OpenAIVLMParser`
|Any OpenAI-compatible chat completions endpoint (vLLM, Ollama, local FastAPI, OpenAI)
|`openai-vlm-parser`
-|Yes
|`ClaudeVLMParser`
|Anthropic Messages API
|`claude-vlm-parser`
-|No
|`GeminiVLMParser`
|Google Gemini `generateContent` API
|`gemini-vlm-parser`
-|No
|===
All three handle the standard OCR image types (`image/ocr-png`, `image/ocr-jpeg`, ...).
diff --git a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/AbstractEmbeddingFilter.java b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/AbstractEmbeddingFilter.java
index 52cf9187a1..b134dd0da8 100644
--- a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/AbstractEmbeddingFilter.java
+++ b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/AbstractEmbeddingFilter.java
@@ -23,6 +23,8 @@ import java.util.Locale;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.tika.config.ParseContextConfig;
+import org.apache.tika.config.SelfConfiguring;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
@@ -54,7 +56,7 @@ import org.apache.tika.parser.ParseContext;
* fully constructed. Setters must not be called concurrently with
* {@link #filter}.
*/
-public abstract class AbstractEmbeddingFilter extends MetadataFilter {
+public abstract class AbstractEmbeddingFilter extends MetadataFilter implements SelfConfiguring {
private static final long serialVersionUID = 1L;
@@ -88,24 +90,60 @@ public abstract class AbstractEmbeddingFilter extends MetadataFilter {
protected abstract void embed(List<Chunk> chunks, InferenceConfig config, ParseContext parseContext)
throws IOException, TikaException;
+ /**
+ * The {@code @TikaComponent} name this filter's per-request JSON config is keyed by
+ * in parse-context (e.g. {@code {"openai-embedding-filter": {...}}}).
+ */
+ protected abstract String getComponentName();
+
@Override
protected void doFilter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException {
- InferenceConfig requestConfig = parseContext.get(InferenceConfig.class);
- if (requestConfig != null && requestConfig.isSkipEmbedding()) {
+ InferenceConfig config = resolveConfig(parseContext);
+ if (config.isSkipEmbedding()) {
return;
}
for (Metadata metadata : metadataList) {
- processOne(metadata, parseContext);
+ processOne(metadata, config, parseContext);
+ }
+ }
+
+ /**
+ * Per-request JSON config, validated through {@link InferenceConfig.RuntimeConfig}
+ * (which rejects baseUrl/apiKey/model changes) and merged over the init-time defaults.
+ * With no JSON, a class-keyed programmatic {@link InferenceConfig} is honored for
+ * skipEmbedding only, preserving the pre-4.1 contract.
+ */
+ private InferenceConfig resolveConfig(ParseContext parseContext) throws TikaException {
+ try {
+ if (ParseContextConfig.hasConfig(parseContext, getComponentName())) {
+ InferenceConfig.RuntimeConfig runtimeConfig = ParseContextConfig.getConfig(
+ parseContext, getComponentName(),
+ InferenceConfig.RuntimeConfig.class, new InferenceConfig.RuntimeConfig());
+ if (runtimeConfig.isSkipEmbedding()) {
+ return runtimeConfig;
+ }
+ return ParseContextConfig.getConfig(parseContext, getComponentName(),
+ InferenceConfig.class, defaultConfig);
+ }
+ } catch (TikaConfigException | IOException e) {
+ throw new TikaException("Failed to resolve per-request config for '"
+ + getComponentName() + "'", e);
+ }
+ InferenceConfig programmatic = parseContext.get(InferenceConfig.class);
+ if (programmatic != null && programmatic.isSkipEmbedding()) {
+ return programmatic;
}
+ return defaultConfig;
}
- private void processOne(Metadata metadata, ParseContext parseContext) throws TikaException {
- String content = metadata.get(defaultConfig.getContentField());
+ private void processOne(Metadata metadata, InferenceConfig config, ParseContext parseContext)
+ throws TikaException {
+ String content = metadata.get(config.getContentField());
if (content == null) {
LOG.debug("No content found at field '{}'; skipping embedding. "
+ "If using this filter standalone, populate metadata using "
+ "TikaCoreProperties.TIKA_CONTENT as the key.",
- defaultConfig.getContentField());
+ config.getContentField());
return;
}
if (content.isBlank()) {
@@ -124,15 +162,15 @@ public abstract class AbstractEmbeddingFilter extends MetadataFilter {
}
MarkdownChunker chunker = new MarkdownChunker(
- defaultConfig.getMaxChunkChars(),
- defaultConfig.getOverlapChars());
+ config.getMaxChunkChars(),
+ config.getOverlapChars());
List<Chunk> chunks = chunker.chunk(content);
if (chunks.isEmpty()) {
return;
}
- int maxChunks = defaultConfig.getMaxChunks();
+ int maxChunks = config.getMaxChunks();
if (maxChunks > 0 && chunks.size() > maxChunks) {
LOG.warn("Document produced {} chunks, truncating to maxChunks={}",
chunks.size(), maxChunks);
@@ -140,20 +178,20 @@ public abstract class AbstractEmbeddingFilter extends MetadataFilter {
}
try {
- int batchSize = defaultConfig.getMaxBatchSize();
+ int batchSize = config.getMaxBatchSize();
for (int i = 0; i < chunks.size(); i += batchSize) {
List<Chunk> batch = chunks.subList(
i, Math.min(i + batchSize, chunks.size()));
- embed(batch, defaultConfig, parseContext);
+ embed(batch, config, parseContext);
}
- ChunkSerializer.mergeInto(metadata, chunks, defaultConfig.getOutputField());
+ ChunkSerializer.mergeInto(metadata, chunks, config.getOutputField());
} catch (IOException e) {
throw new TikaException(
"Embedding inference failed: " + e.getMessage(), e);
}
- if (defaultConfig.isClearContentAfterChunking()) {
- metadata.remove(defaultConfig.getContentField());
+ if (config.isClearContentAfterChunking()) {
+ metadata.remove(config.getContentField());
}
}
diff --git a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/JinaEmbeddingFilter.java b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/JinaEmbeddingFilter.java
index 2cf3671d4f..2985c77498 100644
--- a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/JinaEmbeddingFilter.java
+++ b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/JinaEmbeddingFilter.java
@@ -61,6 +61,11 @@ public class JinaEmbeddingFilter extends OpenAIEmbeddingFilter {
super(config);
}
+ @Override
+ protected String getComponentName() {
+ return "jina-embedding-filter";
+ }
+
@Override
String buildRequest(List<Chunk> chunks, InferenceConfig config) {
ObjectNode root = MAPPER.createObjectNode();
diff --git a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/OpenAIEmbeddingFilter.java b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/OpenAIEmbeddingFilter.java
index d0b7e00902..54afc1712b 100644
--- a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/OpenAIEmbeddingFilter.java
+++ b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/OpenAIEmbeddingFilter.java
@@ -78,6 +78,18 @@ public class OpenAIEmbeddingFilter extends AbstractEmbeddingFilter {
this.httpClient = TikaHttpClient.build(30);
}
+ @Override
+ protected String getComponentName() {
+ return "openai-embedding-filter";
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (httpClient != null) {
+ httpClient.close();
+ }
+ }
+
@Override
protected void embed(List<Chunk> chunks, InferenceConfig config, ParseContext parseContext)
throws IOException, TikaException {
diff --git a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingFilterTest.java b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingFilterTest.java
index a9087e6b64..fcf2c2d0d1 100644
--- a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingFilterTest.java
+++ b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingFilterTest.java
@@ -35,6 +35,7 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.http.TikaTestHttpServer;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
public class OpenAIEmbeddingFilterTest {
@@ -230,6 +231,62 @@ public class OpenAIEmbeddingFilterTest {
assertNotNull(merged.get(1).getVector());
}
+ @Test
+ void testPerRequestSkipEmbedding() throws Exception {
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "# Section A\n\nSome text.");
+ List<Metadata> list = new ArrayList<>();
+ list.add(metadata);
+
+ ParseContext context = new ParseContext();
+ context.setJsonConfig("openai-embedding-filter", "{\"skipEmbedding\": true}");
+ filter.filter(list, context);
+
+ assertNull(metadata.get(TikaCoreProperties.TIKA_CHUNKS));
+ assertEquals(0, server.getRequestCount());
+ }
+
+ @Test
+ void testPerRequestBaseUrlRejected() {
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "# Section A\n\nSome text.");
+ List<Metadata> list = new ArrayList<>();
+ list.add(metadata);
+
+ ParseContext context = new ParseContext();
+ context.setJsonConfig("openai-embedding-filter",
+ "{\"baseUrl\": \"http://attacker.example.com\"}");
+ assertThrows(TikaException.class, () -> filter.filter(list, context));
+ assertEquals(0, server.getRequestCount());
+ }
+
+ @Test
+ void testPerRequestConfigMergesOverDefaults() throws Exception {
+ server.enqueue(new TikaTestHttpServer.MockResponse(200,
+ buildEmbeddingResponse(1, 3)));
+
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.TIKA_CONTENT, "# Section A\n\nSome text.");
+ List<Metadata> list = new ArrayList<>();
+ list.add(metadata);
+
+ // an allowed runtime override; baseUrl/model must survive from the init-time config
+ ParseContext context = new ParseContext();
+ context.setJsonConfig("openai-embedding-filter", "{\"maxChunkChars\": 5000}");
+ filter.filter(list, context);
+
+ assertNotNull(metadata.get(TikaCoreProperties.TIKA_CHUNKS));
+ assertEquals(1, server.getRequestCount());
+ TikaTestHttpServer.RecordedRequest request = server.takeRequest();
+ assertEquals("/v1/embeddings", request.path());
+ }
+
+ @Test
+ void testCloseReleasesClient() throws Exception {
+ filter.close();
+ filter.close();
+ }
+
/**
* Build a mock OpenAI embeddings response.
*/
diff --git a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/OpenAIVLMParser.java b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/OpenAIVLMParser.java
index bf5d824e7b..517fe5f0ab 100644
--- a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/OpenAIVLMParser.java
+++ b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/OpenAIVLMParser.java
@@ -57,7 +57,8 @@ import org.apache.tika.utils.StringUtils;
*
* @since Apache Tika 4.0
*/
-@TikaComponent(name = "openai-vlm-parser")
+// spi = false: VLM parsers are selected by name in config, never auto-registered (TIKA-4871)
+@TikaComponent(name = "openai-vlm-parser", spi = false)
public class OpenAIVLMParser extends AbstractVLMParser {
private static final long serialVersionUID = 1L;
diff --git a/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/OpenAIVLMParserTest.java b/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/OpenAIVLMParserTest.java
index 71f6d84deb..3fc7039e64 100644
--- a/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/OpenAIVLMParserTest.java
+++ b/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/OpenAIVLMParserTest.java
@@ -285,6 +285,23 @@ public class OpenAIVLMParserTest {
assertEquals(0, parser.getSupportedTypes(new ParseContext()).size());
}
+ /**
+ * No VLM parser may auto-register via SPI; VLM parsers must be selected by name
+ * in config (TIKA-4871).
+ */
+ @Test
+ void testNoVlmParserAutoRegisters() throws Exception {
+ java.util.Enumeration<java.net.URL> resources = getClass().getClassLoader()
+ .getResources("META-INF/services/org.apache.tika.parser.Parser");
+ while (resources.hasMoreElements()) {
+ java.net.URL url = resources.nextElement();
+ String content = new String(url.openStream().readAllBytes(),
+ java.nio.charset.StandardCharsets.UTF_8);
+ assertTrue(!content.contains("org.apache.tika.parser.vlm."),
+ "VLM parser auto-registered via SPI in " + url + ":\n" + content);
+ }
+ }
+
private String buildChatResponse(String content, int prompt, int completion) {
return String.format(java.util.Locale.ROOT,
"{\"choices\":[{\"message\":{\"content\":\"%s\"}}],"
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index aa57face74..fb8a725e31 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -598,8 +598,10 @@ class AbstractPDF2XHTML extends PDFTextStripper {
// from image embedding parsers) back to the parent document so it isn't
// silently discarded when the renderMetadata goes out of scope.
String renderChunks = renderMetadata.get(TikaCoreProperties.TIKA_CHUNKS);
- if (renderChunks != null && metadata.get(TikaCoreProperties.TIKA_CHUNKS) == null) {
- metadata.set(TikaCoreProperties.TIKA_CHUNKS, renderChunks);
+ if (renderChunks != null) {
+ metadata.setTrusted(TikaCoreProperties.TIKA_CHUNKS.getName(),
+ mergeChunkArrays(metadata.get(TikaCoreProperties.TIKA_CHUNKS),
+ renderChunks));
}
}
} catch (IOException e) {
@@ -611,6 +613,31 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
}
+ /**
+ * Appends two serialized tk:chunks JSON arrays without a JSON dependency, so each
+ * OCR'd page's chunks accumulate on the parent instead of first-page-wins.
+ * Falls back to the new value if either side is not an array.
+ */
+ static String mergeChunkArrays(String existing, String added) {
+ if (existing == null || existing.isBlank()) {
+ return added;
+ }
+ String e = existing.trim();
+ String a = added.trim();
+ if (!e.startsWith("[") || !e.endsWith("]") || !a.startsWith("[") || !a.endsWith("]")) {
+ return a;
+ }
+ String eBody = e.substring(1, e.length() - 1).trim();
+ String aBody = a.substring(1, a.length() - 1).trim();
+ if (eBody.isEmpty()) {
+ return a;
+ }
+ if (aBody.isEmpty()) {
+ return e;
+ }
+ return "[" + eBody + "," + aBody + "]";
+ }
+
private RenderResult renderCurrentPage(PDPage pdPage, TemporaryResources tmpResources)
throws IOException, TikaException {
PDFRenderingState renderingState = context.get(PDFRenderingState.class);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index da6d0a7282..b44ed01df7 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -1619,6 +1619,58 @@ public class PDFParserTest extends TikaTest {
assertTrue(sawTimeoutWarning,
"the timeout must be recorded, not silently dropped, once caught and continued past");
}
+
+ /**
+ * tk:chunks written by the OCR-slot parser (e.g. an image-embedding parser) on each
+ * rendered page must all reach the parent metadata, not just the first page's.
+ */
+ @Test
+ public void testChunksFromAllOcrPagesReachParent() throws Exception {
+ PDFParserConfig config = new PDFParserConfig();
+ config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ context.set(Parser.class, new Parser() {
+ @Override
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return Collections.singleton(
+ MediaType.image("ocr-" + config.getOcr().getImageFormat().getFormatName()));
+ }
+
+ @Override
+ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata,
+ ParseContext context) throws IOException, SAXException, TikaException {
+ int currentPage = context.get(OCRPageCounter.class).getCount();
+ metadata.setTrusted(TikaCoreProperties.TIKA_CHUNKS.getName(),
+ "[{\"text\":\"chunk-page-" + currentPage + "\"}]");
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+ xhtml.startDocument();
+ xhtml.endDocument();
+ }
+ });
+
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = getResourceAsStream("/test-documents/testPDF_bookmarks.pdf")) {
+ new PDFParser().parse(tis, new ToXMLContentHandler(), metadata, context);
+ }
+
+ String chunks = metadata.get(TikaCoreProperties.TIKA_CHUNKS);
+ assertNotNull(chunks);
+ assertContains("chunk-page-1", chunks);
+ assertContains("chunk-page-2", chunks);
+ }
+
+ @Test
+ public void testMergeChunkArrays() {
+ assertEquals("[b]", AbstractPDF2XHTML.mergeChunkArrays(null, "[b]"));
+ assertEquals("[b]", AbstractPDF2XHTML.mergeChunkArrays(" ", "[b]"));
+ assertEquals("[a,b]", AbstractPDF2XHTML.mergeChunkArrays("[a]", "[b]"));
+ assertEquals("[a]", AbstractPDF2XHTML.mergeChunkArrays("[a]", "[]"));
+ assertEquals("[b]", AbstractPDF2XHTML.mergeChunkArrays("[]", "[b]"));
+ assertEquals("[b]", AbstractPDF2XHTML.mergeChunkArrays("not-an-array", "[b]"));
+ }
+
/**
* TODO -- need to test signature extraction
*/