You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/05/13 16:45:01 UTC

(tika) branch haystack-pipes-parsemode created (now 7b1b698e66)

This is an automated email from the ASF dual-hosted git repository.

tballison pushed a change to branch haystack-pipes-parsemode
in repository https://gitbox.apache.org/repos/asf/tika.git


      at 7b1b698e66 improve parsemode configuration

This branch includes the following new commits:

     new 7b1b698e66 improve parsemode configuration

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(tika) 01/01: improve parsemode configuration

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tballison pushed a commit to branch haystack-pipes-parsemode
in repository https://gitbox.apache.org/repos/asf/tika.git

commit 7b1b698e6680d5bb24b31e1651398b15270633cd
Author: tballison <ti...@elastic.co>
AuthorDate: Wed May 13 12:44:49 2026 -0400

    improve parsemode configuration
---
 docs/modules/ROOT/pages/pipes/parse-modes.adoc        |  7 ++++---
 .../tika/pipes/core/server/ConnectionHandler.java     |  2 +-
 .../apache/tika/pipes/core/server/PipesServer.java    |  6 ++++--
 .../apache/tika/pipes/core/server/PipesWorker.java    |  8 +++++++-
 .../tika/pipes/core/server/ServerProtocolIO.java      | 19 ++++++++++++++-----
 .../tika/pipes/core/server/SharedServerResources.java |  1 +
 tika-pipes/tika-pipes-fork-parser/pom.xml             | 11 +++++++++++
 7 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/docs/modules/ROOT/pages/pipes/parse-modes.adoc b/docs/modules/ROOT/pages/pipes/parse-modes.adoc
index 2a1af6a593..6e5f47fa4e 100644
--- a/docs/modules/ROOT/pages/pipes/parse-modes.adoc
+++ b/docs/modules/ROOT/pages/pipes/parse-modes.adoc
@@ -20,7 +20,8 @@
 :toclevels: 3
 
 Tika Pipes uses `ParseMode` to control how documents are parsed and how results are emitted.
-The parse mode is set on the `ParseContext` or configured in `PipesConfig`.
+The parse mode is configured in the `pipes` section of the JSON config, or overridden per-request
+in the `parseContext` field of a `FetchEmitTuple`.
 
 == Available Parse Modes
 
@@ -87,7 +88,7 @@ to all modes that produce content (`RMETA`, `CONCATENATE`, `CONTENT_ONLY`).
 [source,json]
 ----
 {
-  "parseContext": {
+  "pipes": {
     "parseMode": "CONCATENATE"
   }
 }
@@ -125,7 +126,7 @@ useful for:
 [source,json]
 ----
 {
-  "parseContext": {
+  "pipes": {
     "parseMode": "CONTENT_ONLY"
   }
 }
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
index 3162f0922a..c6f802e516 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
@@ -230,7 +230,7 @@ public class ConnectionHandler implements Runnable, Closeable {
                 resources.getEmitStrategy(), resources.getEmitterManager(), threshold);
         return new PipesWorker(fetchEmitTuple, mergedContext, resources.getAutoDetectParser(),
                 resources.getEmitterManager(), fetchHandler, parseHandler, emitHandler,
-                resources.getDefaultMetadataWriteLimiterFactory());
+                resources.getDefaultMetadataWriteLimiterFactory(), pipesConfig.getParseMode());
     }
 
     private void loopUntilDone(FetchEmitTuple fetchEmitTuple, ParseContext mergedContext,
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
index fb7a74551f..1fd9df1a2a 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
@@ -61,6 +61,7 @@ import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.RecursiveParserWrapper;
 import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.api.ParseMode;
 import org.apache.tika.pipes.api.PipesResult;
 import org.apache.tika.pipes.core.EmitStrategy;
 import org.apache.tika.pipes.core.EmitStrategyConfig;
@@ -403,7 +404,8 @@ public class PipesServer implements AutoCloseable {
         long threshold = (thresholdBytes != null) ? thresholdBytes : EmitStrategyConfig.DEFAULT_DIRECT_EMIT_THRESHOLD_BYTES;
         EmitHandler emitHandler = new EmitHandler(defaultMetadataFilter, emitStrategy, emitterManager, threshold);
         return new PipesWorker(fetchEmitTuple, mergedContext, autoDetectParser, emitterManager,
-                fetchHandler, parseHandler, emitHandler, defaultMetadataWriteLimiterFactory);
+                fetchHandler, parseHandler, emitHandler, defaultMetadataWriteLimiterFactory,
+                pipesConfig.getParseMode());
     }
 
     private void loopUntilDone(FetchEmitTuple fetchEmitTuple, ParseContext mergedContext,
@@ -550,7 +552,7 @@ public class PipesServer implements AutoCloseable {
         if (mergedContext.get(EmbeddedDocumentExtractorFactory.class) == null) {
             mergedContext.set(EmbeddedDocumentExtractorFactory.class, new UnpackExtractorFactory());
         }
-        // Overlay request's values (request takes precedence)
+        // Request-level values override config defaults
         mergedContext.copyFrom(requestContext);
         return mergedContext;
     }
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
index 136853e5d2..a76defc641 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
@@ -69,10 +69,12 @@ class PipesWorker implements Callable<PipesResult> {
     private final ParseHandler parseHandler;
     private final EmitHandler emitHandler;
     private final MetadataWriteLimiterFactory defaultMetadataWriteLimiterFactory;
+    private final ParseMode defaultParseMode;
 
     public PipesWorker(FetchEmitTuple fetchEmitTuple, ParseContext parseContext, AutoDetectParser autoDetectParser,
                        EmitterManager emitterManager, FetchHandler fetchHandler, ParseHandler parseHandler,
-                       EmitHandler emitHandler, MetadataWriteLimiterFactory defaultMetadataWriteLimiterFactory) {
+                       EmitHandler emitHandler, MetadataWriteLimiterFactory defaultMetadataWriteLimiterFactory,
+                       ParseMode defaultParseMode) {
         this.fetchEmitTuple = fetchEmitTuple;
         this.parseContext = parseContext;
         this.autoDetectParser = autoDetectParser;
@@ -81,6 +83,7 @@ class PipesWorker implements Callable<PipesResult> {
         this.parseHandler = parseHandler;
         this.emitHandler = emitHandler;
         this.defaultMetadataWriteLimiterFactory = defaultMetadataWriteLimiterFactory;
+        this.defaultParseMode = defaultParseMode;
     }
 
     @Override
@@ -607,6 +610,9 @@ class PipesWorker implements Callable<PipesResult> {
         }
 
         ParseMode parseMode = parseContext.get(ParseMode.class);
+        if (parseMode == null) {
+            parseMode = defaultParseMode;
+        }
         UnpackConfig unpackConfig = parseContext.get(UnpackConfig.class);
 
         // For UNPACK mode, automatically set up byte extraction
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
index 3d71f87457..531db0036f 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
@@ -20,6 +20,9 @@ import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.tika.exception.TikaConfigException;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.parser.ParseContext;
@@ -45,6 +48,8 @@ import org.apache.tika.utils.StringUtils;
  */
 public class ServerProtocolIO {
 
+    private static final Logger LOG = LoggerFactory.getLogger(ServerProtocolIO.class);
+
     private final DataInputStream input;
     private final DataOutputStream output;
 
@@ -122,12 +127,16 @@ public class ServerProtocolIO {
         UnpackConfig unpackConfig = requestContext.get(UnpackConfig.class);
         ParseMode parseMode = requestContext.get(ParseMode.class);
 
+        // Warn (don't throw) when UnpackConfig has an emitter but ParseMode is not UNPACK.
+        // The global parse-context may include UnpackConfig as a default for UNPACK pipe runs,
+        // but the /rmeta and /tika endpoints explicitly set RMETA mode and PipesWorker correctly
+        // ignores UnpackConfig for non-UNPACK modes. Throwing here would crash the child process.
         if (unpackConfig != null && !StringUtils.isBlank(unpackConfig.getEmitter())
-                && parseMode != ParseMode.UNPACK) {
-            throw new TikaConfigException(
-                    "FetchEmitTuple has UnpackConfig with emitter '" + unpackConfig.getEmitter() +
-                            "' but ParseMode is " + parseMode + ". " +
-                            "To extract embedded bytes, set ParseMode.UNPACK in the ParseContext.");
+                && parseMode != null && parseMode != ParseMode.UNPACK) {
+            LOG.warn("FetchEmitTuple has UnpackConfig with emitter '{}' but ParseMode is {}. "
+                    + "UnpackConfig will be ignored. "
+                    + "To extract embedded bytes, set ParseMode.UNPACK in the ParseContext.",
+                    unpackConfig.getEmitter(), parseMode);
         }
     }
 }
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java
index e92f3455cd..cb5f8412a6 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java
@@ -158,6 +158,7 @@ public class SharedServerResources {
         if (mergedContext.get(EmbeddedDocumentExtractorFactory.class) == null) {
             mergedContext.set(EmbeddedDocumentExtractorFactory.class, new UnpackExtractorFactory());
         }
+        // Request-level values override config defaults
         mergedContext.copyFrom(requestContext);
         return mergedContext;
     }
diff --git a/tika-pipes/tika-pipes-fork-parser/pom.xml b/tika-pipes/tika-pipes-fork-parser/pom.xml
index fd5856aa6a..8d381e9ed3 100644
--- a/tika-pipes/tika-pipes-fork-parser/pom.xml
+++ b/tika-pipes/tika-pipes-fork-parser/pom.xml
@@ -59,6 +59,17 @@
       <version>${project.version}</version>
       <type>pom</type>
     </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>tika-langdetect-charsoup</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <!-- tika-eval: TikaEvalMetadataFilter (tokens, OOV, lang, languageness) -->
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>tika-eval-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>tika-core</artifactId>