You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/09/01 11:30:43 UTC
(tika) branch main updated: TIKA-4848: cross-PR final review (#3108)
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new ea3cc5480c TIKA-4848: cross-PR final review (#3108)
ea3cc5480c is described below
commit ea3cc5480c112034b8333504c01f026424ac7be1
Author: Tim Allison <ta...@apache.org>
AuthorDate: Tue Sep 1 07:30:36 2026 -0400
TIKA-4848: cross-PR final review (#3108)
---
.skills/devs/feature-workflow/SKILL.md | 27 ++++++
CHANGES.txt | 28 +++---
.../integration-testing/run-uat-script.adoc | 5 +
.../ROOT/pages/advanced/setting-limits.adoc | 60 +++++++++---
.../migration-to-4x/migrating-tika-server-4x.adoc | 14 ++-
docs/modules/ROOT/pages/pipes/configuration.adoc | 4 +
docs/modules/ROOT/pages/pipes/parse-modes.adoc | 2 +
.../ROOT/pages/pipes/plugins/filesystem.adoc | 2 +-
docs/modules/ROOT/pages/using-tika/cli/index.adoc | 4 +
docs/modules/ROOT/pages/using-tika/grpc/index.adoc | 7 ++
.../ROOT/pages/using-tika/server/index.adoc | 18 +++-
release-tools/uat/run-uat.sh | 44 +++++++++
.../org/apache/tika/config/ExceptionReporting.java | 26 ++---
.../tika/extractor/EmbeddedDocumentUtil.java | 18 ++--
.../apache/tika/parser/RecursiveParserWrapper.java | 2 +-
.../java/org/apache/tika/utils/ExceptionUtils.java | 51 +++++-----
.../java/org/apache/tika/utils/ParserUtils.java | 5 +-
.../tika/config/ExceptionReportingParseTest.java | 5 +-
.../apache/tika/config/ExceptionReportingTest.java | 31 +++---
.../tika/extractor/EmbeddedDocumentUtilTest.java | 108 +++++++++++++++++++++
.../org/apache/tika/utils/ExceptionUtilsTest.java | 46 +++++----
.../tika/eval/core/util/EvalExceptionUtils.java | 9 +-
.../eval/core/util/EvalExceptionUtilsTest.java | 23 ++++-
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 85 +++++++++++++---
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 92 ++++++++++++++++++
.../forbidden-apis-exception-reporting-bans.txt | 26 +++++
tika-parent/forbidden-apis-none.txt | 18 ++++
tika-parent/pom.xml | 26 ++++-
.../tika-parser-scientific-module/pom.xml | 20 +---
.../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 22 ++---
.../apache/tika/parser/crypto/TSDParserTest.java | 4 +-
.../tika-parser-apple-module/pom.xml | 20 +---
.../tika-parser-audiovideo-module/pom.xml | 20 +---
.../java/org/apache/tika/parser/mp4/MP4Parser.java | 13 ++-
.../org/apache/tika/parser/mp4/TikaMp4Reader.java | 17 ++--
.../org/apache/tika/parser/mp4/MP4ParserTest.java | 28 +++++-
.../tika-parser-cad-module/pom.xml | 20 +---
.../org/apache/tika/parser/crypto/TSDParser.java | 4 +-
.../tika-parser-font-module/pom.xml | 20 +---
.../tika-parser-image-module/pom.xml | 20 +---
.../tika-parser-mail-module/pom.xml | 20 +---
.../tika-parser-microsoft-module/pom.xml | 20 +---
.../tika/parser/microsoft/HSLFExtractor.java | 2 +-
.../tika/parser/microsoft/JackcessExtractor.java | 14 ++-
.../parser/microsoft/onenote/OneNoteParser.java | 31 +++---
.../microsoft/ooxml/AbstractOOXMLExtractor.java | 5 +-
.../ooxml/SXSLFPowerPointExtractorDecorator.java | 2 -
.../ooxml/SXWPFWordExtractorDecorator.java | 2 -
.../microsoft/ooxml/VSDXExtractorDecorator.java | 3 -
.../ooxml/XSSFBExcelExtractorDecorator.java | 1 -
.../ooxml/XSSFExcelExtractorDecorator.java | 64 +++++-------
.../microsoft/ooxml/xps/XPSExtractorDecorator.java | 2 -
.../parser/microsoft/rtf/RTFEmbObjHandler.java | 4 +-
.../parser/microsoft/rtf/RTFObjDataParser.java | 27 ++----
.../microsoft/onenote/OneNoteParserTest.java | 79 ++++++++++-----
.../tika-parser-miscoffice-module/pom.xml | 20 +---
.../tika-parser-pdf-module/pom.xml | 20 +---
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 81 ++++++----------
.../tika/parser/pdf/image/ImageGraphicsEngine.java | 8 +-
.../parser/pdf/image/ImageGraphicsEngineTest.java | 90 +++++++++++++++++
.../java/org/apache/tika/parser/pkg/ZipParser.java | 2 +-
.../tika-parser-webarchive-module/pom.xml | 20 +---
.../tika-parser-xml-module/pom.xml | 20 +---
.../org/apache/tika/pipes/core/PipesClient.java | 15 +--
.../org/apache/tika/pipes/core/PipesConfig.java | 39 ++++++--
.../apache/tika/pipes/core/async/AsyncEmitter.java | 9 +-
.../apache/tika/pipes/core/server/EmitHandler.java | 21 +---
.../tika/pipes/core/server/FetchHandler.java | 21 +---
.../apache/tika/pipes/core/server/PipesServer.java | 44 ++++-----
.../tika/pipes/core/server/ServerProtocolIO.java | 8 +-
.../pipes/core/server/SharedServerResources.java | 2 +-
.../tika/pipes/core/TikaPipesConfigTest.java | 19 ++++
.../pipes/core/server/ServerProtocolIOTest.java | 4 +-
.../pipes/core/ExceptionReportingPipesTest.java | 70 +++++++++++--
.../apache/tika/pipes/core/PipesClientTest.java | 6 ++
.../tika-config-exception-reporting-emit-all.json | 41 ++++++++
...config-exception-reporting-startup-failure.json | 29 ++++++
.../pipes/reporter/fs/FileSystemJsonlReporter.java | 3 +-
.../reporter/fs/FileSystemStatusReporter.java | 3 +-
.../serdes/ParseContextSerializer.java | 21 +++-
....java => ExceptionReportingConfigLoadTest.java} | 11 +--
.../TestParseContextSerialization.java | 48 +++++++++
.../WireRestrictedParseContextTest.java | 10 +-
.../tika/server/core/CatchAllExceptionMapper.java | 75 ++++++++++++++
.../core/TikaServerParseExceptionMapper.java | 53 +++-------
.../apache/tika/server/core/TikaServerProcess.java | 1 +
.../server/core/resource/MetadataResource.java | 10 +-
.../server/core/resource/PipesParsingHelper.java | 19 +++-
.../tika/server/core/resource/TikaResource.java | 15 ++-
...rTest.java => CatchAllExceptionMapperTest.java} | 37 +++----
.../tika/server/core/RedactedStackTraceTest.java | 96 +++++++++++++++++-
.../apache/tika/server/core/StackTraceTest.java | 3 +
.../core/TikaServerParseExceptionMapperTest.java | 64 +++++++++---
93 files changed, 1587 insertions(+), 741 deletions(-)
diff --git a/.skills/devs/feature-workflow/SKILL.md b/.skills/devs/feature-workflow/SKILL.md
index 218c1a5ed8..8ac3fb5fa3 100644
--- a/.skills/devs/feature-workflow/SKILL.md
+++ b/.skills/devs/feature-workflow/SKILL.md
@@ -77,3 +77,30 @@ widen a PR during review.
| PR review changes an interface | pull piece back to spike |
| PR review finds out-of-scope issue | todo doc, not this PR |
| Third review round on a PR | one of the two above applies |
+
+## Cross-cutting refactors (chokepoints, wire formats)
+
+A refactor claiming "every X goes through Y" — a policy chokepoint, a single
+serialization route — is a contract too, with three extra rules. (Learned on
+TIKA-4848, where all three were violated and each violation became a
+post-merge review round.)
+
+- **Enforcement ships with the invariant, default-on.** The PR that creates
+ the chokepoint also adds the build-breaking check (forbidden-apis ban,
+ architecture test) that keeps call sites on it — every module, explicit
+ opt-outs. Opt-in checks document an invariant; only default-on maintains
+ it. Without this, each missed call site is a review finding instead of a
+ compile failure.
+
+- **One thin end-to-end test per transport surface.** A knob that crosses a
+ JVM boundary is exercised through each surface it rides (in-JVM, server,
+ pipes fork, grpc) in the invariant's own PR. Diff review cannot see a
+ parent-resolves/fork-deserializes seam; a wire test can.
+
+- **Producer and consumer change together.** Both halves of a
+ serializer/deserializer or parent/fork protocol pair go in the same PR.
+ Cutting by module puts every seam between two reviews, and seams are where
+ cross-cutting bugs live.
+
+No compat shim without a caller: grep before deprecating-in-place. A shim
+for an API nothing calls is dead weight that still carries bugs.
diff --git a/CHANGES.txt b/CHANGES.txt
index 572e357624..ce1baa8f55 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,16 @@
Release 4.1.0 - unreleased
+ * Add "exception-reporting" parse-context config to redact and bound
+ exception text in metadata, tika-server error bodies and pipes/grpc
+ messages; FileSystemEmitter writes atomically (TIKA-4848).
+ Compat notes: a truncated TSD envelope now records its read failure
+ under tk:exception:embedded-stream-exception rather than
+ tk:exception:embedded-exception; recordException and
+ recordEmbeddedStreamException no longer strip a bare TikaException
+ wrapper, so the first line of tk:exception:* values may name the
+ wrapper (affects consumers keyed on that line, e.g. eval cause
+ counts across the 4.1 boundary).
+
* Audio cover art is emitted as a THUMBNAIL embedded document, like the
preview image of the document container formats: the front cover (ID3
APIC and FLAC/Vorbis picture type 3), else the first picture of type
@@ -33,7 +44,7 @@ Release 4.1.0 - unreleased
records; part of TIKA-4848 step 3 (TIKA-4848).
* Allow image compression settings in PDFBox-based renderer (TIKA-4862).
-
+
* The tika-server full and tika-grpc Docker images set OMP_THREAD_LIMIT=1:
to avoid oversubscribing the CPU under forked parse workers (TIKA-4863).
@@ -75,21 +86,6 @@ Release 4.1.0 - unreleased
THUMBNAIL, so a client had to compare them to find the representative
one (TIKA-4851).
- * New exception-reporting parse-context config controls how much of an
- exception is written to tk:exception:* metadata for the container and
- embedded documents alike: FULL (default), MESSAGE_REDACTED (stack trace
- without messages) or REDACTED (class names only), plus a maxLength.
- Config-only; rejected in per-request parse-context. Parsers should
- format exceptions via ExceptionUtils.format(Throwable, ParseContext);
- ExceptionUtils.getStackTrace/getFilteredStackTrace and the ParseContext-less
- EmbeddedDocumentUtil.recordException* / ParserUtils.recordParserFailure
- overloads are deprecated (TIKA-4848).
-
- * FileSystemEmitter writes to a sibling ".tmp" file and renames it into
- place, so readers of the output directory never see a partially written
- file. Set "atomicWrites": false on the emitter to restore in-place
- writes (TIKA-4848).
-
* tika-eval: Profile/Compare accept the batch run's jsonl crash ledger
(--pipesReport, -pa/-pb) and a run-info json (--runInfo, -ra/-rb), and
read both from <extracts>/.run-info/ by default (refusing an ambiguous
diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/run-uat-script.adoc b/docs/modules/ROOT/pages/advanced/integration-testing/run-uat-script.adoc
index f1d745cf1a..6aaa3d7fa4 100644
--- a/docs/modules/ROOT/pages/advanced/integration-testing/run-uat-script.adoc
+++ b/docs/modules/ROOT/pages/advanced/integration-testing/run-uat-script.adoc
@@ -73,6 +73,11 @@ Coverage includes:
4.0.0-alpha-1 and beta-1: the resource failed to load and the endpoint 406'd
* Removed endpoints stay removed (`/translate/*` returns 404)
* 404 / 405 error handling
+* Exception reporting on a corrupt document (TIKA-4848): `/rmeta` embeds
+ `tk:exception:container-exception` at `200`; raw `/tika` returns `422` with a
+ content-only body; `/unpack` and `/meta/\{field}` carry the container exception
+ in their `422` bodies, asserted free of this server's own frames
+ (`PipesParsingHelper`, `MetadataResource`)
* OCR (`PUT /tika/text` of an image whose text only exists as pixels): skipped when the
server has no tesseract (the minimal image / a plain `java -jar` server), and a hard
failure when `TIKA_UAT_REQUIRE_OCR=1` is set. `docker-tool.sh test-uat` sets that env
diff --git a/docs/modules/ROOT/pages/advanced/setting-limits.adoc b/docs/modules/ROOT/pages/advanced/setting-limits.adoc
index 98588bf65f..2f8c208cf6 100644
--- a/docs/modules/ROOT/pages/advanced/setting-limits.adoc
+++ b/docs/modules/ROOT/pages/advanced/setting-limits.adoc
@@ -55,8 +55,9 @@ throughout the parse.
== Configuring limits
-Every limit group is a `parse-context` block. This is the configuration
-exercised by `AllLimitsTest` (`tika-serialization/src/test/resources/configs/all-limits-test.json`):
+Every limit group is a `parse-context` block. `AllLimitsTest`
+(`tika-serialization/src/test/resources/configs/all-limits-test.json`) exercises all of
+these but `exception-reporting`, which has its own test:
[source,json]
----
@@ -86,6 +87,10 @@ exercised by `AllLimitsTest` (`tika-serialization/src/test/resources/configs/all
"maxFieldSize": 102400,
"maxKeySize": 1024,
"maxValuesPerField": 100
+ },
+ "exception-reporting": {
+ "level": "MESSAGE_REDACTED",
+ "maxLength": 10000
}
}
}
@@ -113,8 +118,9 @@ context.set(OutputLimits.class, new OutputLimits(50000, true, 50, 5, 500000, 50)
context.set(TimeoutLimits.class, new TimeoutLimits(7200000, 120000));
----
-Tests: `AllLimitsTest`, `EmbeddedLimitsTest`, `OutputLimitsTest`, and
-`TimeoutLimitsTest` under `tika-serialization/src/test/java/org/apache/tika/config/`.
+Tests: `AllLimitsTest`, `EmbeddedLimitsTest`, `OutputLimitsTest`, `TimeoutLimitsTest` and
+`ExceptionReportingConfigLoadTest` under `tika-serialization/src/test/java/org/apache/tika/config/`;
+`ExceptionReportingParseTest` (tika-core) drives the policy through a real parse.
== Embedded document limits
@@ -354,10 +360,17 @@ if ("true".equals(metadata.get(TikaCoreProperties.TRUNCATED_METADATA))) {
`ExceptionReporting` controls how much detail Tika reports when a parse fails:
the `tk:exception:container-exception`, `tk:exception:embedded-exception`,
-`tk:exception:warn` and `tk:exception:embedded-stream` metadata values. Stack
-traces and exception messages can carry file paths, hostnames and fragments of
-the document; the metadata limiter deliberately never truncates them (see
-<<Always-included fields>>), so this is the only bound on them.
+`tk:exception:warn` and `tk:exception:embedded-stream-exception` metadata
+values. Stack traces and messages can carry file paths, hostnames and fragments
+of the document, and the metadata limiter is no backstop: with no
+`MetadataWriteLimiterFactory` configured these values are unbounded, and with
+one, only `container-exception` and `embedded-exception` are always-included
+(cut at `max(maxFieldSize, 300)` UTF-16 bytes); `warn` and
+`embedded-stream-exception` are ordinary fields, subject to
+`includeFields`/`excludeFields` and dropped past `maxValuesPerField` (see
+<<Always-included fields>>). The limiter's cut appends no marker, so keep
+`maxLength` (characters) under half `maxFieldSize` (UTF-16 bytes) if you want
+the `...[truncated]` marker to survive.
[cols="2,1,3"]
|===
@@ -371,8 +384,8 @@ chain are kept). `REDACTED`: exception class names only, no frames.
|`maxLength`
|-1 (unlimited)
-|Maximum characters in the formatted exception; longer values end with
-`...[truncated]`.
+|Length at which the formatted exception is cut; a truncated value is
+`maxLength` characters plus the `...[truncated]` marker.
|===
[source,json]
@@ -384,14 +397,29 @@ chain are kept). `REDACTED`: exception class names only, no frames.
----
The same policy applies to the container and to every embedded document, to
-parser-level warnings (`tk:exception:warn`, `tk:exception:embedded-stream`), to
-the messages a pipes worker returns on fetch/emit/crash, and to tika-server's
-`422` error bodies. It is loaded from the config only and is rejected in a per-request `parse-context`
-(tika-server `/rmeta/config` and friends, `/pipes`, `/async`), so a caller cannot
-turn redaction back off.
+parser-level warnings (`tk:exception:warn`,
+`tk:exception:embedded-stream-exception`), to every pipes result message
+(fetch/emit/crash inside the worker, and the results the parent fabricates when
+the worker never reports one), and to every tika-server error body built from an
+exception -- `422` and `500`. It is loaded from the config only and is
+rejected in a per-request `parse-context` -- tika-server `/rmeta/config` and
+friends, `/pipes` and `/async` answer `400` with the reason, tika-grpc's
+`parse_context_json` answers `INVALID_ARGUMENT` -- so a caller cannot turn
+redaction back off.
Programmatically, parsers and callers should format exceptions through
-`ExceptionUtils.format(Throwable, ParseContext)`.
+`ExceptionUtils.format(Throwable, ParseContext)`. A parser still calling the
+deprecated two-argument `EmbeddedDocumentUtil.recordException` /
+`recordEmbeddedStreamException` -- including any third-party parser compiled
+against 4.0.0 -- reports `FULL` whatever the policy says. To set a policy in a
+parser test:
+
+[source,java]
+----
+ParseContext context = new ParseContext();
+context.set(ExceptionReporting.class,
+ new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, 500));
+----
== Recommendations
diff --git a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
index 5452390246..e1859febae 100644
--- a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
+++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
@@ -155,7 +155,9 @@ at least a `status` field:
----
A `message` field is included when one is available and may contain a server-side stack
-trace, e.g. `{"status": "TIMEOUT", "message": "Task timed out after 60000ms"}`.
+trace, e.g. `{"status": "TIMEOUT", "message": "Task timed out after 60000ms"}`. How much of
+that trace is returned is set by `parse-context.exception-reporting` (default `FULL`) — see
+xref:advanced/setting-limits.adoc#_exception_reporting[Exception reporting].
The HTTP status codes are also more precise:
@@ -329,8 +331,14 @@ The following `TikaServerConfig` options have been removed:
exception detail — including messages carrying file paths and document fragments —
continued to travel in `tk:exception:*` metadata on successful parses and inside
`/unpack` zips. Setting it to `false` did not keep stack traces inside the server, which
- is what the name implied. Exception detail is now always returned; filter Tika's output
- with a `MetadataFilter` before forwarding it somewhere less trusted.
+ is what the name implied. The replacement is `parse-context.exception-reporting`
+ (`MESSAGE_REDACTED` or `REDACTED`, plus a `maxLength`), one operator-set policy over every
+ channel that turns an exception into caller-visible text — metadata values, error bodies and
+ pipes result messages alike. See
+ xref:advanced/setting-limits.adoc#_exception_reporting[Exception reporting]. A
+ `MetadataFilter` dropping `tk:exception:*` is a convenience, not the replacement: it reaches
+ neither error bodies nor pipes result messages, and with `allowPerRequestConfig` on, a filter
+ supplied in a request's config *replaces* the operator's rather than adding to it.
* `taskTimeoutMillis` - Was the total time allowed per task before the forked process was
killed; that is now `parse-context.timeout-limits.totalTaskTimeoutMillis` (same
diff --git a/docs/modules/ROOT/pages/pipes/configuration.adoc b/docs/modules/ROOT/pages/pipes/configuration.adoc
index f356d23c8c..fb1c205117 100644
--- a/docs/modules/ROOT/pages/pipes/configuration.adoc
+++ b/docs/modules/ROOT/pages/pipes/configuration.adoc
@@ -141,6 +141,10 @@ set `EMIT`. Nothing reads the value from this section.
|When `false`, stop the pipeline on configuration errors (missing fetcher/emitter). When `true`, only stop on fatal initialization failures. Use `true` for server mode, `false` for batch mode.
|===
+How much of a failure is written into `tk:exception:*` metadata and into a `PipesResult`
+message is set by `parse-context.exception-reporting`, not here -- see
+xref:advanced/setting-limits.adoc#_exception_reporting[Exception reporting].
+
== Async / Emit Batching
These settings control how parsed results are batched before sending to emitters.
diff --git a/docs/modules/ROOT/pages/pipes/parse-modes.adoc b/docs/modules/ROOT/pages/pipes/parse-modes.adoc
index eb3155bc2b..b5c9b60de1 100644
--- a/docs/modules/ROOT/pages/pipes/parse-modes.adoc
+++ b/docs/modules/ROOT/pages/pipes/parse-modes.adoc
@@ -130,6 +130,8 @@ these values. If unset, the default is `MARKDOWN`.
If the container parse fails (`SAXException`, `EncryptedDocumentException`, or any other `Exception`), the stack trace is caught, logged, and stored on the container metadata as `tk:exception:container-exception`. The parse continues to a return value rather than throwing — callers must check this field if they need to detect failure.
+How much of that trace is stored is set by `parse-context.exception-reporting` — see xref:advanced/setting-limits.adoc#_exception_reporting[Exception reporting].
+
If the configured write limit is reached during concatenation, `tk:exception:write-limit-reached` is set to `true`.
[#content-only-mode]
diff --git a/docs/modules/ROOT/pages/pipes/plugins/filesystem.adoc b/docs/modules/ROOT/pages/pipes/plugins/filesystem.adoc
index f25773c311..c5000a594a 100644
--- a/docs/modules/ROOT/pages/pipes/plugins/filesystem.adoc
+++ b/docs/modules/ROOT/pages/pipes/plugins/filesystem.adoc
@@ -146,7 +146,7 @@ Writes parsed results as files under `basePath`. The relative output path is der
|`atomicWrites`
|`true`
-|Write each output to a sibling `<name>.<uuid>.tmp` and rename it into place, so a reader of the output directory never sees a partial file. Set to `false` to write in place (one fewer rename per file; needed on filesystems where rename is slow or not atomic). Init-time only.
+|Write each output to a sibling `<name>.<uuid>.tmp` and rename it into place. A reader of the output directory never sees a partial file, but it does see the tmp files: crawlers and watchers over the output directory must ignore `*.tmp`. A worker killed mid-write leaves its `.tmp` behind, and nothing sweeps them. The `onExists: EXCEPTION`/`SKIP` check runs immediately before the rename and is best-effort against two workers emitting the same key concurrently (last writer wins) -- give wo [...]
|===
[#file-system-iterator]
diff --git a/docs/modules/ROOT/pages/using-tika/cli/index.adoc b/docs/modules/ROOT/pages/using-tika/cli/index.adoc
index c02c2f8771..27670fad86 100644
--- a/docs/modules/ROOT/pages/using-tika/cli/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/cli/index.adoc
@@ -115,6 +115,10 @@ For Pipes-mode options see <<_tika_pipes_processing,Tika Pipes Processing>>.
Redirect to save: `--convert-config-xml-to-json=tika-config.xml > tika-config.json`.
|===
+The config's `parse-context` applies to the CLI too, so `exception-reporting` bounds the
+`tk:exception:*` values in `-J`/`-j` output — see
+xref:advanced/setting-limits.adoc#_exception_reporting[Exception reporting].
+
=== Output Formatting
[cols="1,3"]
diff --git a/docs/modules/ROOT/pages/using-tika/grpc/index.adoc b/docs/modules/ROOT/pages/using-tika/grpc/index.adoc
index 748a5d1a2b..0e9b7a7135 100644
--- a/docs/modules/ROOT/pages/using-tika/grpc/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/grpc/index.adoc
@@ -201,6 +201,13 @@ request. Keys are parse-context component names; values are their JSON configs.
See `META-INF/tika/parse-context.idx` (generated at build time from `@TikaComponent` annotations)
for the available component names.
+A few components are operator-only and never accepted from the wire —
+`exception-reporting` (xref:advanced/setting-limits.adoc#_exception_reporting[Exception
+reporting]) is one. A request naming one is rejected at the API boundary with
+`INVALID_ARGUMENT` and the reason, before the forked worker sees it. A misspelled or
+unregistered component name is refused the same way, so a typo is a client error rather
+than a silent fallback to the server's defaults.
+
NOTE: This is disabled by default. A request setting `parse_context_json` or
`additional_fetch_config_json` is rejected with `PERMISSION_DENIED` unless `allowPerRequestConfig`
is enabled — see <<_capability_flags,Capability flags>>.
diff --git a/docs/modules/ROOT/pages/using-tika/server/index.adoc b/docs/modules/ROOT/pages/using-tika/server/index.adoc
index 12f3c6df55..4c0048228a 100644
--- a/docs/modules/ROOT/pages/using-tika/server/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/server/index.adoc
@@ -451,8 +451,14 @@ spooled file's path, the source filename, and fragments of the document. That de
`tk:exception:*` metadata on **successful** parses as well as in error bodies — including inside
the `*.metadata.json` entries of an `/unpack` zip.
-One config setting governs all of those channels — the metadata values, the `422` bodies above,
-and the `message` of a `/pipes` or `/async` result:
+One config setting governs every one of those channels — the metadata values, every error body
+built from an exception (`422`, `500`; a `400` echoes the caller's own request problem, `415` and
+`404`/`405` carry no exception text, and the plain-text `413` from the request-size filter is
+fixed text), and the `message` of a `/pipes` or `/async` result. That includes the
+results the parent fabricates when the fork never reported one (`FAILED_TO_INITIALIZE`,
+`PAYLOAD_LIMIT_EXCEEDED`, a `TIMEOUT`/`OOM`/crash the fork died before reporting): those carry
+either no message at all or the trace of the parent's own infrastructure exception — never
+document content — formatted under the same policy:
[source,json]
----
@@ -462,10 +468,12 @@ and the `message` of a `/pipes` or `/async` result:
----
`MESSAGE_REDACTED` keeps class names and frames but strips every message; `REDACTED` keeps class
-names only. It is refused in a per-request config so callers cannot turn it back off. See
+names only. A per-request config carrying it is refused with a `400` naming the reason, so callers
+cannot turn it back off. See
xref:advanced/setting-limits.adoc#_exception_reporting[Exception reporting]. Set it whenever the
-server's callers are less trusted than its operator; a `MetadataFilter` can additionally drop the
-`tk:exception:*` fields outright.
+server's callers are less trusted than its operator. Prefer it to a `MetadataFilter` that drops
+`tk:exception:*`: a filter supplied in a per-request config *replaces* the operator's rather than
+adding to it.
====
=== Truncated results (`PARTIAL_TIMEOUT`)
diff --git a/release-tools/uat/run-uat.sh b/release-tools/uat/run-uat.sh
index 2be9361cdb..25f2627f37 100755
--- a/release-tools/uat/run-uat.sh
+++ b/release-tools/uat/run-uat.sh
@@ -141,6 +141,50 @@ assert_status "T33 unknown handler name -> 400" "400" "$BADH"
XMP=$(curl -s -X PUT -T "$PDF" -H 'Accept: application/rdf+xml' "$BASE/meta")
assert_contains "T34 PUT /meta as rdf+xml" "rdf:RDF" "$XMP"
+# --- exception reporting (TIKA-4848), default FULL policy ---
+# Truncate the PDF, not the docx: a truncated zip gets rescued by zip salvage,
+# a truncated PDF reliably fails the parse.
+CORRUPT=$(mktemp); head -c 400 "$PDF" > "$CORRUPT"
+
+# put_corrupt <path> -- PUTs the corrupt file and sets CODE/BODY
+put_corrupt() {
+ local resp
+ resp=$(curl -s -w '\n%{http_code}' -X PUT -T "$CORRUPT" "$BASE/$1")
+ CODE=$(printf '%s' "$resp" | tail -1)
+ BODY=$(printf '%s' "$resp" | sed '$d')
+}
+
+put_corrupt rmeta
+if [[ "$CODE" == "200" ]] && printf '%s' "$BODY" | grep -qF 'tk:exception:container-exception'; then
+ ok "T36 corrupt PUT /rmeta -> 200 + container-exception"
+else
+ bad "T36 corrupt PUT /rmeta" "200 with tk:exception:container-exception" "HTTP $CODE $BODY"
+fi
+
+put_corrupt tika/text
+if [[ "$CODE" == "422" ]] && ! printf '%s' "$BODY" | grep -qiF 'exception'; then
+ ok "T37 corrupt PUT /tika/text -> 422, content-only body"
+else
+ bad "T37 corrupt PUT /tika/text" "422 with no exception text in body" "HTTP $CODE $BODY"
+fi
+
+put_corrupt unpack/all
+if [[ "$CODE" == "422" ]] && printf '%s' "$BODY" | grep -qF 'Exception' \
+ && ! printf '%s' "$BODY" | grep -qF 'PipesParsingHelper'; then
+ ok "T38 corrupt PUT /unpack/all -> 422, container exception only"
+else
+ bad "T38 corrupt PUT /unpack/all" "422 with exception, no PipesParsingHelper frames" "HTTP $CODE $BODY"
+fi
+
+put_corrupt meta/Content-Type
+if [[ "$CODE" == "422" ]] && printf '%s' "$BODY" | grep -qF 'Exception' \
+ && ! printf '%s' "$BODY" | grep -qF 'MetadataResource'; then
+ ok "T39 corrupt PUT /meta/{field} -> 422, container exception only"
+else
+ bad "T39 corrupt PUT /meta/{field}" "422 with exception, no MetadataResource frames" "HTTP $CODE $BODY"
+fi
+rm -f "$CORRUPT"
+
# --- removed endpoints must be gone, not silently doing something ---
TR404=$(curl -s -o /dev/null -w '%{http_code}' -X PUT -T "$PDF" "$BASE/translate/all/foo/en/es")
assert_status "T35 /translate removed -> 404" "404" "$TR404"
diff --git a/tika-core/src/main/java/org/apache/tika/config/ExceptionReporting.java b/tika-core/src/main/java/org/apache/tika/config/ExceptionReporting.java
index 771162dd6c..7548cb4ca9 100644
--- a/tika-core/src/main/java/org/apache/tika/config/ExceptionReporting.java
+++ b/tika-core/src/main/java/org/apache/tika/config/ExceptionReporting.java
@@ -36,20 +36,11 @@ import org.apache.tika.parser.ParseContext;
* <li>{@link Level#REDACTED} - exception class names only (the cause chain, no frames,
* no messages)</li>
* </ul>
- * {@code maxLength} truncates the formatted string to that many characters (-1 = unlimited).
+ * {@code maxLength} caps the formatted string: a longer value is cut to {@code maxLength}
+ * characters and a {@code ...[truncated]} marker is appended (-1 = unlimited).
* <p>
* Loaded from the {@code parse-context} section of the config and deliberately not settable
* per request: a caller could otherwise turn redaction back off.
- * <pre>
- * {
- * "parse-context": {
- * "exception-reporting": {
- * "level": "MESSAGE_REDACTED",
- * "maxLength": 10000
- * }
- * }
- * }
- * </pre>
*
* @since Apache Tika 4.1
*/
@@ -64,8 +55,6 @@ public class ExceptionReporting implements Serializable {
public static final int UNLIMITED = -1;
- public static final ExceptionReporting DEFAULT = new ExceptionReporting();
-
private Level level = Level.FULL;
private int maxLength = UNLIMITED;
@@ -89,10 +78,8 @@ public class ExceptionReporting implements Serializable {
return maxLength;
}
- /**
- * @param maxLength maximum characters in the formatted exception, or -1 for unlimited
- */
public void setMaxLength(int maxLength) {
+ //0 is rejected rather than treated as "report nothing": it would emit only the marker
if (maxLength < UNLIMITED || maxLength == 0) {
throw new IllegalArgumentException(
"maxLength must be positive or -1 for unlimited, was " + maxLength);
@@ -101,15 +88,14 @@ public class ExceptionReporting implements Serializable {
}
/**
- * @return the ExceptionReporting from the context, or {@link #DEFAULT} if the context is
- * null or has none
+ * @return the context's policy, or a new instance with the defaults
*/
public static ExceptionReporting get(ParseContext context) {
if (context == null) {
- return DEFAULT;
+ return new ExceptionReporting();
}
ExceptionReporting reporting = context.get(ExceptionReporting.class);
- return reporting != null ? reporting : DEFAULT;
+ return reporting != null ? reporting : new ExceptionReporting();
}
@Override
diff --git a/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java b/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java
index 24d8074cba..e00ed0c973 100644
--- a/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java
+++ b/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java
@@ -233,12 +233,15 @@ public class EmbeddedDocumentUtil {
public static void recordException(Throwable t, Metadata m, ParseContext context) {
m.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
- ExceptionUtils.format(ExceptionUtils.unwrapTikaException(t), context));
+ ExceptionUtils.format(t, context));
}
/**
- * @deprecated use {@link #recordException(Throwable, Metadata, ParseContext)} so the
- * configured {@link org.apache.tika.config.ExceptionReporting} applies
+ * @deprecated since 4.1, removal planned for 5.0; use
+ * {@link #recordException(Throwable, Metadata, ParseContext)} so the configured
+ * {@link org.apache.tika.config.ExceptionReporting} applies. Unlike in 4.0.0, this now
+ * keeps a bare {@link org.apache.tika.exception.TikaException} wrapper rather than
+ * stripping it, so the first line of the recorded value may name the wrapper.
*/
@Deprecated
public static void recordException(Throwable t, Metadata m) {
@@ -248,12 +251,15 @@ public class EmbeddedDocumentUtil {
public static void recordEmbeddedStreamException(Throwable t, Metadata m,
ParseContext context) {
m.add(TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM,
- ExceptionUtils.format(ExceptionUtils.unwrapTikaException(t), context));
+ ExceptionUtils.format(t, context));
}
/**
- * @deprecated use {@link #recordEmbeddedStreamException(Throwable, Metadata, ParseContext)}
- * so the configured {@link org.apache.tika.config.ExceptionReporting} applies
+ * @deprecated since 4.1, removal planned for 5.0; use
+ * {@link #recordEmbeddedStreamException(Throwable, Metadata, ParseContext)} so the
+ * configured {@link org.apache.tika.config.ExceptionReporting} applies. Unlike in 4.0.0,
+ * this now keeps a bare {@link org.apache.tika.exception.TikaException} wrapper rather
+ * than stripping it, so the first line of the recorded value may name the wrapper.
*/
@Deprecated
public static void recordEmbeddedStreamException(Throwable t, Metadata m) {
diff --git a/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java b/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java
index e522382132..7809deb96a 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java
@@ -171,7 +171,7 @@ public class RecursiveParserWrapper extends ParserDecorator {
metadata.set(TikaCoreProperties.WRITE_LIMIT_REACHED, "true");
} else {
metadata.add(TikaCoreProperties.CONTAINER_EXCEPTION,
- ExceptionUtils.format(ExceptionUtils.unwrapTikaException(e), context));
+ ExceptionUtils.format(e, context));
throw e;
}
} finally {
diff --git a/tika-core/src/main/java/org/apache/tika/utils/ExceptionUtils.java b/tika-core/src/main/java/org/apache/tika/utils/ExceptionUtils.java
index 8e2e539223..09ba412fe3 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/ExceptionUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/ExceptionUtils.java
@@ -29,9 +29,9 @@ import org.apache.tika.parser.ParseContext;
/**
* Turns a {@link Throwable} into the string Tika reports to callers, under the
- * {@link ExceptionReporting} policy in effect. Every place Tika records or emits an exception
- * as text should go through {@link #format(Throwable, ParseContext)} so that one config
- * setting governs all channels.
+ * {@link ExceptionReporting} policy in effect. Tika's own channels record and emit exception
+ * text through {@link #format(Throwable, ParseContext)} so that one config setting governs
+ * them all; exception text produced by third-party libraries is outside its reach.
* <p>
* NOTE: If your stacktraces are truncated, make sure to start your jvm
* with: -XX:-OmitStackTraceInFastThrow
@@ -43,20 +43,20 @@ public class ExceptionUtils {
/**
* Formats {@code t} under the {@link ExceptionReporting} found in {@code context}
- * (or {@link ExceptionReporting#DEFAULT} if the context is null or has none).
+ * (or the default policy if the context is null or has none).
*/
public static String format(Throwable t, ParseContext context) {
return format(t, ExceptionReporting.get(context));
}
/**
- * Formats {@code t} under {@code reporting}; a null policy means
- * {@link ExceptionReporting#DEFAULT}. For channels that have no ParseContext, such as
- * server error responses and pipes crash messages.
+ * Formats {@code t} for channels that have no ParseContext, such as server error
+ * responses and pipes crash messages.
*/
public static String format(Throwable t, ExceptionReporting reporting) {
if (reporting == null) {
- reporting = ExceptionReporting.DEFAULT;
+ //never NPE while formatting someone else's exception
+ reporting = new ExceptionReporting();
}
String s;
switch (reporting.getLevel()) {
@@ -73,31 +73,25 @@ public class ExceptionUtils {
}
/**
- * A bare {@link TikaException} wrapper adds nothing to a report; returns its cause
- * instead. Subclasses and unwrapped throwables are returned as is.
- */
- public static Throwable unwrapTikaException(Throwable t) {
- if (t.getClass().equals(TikaException.class) && t.getCause() != null) {
- return t.getCause();
- }
- return t;
- }
-
- /**
- * @deprecated use {@link #format(Throwable, ParseContext)} on
- * {@link #unwrapTikaException(Throwable)} so the configured policy applies
+ * @deprecated since 4.1, removal planned for 5.0; use
+ * {@link #format(Throwable, ParseContext)} so the configured policy applies. Unlike this
+ * method, it keeps a bare {@link TikaException} wrapper.
*/
@Deprecated
public static String getFilteredStackTrace(Throwable t) {
- return format(unwrapTikaException(t), ExceptionReporting.DEFAULT);
+ //legacy semantics: a bare TikaException wrapper is stripped, subclasses are not
+ Throwable unwrapped = t.getClass().equals(TikaException.class) && t.getCause() != null ?
+ t.getCause() : t;
+ return format(unwrapped, new ExceptionReporting());
}
/**
- * @deprecated use {@link #format(Throwable, ParseContext)} so the configured policy applies
+ * @deprecated since 4.1, removal planned for 5.0; use
+ * {@link #format(Throwable, ParseContext)} so the configured policy applies
*/
@Deprecated
public static String getStackTrace(Throwable t) {
- return format(t, ExceptionReporting.DEFAULT);
+ return format(t, new ExceptionReporting());
}
private static String full(Throwable t) {
@@ -108,11 +102,16 @@ public class ExceptionUtils {
return result.toString();
}
- private static String truncate(String s, int maxLength) {
+ /**
+ * Truncates {@code s} to the policy's {@link ExceptionReporting#getMaxLength()} (no-op when
+ * unlimited), for channels that report exception text they did not format themselves.
+ */
+ public static String truncate(String s, int maxLength) {
if (maxLength < 0 || s.length() <= maxLength) {
return s;
}
int cut = maxLength;
+ //don't split a surrogate pair
if (Character.isHighSurrogate(s.charAt(cut - 1))) {
cut--;
}
@@ -153,11 +152,11 @@ public class ExceptionUtils {
.append(t.getClass().getName()).append("]\n");
return;
}
- seen.add(t);
if (depth > MAX_CAUSE_DEPTH) {
sb.append(prefix).append("... cause chain truncated\n");
return;
}
+ seen.add(t);
sb.append(prefix).append(caption).append(t.getClass().getName()).append('\n');
StackTraceElement[] trace = t.getStackTrace();
if (frames) {
diff --git a/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java b/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java
index 02a53ca622..8688af81b8 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java
@@ -90,8 +90,9 @@ public class ParserUtils {
}
/**
- * @deprecated use {@link #recordParserFailure(Parser, Throwable, Metadata, ParseContext)}
- * so the configured {@link org.apache.tika.config.ExceptionReporting} applies
+ * @deprecated since 4.1, removal planned for 5.0; use
+ * {@link #recordParserFailure(Parser, Throwable, Metadata, ParseContext)} so the
+ * configured {@link org.apache.tika.config.ExceptionReporting} applies
*/
@Deprecated
public static void recordParserFailure(Parser parser, Throwable failure, Metadata metadata) {
diff --git a/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingParseTest.java b/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingParseTest.java
index 60b127fa60..fdb297f896 100644
--- a/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingParseTest.java
+++ b/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingParseTest.java
@@ -58,7 +58,10 @@ public class ExceptionReportingParseTest extends TikaTest {
for (Level level : Level.values()) {
List<Metadata> list =
getRecursiveMetadata("embedded_then_npe.xml", context(level, -1), true);
- assertLevel(level, list.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION));
+ String trace = list.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
+ assertLevel(level, trace);
+ // the wrapper is kept: it names the failing parser and matches pipes output
+ assertTrue(trace.startsWith("org.apache.tika.exception.TikaException"), trace);
}
}
diff --git a/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java b/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingTest.java
similarity index 50%
copy from tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java
copy to tika-core/src/test/java/org/apache/tika/config/ExceptionReportingTest.java
index 3b60a535ea..def7ce1d54 100644
--- a/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java
+++ b/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingTest.java
@@ -17,29 +17,32 @@
package org.apache.tika.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
-import org.apache.tika.TikaTest;
-import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.parser.ParseContext;
-public class ExceptionReportingTest extends TikaTest {
+public class ExceptionReportingTest {
@Test
- public void testLoadFromConfig() throws Exception {
- TikaLoader loader = TikaLoader.load(getConfigPath(getClass(), "exception-reporting-test.json"));
- ParseContext context = loader.loadParseContext();
- ExceptionReporting reporting = ExceptionReporting.get(context);
- assertEquals(new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, 500), reporting);
+ public void testDefaults() {
+ ExceptionReporting defaults = new ExceptionReporting();
+ assertEquals(ExceptionReporting.Level.FULL, defaults.getLevel());
+ assertEquals(ExceptionReporting.UNLIMITED, defaults.getMaxLength());
+ assertEquals(defaults, ExceptionReporting.get(null));
+ assertEquals(defaults, ExceptionReporting.get(new ParseContext()));
+ // fresh instance each call
+ assertNotSame(ExceptionReporting.get(null), ExceptionReporting.get(null));
}
@Test
- public void testDefaults() {
- assertSame(ExceptionReporting.DEFAULT, ExceptionReporting.get(null));
- assertSame(ExceptionReporting.DEFAULT, ExceptionReporting.get(new ParseContext()));
- assertEquals(ExceptionReporting.Level.FULL, ExceptionReporting.DEFAULT.getLevel());
- assertEquals(ExceptionReporting.UNLIMITED, ExceptionReporting.DEFAULT.getMaxLength());
+ public void invalidMaxLength() {
+ assertThrows(IllegalArgumentException.class,
+ () -> new ExceptionReporting(ExceptionReporting.Level.FULL, 0));
+ assertThrows(IllegalArgumentException.class,
+ () -> new ExceptionReporting(ExceptionReporting.Level.FULL, -2));
+ assertThrows(NullPointerException.class, () -> new ExceptionReporting(null, -1));
}
}
diff --git a/tika-core/src/test/java/org/apache/tika/extractor/EmbeddedDocumentUtilTest.java b/tika-core/src/test/java/org/apache/tika/extractor/EmbeddedDocumentUtilTest.java
new file mode 100644
index 0000000000..916b62ad81
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/extractor/EmbeddedDocumentUtilTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.extractor;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.config.ExceptionReporting;
+import org.apache.tika.config.ExceptionReporting.Level;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * The configured policy must govern tk:exception:warn and tk:exception:embedded-stream-exception,
+ * the two keys parsers write through EmbeddedDocumentUtil.
+ */
+public class EmbeddedDocumentUtilTest {
+
+ private static final String MESSAGE = "inner secret";
+
+ private static Throwable wrapped() {
+ return new TikaException("Unexpected RuntimeException from a parser",
+ new IOException(MESSAGE));
+ }
+
+ private static ParseContext context(Level level) {
+ ParseContext context = new ParseContext();
+ context.set(ExceptionReporting.class,
+ new ExceptionReporting(level, ExceptionReporting.UNLIMITED));
+ return context;
+ }
+
+ private static void assertLevel(Level level, Metadata metadata, Property key) {
+ String recorded = metadata.get(key);
+ assertTrue(recorded.startsWith("org.apache.tika.exception.TikaException"), recorded);
+ assertTrue(recorded.contains("java.io.IOException"), recorded);
+ assertEquals(level == Level.FULL, recorded.contains(MESSAGE), level + ":\n" + recorded);
+ assertEquals(level != Level.REDACTED, recorded.contains("\tat "),
+ level + ":\n" + recorded);
+ }
+
+ @Test
+ public void recordExceptionHonorsPolicy() {
+ for (Level level : Level.values()) {
+ Metadata metadata = new Metadata();
+ EmbeddedDocumentUtil.recordException(wrapped(), metadata, context(level));
+ assertLevel(level, metadata, TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ }
+ }
+
+ @Test
+ public void recordEmbeddedStreamExceptionHonorsPolicy() {
+ for (Level level : Level.values()) {
+ Metadata metadata = new Metadata();
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(wrapped(), metadata,
+ context(level));
+ assertLevel(level, metadata,
+ TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM);
+ }
+ }
+
+ @Test
+ public void deprecatedFormsAreFullAndKeepTheWrapper() {
+ Metadata metadata = new Metadata();
+ EmbeddedDocumentUtil.recordException(wrapped(), metadata);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(wrapped(), metadata);
+ assertLevel(Level.FULL, metadata, TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ assertLevel(Level.FULL, metadata,
+ TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM);
+ }
+
+ @Test
+ public void maxLengthBoundsBothKeys() {
+ ParseContext context = new ParseContext();
+ context.set(ExceptionReporting.class, new ExceptionReporting(Level.FULL, 40));
+ Metadata metadata = new Metadata();
+ EmbeddedDocumentUtil.recordException(wrapped(), metadata, context);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(wrapped(), metadata, context);
+ for (Property key : new Property[]{TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM}) {
+ String recorded = metadata.get(key);
+ assertTrue(recorded.endsWith("...[truncated]"), recorded);
+ assertFalse(recorded.contains(MESSAGE), recorded);
+ }
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/utils/ExceptionUtilsTest.java b/tika-core/src/test/java/org/apache/tika/utils/ExceptionUtilsTest.java
index bd1674565b..11f98c6dcf 100644
--- a/tika-core/src/test/java/org/apache/tika/utils/ExceptionUtilsTest.java
+++ b/tika-core/src/test/java/org/apache/tika/utils/ExceptionUtilsTest.java
@@ -18,10 +18,11 @@ package org.apache.tika.utils;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import org.junit.jupiter.api.Test;
@@ -57,6 +58,11 @@ public class ExceptionUtilsTest {
public void fullMatchesPrintStackTrace() {
Throwable t = chain();
String full = format(t, Level.FULL);
+ StringWriter expected = new StringWriter();
+ try (PrintWriter writer = new PrintWriter(expected)) {
+ t.printStackTrace(writer);
+ }
+ assertEquals(expected.toString(), full);
for (String m : MESSAGES) {
assertTrue(full.contains(m));
}
@@ -73,8 +79,9 @@ public class ExceptionUtilsTest {
assertTrue(redacted.contains("Caused by: java.io.IOException\n"));
assertTrue(redacted.contains("\tSuppressed: java.lang.RuntimeException\n"));
assertTrue(redacted.contains(" more\n"), "common-frame elision kept");
- // Strip ": message" from every header line of FULL; the rest must be identical.
- String expected = full.replaceAll("(?m)^((?:\\t*Suppressed: |Caused by: )?[\\w.$]+Exception): .*$", "$1");
+ // Strip ": message" from every header line of FULL (frame lines start with a tab,
+ // so they cannot match); the rest must be identical.
+ String expected = full.replaceAll("(?m)^((?:\\t*Suppressed: |Caused by: )?[\\w.$]+): .*$", "$1");
expected = expected.replace("\r", ""); // windows
assertEquals(expected, redacted);
}
@@ -109,7 +116,8 @@ public class ExceptionUtilsTest {
}
String s = format(t, Level.REDACTED);
assertTrue(s.contains("cause chain truncated"));
- assertTrue(s.split("\n").length < 100);
+ long causes = s.lines().filter(l -> l.startsWith("Caused by: ")).count();
+ assertEquals(64, causes, "cause chain must stop at MAX_CAUSE_DEPTH");
}
@Test
@@ -131,35 +139,33 @@ public class ExceptionUtilsTest {
}
@Test
- public void nullContextAndPolicyAreFull() {
+ public void nullContextIsFull() {
Throwable t = chain();
String full = format(t, Level.FULL);
assertEquals(full, ExceptionUtils.format(t, (ParseContext) null));
- assertEquals(full, ExceptionUtils.format(t, (ExceptionReporting) null));
assertEquals(full, ExceptionUtils.format(t, new ParseContext()));
}
@Test
- public void contextPolicyApplies() {
- ParseContext context = new ParseContext();
- context.set(ExceptionReporting.class, new ExceptionReporting(Level.REDACTED, -1));
- assertNoMessages(ExceptionUtils.format(chain(), context));
+ public void nullReportingIsFull() {
+ // formatting someone else's exception must never NPE
+ Throwable t = chain();
+ assertEquals(format(t, Level.FULL),
+ ExceptionUtils.format(t, (ExceptionReporting) null));
}
@Test
- public void unwrapTikaException() {
+ public void getFilteredStackTraceStillUnwrapsBareTikaException() {
Throwable t = chain();
- assertEquals(t.getCause(), ExceptionUtils.unwrapTikaException(t));
- Throwable sub = new org.apache.tika.exception.EncryptedDocumentException(t);
- assertEquals(sub, ExceptionUtils.unwrapTikaException(sub));
- Throwable bare = new TikaException("no cause");
- assertEquals(bare, ExceptionUtils.unwrapTikaException(bare));
+ assertEquals(ExceptionUtils.format(t.getCause(), (ParseContext) null),
+ ExceptionUtils.getFilteredStackTrace(t));
}
@Test
- public void invalidMaxLength() {
- assertThrows(IllegalArgumentException.class, () -> new ExceptionReporting(Level.FULL, 0));
- assertThrows(IllegalArgumentException.class, () -> new ExceptionReporting(Level.FULL, -2));
- assertThrows(NullPointerException.class, () -> new ExceptionReporting(null, -1));
+ public void contextPolicyApplies() {
+ ParseContext context = new ParseContext();
+ context.set(ExceptionReporting.class, new ExceptionReporting(Level.REDACTED, -1));
+ assertNoMessages(ExceptionUtils.format(chain(), context));
}
+
}
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/util/EvalExceptionUtils.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/util/EvalExceptionUtils.java
index 488e8e668b..8ee8a7a1e5 100644
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/util/EvalExceptionUtils.java
+++ b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/util/EvalExceptionUtils.java
@@ -25,8 +25,9 @@ public class EvalExceptionUtils {
//these remove runtime info from the stacktraces so
//that actual causes can be counted.
+ //\t*: a suppressed exception's cause is tab-indented by printStackTrace
private final static Pattern CAUSED_BY_SNIPPER =
- Pattern.compile("(Caused by: [^:]+):[^\\r\\n]+");
+ Pattern.compile("(?m)^(\\t*Caused by: [^:\\r\\n]+):[^\\r\\n]+");
//strips the exception message off the first line, so that the same cause with
//differing runtime detail collapses to one entry. Anchored to the first line: a
@@ -46,10 +47,6 @@ public class EvalExceptionUtils {
}
private static String trimMessage(String trace) {
- Matcher msgMatcher = MSG_PATTERN.matcher(trace);
- if (msgMatcher.find()) {
- return msgMatcher.replaceFirst("$1");
- }
- return trace;
+ return MSG_PATTERN.matcher(trace).replaceFirst("$1");
}
}
diff --git a/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/util/EvalExceptionUtilsTest.java b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/util/EvalExceptionUtilsTest.java
index 6e0ec6ca93..16dcb3cf9d 100644
--- a/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/util/EvalExceptionUtilsTest.java
+++ b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/util/EvalExceptionUtilsTest.java
@@ -36,10 +36,29 @@ public class EvalExceptionUtilsTest {
@Test
public void messageRedactedTraceNormalizesToSameKey() {
+ // The redacted "Caused by:" line has no colon; the snipper must not run past the
+ // newline into the following frame.
String full = "org.apache.tika.exception.TikaException: secret\n"
- + "\tat org.apache.tika.Foo.bar(Foo.java:12)\n";
+ + "\tat org.apache.tika.Foo.bar(Foo.java:12)\n"
+ + "Caused by: java.io.IOException: other secret\n"
+ + "\tat org.apache.tika.Foo.baz(Foo.java:34)\n";
String redacted = "org.apache.tika.exception.TikaException\n"
- + "\tat org.apache.tika.Foo.bar(Foo.java:12)\n";
+ + "\tat org.apache.tika.Foo.bar(Foo.java:12)\n"
+ + "Caused by: java.io.IOException\n"
+ + "\tat org.apache.tika.Foo.baz(Foo.java:34)\n";
assertEquals(EvalExceptionUtils.normalize(full), EvalExceptionUtils.normalize(redacted));
}
+
+ @Test
+ public void suppressedCauseMessageIsSnipped() {
+ // printStackTrace tab-indents a suppressed exception's cause; traces differing
+ // only in that runtime detail must normalize to the same key.
+ String a = "org.apache.tika.exception.TikaException: top\n"
+ + "\tSuppressed: java.io.IOException: close failed\n"
+ + "\t\tat org.apache.tika.Foo.close(Foo.java:56)\n"
+ + "\tCaused by: java.nio.file.NoSuchFileException: /tmp/spool-123.tmp\n"
+ + "\tat org.apache.tika.Foo.bar(Foo.java:12)\n";
+ String b = a.replace("/tmp/spool-123.tmp", "/tmp/spool-456.tmp");
+ assertEquals(EvalExceptionUtils.normalize(a), EvalExceptionUtils.normalize(b));
+ }
}
diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
index 9608c2d504..d8682b1c61 100644
--- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
+++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java
@@ -77,6 +77,9 @@ import org.apache.tika.pipes.grpc.proto.SavePipesIteratorRequest;
import org.apache.tika.pipes.grpc.proto.TikaGrpc;
import org.apache.tika.plugins.ExtensionConfig;
import org.apache.tika.plugins.TikaPluginManager;
+import org.apache.tika.serialization.ComponentNameResolver;
+import org.apache.tika.serialization.ParseContextUtils;
+import org.apache.tika.serialization.serdes.ParseContextDeserializer;
class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
private static final Logger LOG = LoggerFactory.getLogger(TikaGrpcServerImpl.class);
@@ -274,7 +277,11 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
if (denyPerRequestConfig(request, responseObserver)) {
return;
}
- fetchAndParseImpl(request, responseObserver);
+ ParseContext parseContext = buildRequestParseContext(request, responseObserver);
+ if (parseContext == null) {
+ return;
+ }
+ fetchAndParseImpl(request, parseContext, responseObserver);
responseObserver.onCompleted();
}
@@ -287,7 +294,12 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
if (denyPerRequestConfig(fetchAndParseRequest, responseObserver)) {
return;
}
- fetchAndParseImpl(fetchAndParseRequest, responseObserver);
+ ParseContext parseContext =
+ buildRequestParseContext(fetchAndParseRequest, responseObserver);
+ if (parseContext == null) {
+ return;
+ }
+ fetchAndParseImpl(fetchAndParseRequest, parseContext, responseObserver);
}
@Override
@@ -308,11 +320,65 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
if (denyPerRequestConfig(request, responseObserver)) {
return;
}
- fetchAndParseImpl(request, responseObserver);
+ ParseContext parseContext = buildRequestParseContext(request, responseObserver);
+ if (parseContext == null) {
+ return;
+ }
+ fetchAndParseImpl(request, parseContext, responseObserver);
responseObserver.onCompleted();
}
- private void fetchAndParseImpl(FetchAndParseRequest request,
+ /**
+ * Builds the per-request {@link ParseContext} from {@code parse_context_json}, or closes the
+ * call with {@code INVALID_ARGUMENT} and returns {@code null}. The caller must {@code return}
+ * immediately on {@code null}.
+ * <p>
+ * The request is untrusted wire input, so it is deserialized restricted and resolved here, as
+ * the tika-server resources do. Left to the fork, the same refusal (of
+ * {@code exception-reporting}, say) is a deserialization failure that exits it: a JVM restart
+ * per bad request, and a crash status with no reason for the caller. The trade-off, accepted
+ * deliberately: a component registered only on the fork's classpath is refused here too --
+ * fail-fast validation runs against this JVM's registries. Serialization still forwards the
+ * caller's original jsonConfigs, not the instances resolution materializes.
+ */
+ private ParseContext buildRequestParseContext(FetchAndParseRequest request,
+ StreamObserver<?> responseObserver) {
+ ParseContext parseContext = new ParseContext();
+ String parseContextJson = request.getParseContextJson();
+ if (StringUtils.isNotBlank(parseContextJson)) {
+ try {
+ parseContext = ParseContextDeserializer.readParseContext(
+ OBJECT_MAPPER.readTree(parseContextJson), true);
+ ParseContextUtils.resolveAll(parseContext, getClass().getClassLoader());
+ } catch (IOException | TikaConfigException e) {
+ responseObserver.onError(io.grpc.Status.INVALID_ARGUMENT
+ .withDescription("Invalid parse_context_json: " + e.getMessage())
+ .asRuntimeException());
+ return null;
+ }
+ }
+ String additionalFetchConfigJson = request.getAdditionalFetchConfigJson();
+ if (StringUtils.isNotBlank(additionalFetchConfigJson)) {
+ // The fork reads this jsonConfig by the fetcher's registered component name
+ // (e.g. "http-fetcher"). An unregistered key fails the fork's resolveAll and a
+ // wire-blocked one is refused by its restricted tuple deserialization -- both
+ // exit the worker. Enforce here: a 400, not a JVM restart per request.
+ var info = ComponentNameResolver.getComponentInfo(request.getFetcherId());
+ if (info.isEmpty() || ComponentNameResolver.isWireBlocked(
+ ComponentNameResolver.determineContextKey(info.get()))) {
+ responseObserver.onError(io.grpc.Status.INVALID_ARGUMENT
+ .withDescription("additional_fetch_config_json requires fetcher_id to "
+ + "be a registered, wire-allowed component name; got '"
+ + request.getFetcherId() + "'")
+ .asRuntimeException());
+ return null;
+ }
+ parseContext.setJsonConfig(request.getFetcherId(), additionalFetchConfigJson);
+ }
+ return parseContext;
+ }
+
+ private void fetchAndParseImpl(FetchAndParseRequest request, ParseContext parseContext,
StreamObserver<FetchAndParseReply> responseObserver) {
Fetcher fetcher;
try {
@@ -323,17 +389,6 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
Metadata tikaMetadata = new Metadata();
try {
- ParseContext parseContext = new ParseContext();
- String additionalFetchConfigJson = request.getAdditionalFetchConfigJson();
- if (StringUtils.isNotBlank(additionalFetchConfigJson)) {
- parseContext.setJsonConfig(request.getFetcherId(), additionalFetchConfigJson);
- }
- String parseContextJson = request.getParseContextJson();
- if (StringUtils.isNotBlank(parseContextJson)) {
- com.fasterxml.jackson.databind.JsonNode contextNode = OBJECT_MAPPER.readTree(parseContextJson);
- contextNode.fields().forEachRemaining(entry ->
- parseContext.setJsonConfig(entry.getKey(), entry.getValue().toString()));
- }
PipesResult pipesResult = pipesParser.parse(new FetchEmitTuple(request.getFetchKey(), new FetchKey(fetcher.getExtensionConfig().id(), request.getFetchKey()),
new EmitKey(), tikaMetadata, parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
FetchAndParseReply.Builder fetchReplyBuilder =
diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java
index b352729f78..237c0c331b 100644
--- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java
+++ b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java
@@ -62,6 +62,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.pipes.api.PipesResult;
import org.apache.tika.pipes.grpc.proto.DeleteFetcherReply;
import org.apache.tika.pipes.grpc.proto.DeleteFetcherRequest;
@@ -269,6 +270,97 @@ public class TikaGrpcServerTest {
assertEquals(Status.Code.PERMISSION_DENIED, ex.getStatus().getCode());
}
+ @Test
+ public void testWireBlockedParseContextIsRejected(Resources resources) throws Exception {
+ // exception-reporting is operator policy, not a request knob. Ungated, the fork refuses it
+ // at deserialization time and exits: UNSPECIFIED_CRASH with no reason, plus a restart.
+ TikaGrpc.TikaBlockingStub blockingStub = startServer(resources, tikaConfigUnlocked);
+
+ StatusRuntimeException ex = Assertions.assertThrows(StatusRuntimeException.class, () ->
+ blockingStub.fetchAndParse(FetchAndParseRequest
+ .newBuilder()
+ .setFetcherId(createFetcherId(1))
+ .setFetchKey("no-such-file")
+ .setParseContextJson("{\"exception-reporting\":{\"level\":\"FULL\"}}")
+ .build()));
+ assertEquals(Status.Code.INVALID_ARGUMENT, ex.getStatus().getCode());
+ assertTrue(String.valueOf(ex.getStatus().getDescription())
+ .contains("may not be supplied via a request parseContext"),
+ "the caller must be told why: " + ex.getStatus().getDescription());
+
+ // ...and the server must still serve the next request.
+ String fetchKey = "wire-blocked-" + UUID.randomUUID() + ".html";
+ File testFile = new File("target", fetchKey);
+ FileUtils.writeStringToFile(testFile,
+ "<html><body>still serving</body></html>", StandardCharsets.UTF_8);
+ try {
+ FetchAndParseReply reply = blockingStub.fetchAndParse(FetchAndParseRequest
+ .newBuilder()
+ .setFetcherId(createFetcherId(1))
+ .setFetchKey(fetchKey)
+ .build());
+ assertEquals(PipesResult.RESULT_STATUS.PARSE_SUCCESS.name(), reply.getStatus());
+ } finally {
+ FileUtils.deleteQuietly(testFile);
+ }
+ }
+
+ @Test
+ public void testPerRequestParseContextStillReachesTheFork(Resources resources)
+ throws Exception {
+ // The gate resolves the request context in this JVM; a legal entry must still round-trip
+ // to the worker. Both shapes: one resolved here, one (self-configuring) left as JSON.
+ TikaGrpc.TikaBlockingStub blockingStub = startServer(resources, tikaConfigUnlocked);
+ String fetchKey = "per-request-" + UUID.randomUUID() + ".html";
+ File testFile = new File("target", fetchKey);
+ FileUtils.writeStringToFile(testFile,
+ "<html><body>per request</body></html>", StandardCharsets.UTF_8);
+ try {
+ FetchAndParseReply reply = blockingStub.fetchAndParse(FetchAndParseRequest
+ .newBuilder()
+ .setFetcherId(createFetcherId(1))
+ .setFetchKey(fetchKey)
+ .setParseContextJson("{\"basic-content-handler-factory\":{\"type\":\"IGNORE\"},"
+ + "\"pdf-parser\":{\"sortByPosition\":true}}")
+ .build());
+ assertEquals(PipesResult.RESULT_STATUS.PARSE_SUCCESS.name(), reply.getStatus());
+ String contentKey = TikaCoreProperties.TIKA_CONTENT.getName();
+ String ignored = reply.getFieldsMap().get(contentKey);
+ assertTrue(ignored == null || ignored.isBlank(),
+ "type=IGNORE must reach the worker: " + reply.getFieldsMap());
+
+ FetchAndParseReply withContent = blockingStub.fetchAndParse(FetchAndParseRequest
+ .newBuilder()
+ .setFetcherId(createFetcherId(1))
+ .setFetchKey(fetchKey)
+ .build());
+ assertEquals(PipesResult.RESULT_STATUS.PARSE_SUCCESS.name(), withContent.getStatus());
+ String content = withContent.getFieldsMap().get(contentKey);
+ assertTrue(content != null && !content.isBlank(),
+ "without the override there must be content, or the check above proves "
+ + "nothing: " + withContent.getFieldsMap());
+ } finally {
+ FileUtils.deleteQuietly(testFile);
+ }
+ }
+
+ @Test
+ public void testUnrecognizedParseContextEntryIsRejected(Resources resources) throws Exception {
+ // Same boundary, other trigger: an unregistered name fails resolveAll in the fork.
+ TikaGrpc.TikaBlockingStub blockingStub = startServer(resources, tikaConfigUnlocked);
+
+ StatusRuntimeException ex = Assertions.assertThrows(StatusRuntimeException.class, () ->
+ blockingStub.fetchAndParse(FetchAndParseRequest
+ .newBuilder()
+ .setFetcherId(createFetcherId(1))
+ .setFetchKey("no-such-file")
+ .setParseContextJson("{\"no-such-component\":{}}")
+ .build()));
+ assertEquals(Status.Code.INVALID_ARGUMENT, ex.getStatus().getCode());
+ assertTrue(String.valueOf(ex.getStatus().getDescription()).contains("no-such-component"),
+ "the caller must be told which entry: " + ex.getStatus().getDescription());
+ }
+
@Test
public void testComponentManagementHidesConfigByDefault(Resources resources) throws Exception {
// With component management off (the default), the read RPCs must return component identity
diff --git a/tika-parent/forbidden-apis-exception-reporting-bans.txt b/tika-parent/forbidden-apis-exception-reporting-bans.txt
new file mode 100644
index 0000000000..cf630d45c2
--- /dev/null
+++ b/tika-parent/forbidden-apis-exception-reporting-bans.txt
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TIKA-4848: the ExceptionReporting policy reaches a parser only through the ParseContext, so
+# the pre-4.1 overloads that take no context silently report FULL. They stay for compatibility;
+# these bans keep in-tree code off them. Default-on for main sources in every module
+# (see the forbiddenapis-check execution in tika-parent); the deprecated overloads' own
+# bodies delegate to the context overloads, so tika-core passes too.
+
+org.apache.tika.utils.ExceptionUtils#getStackTrace(java.lang.Throwable) @ use format(Throwable, ParseContext) so the exception-reporting policy applies
+org.apache.tika.utils.ExceptionUtils#getFilteredStackTrace(java.lang.Throwable) @ use format(Throwable, ParseContext) so the exception-reporting policy applies
+org.apache.tika.extractor.EmbeddedDocumentUtil#recordException(java.lang.Throwable,org.apache.tika.metadata.Metadata) @ use the ParseContext overload so the exception-reporting policy applies
+org.apache.tika.extractor.EmbeddedDocumentUtil#recordEmbeddedStreamException(java.lang.Throwable,org.apache.tika.metadata.Metadata) @ use the ParseContext overload so the exception-reporting policy applies
+org.apache.tika.utils.ParserUtils#recordParserFailure(org.apache.tika.parser.Parser,java.lang.Throwable,org.apache.tika.metadata.Metadata) @ use the ParseContext overload so the exception-reporting policy applies
diff --git a/tika-parent/forbidden-apis-none.txt b/tika-parent/forbidden-apis-none.txt
new file mode 100644
index 0000000000..e7e99ec896
--- /dev/null
+++ b/tika-parent/forbidden-apis-none.txt
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Intentionally empty: default value for forbiddenapis.module.signaturesFile.
+# Modules override that property to layer extra main-source bans
+# (e.g. ${metadata.forbiddenapis.signaturesFile}); see tika-parent/pom.xml.
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index e6b098167c..1e38a2a9cc 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -314,9 +314,16 @@
<spotless.goal>apply</spotless.goal>
<spotless.header.file>${maven.multiModuleProjectDirectory}/tika-parent/license-header.txt</spotless.header.file>
<!-- TIKA-4816: Metadata#set/add(String,String) etc. ban for migrated parser modules;
- opted into per-module via a "forbiddenapis-check" execution override (main sources
- only: see that execution's id in the forbiddenapis plugin below). -->
+ opted into per-module by overriding forbiddenapis.module.signaturesFile (main
+ sources only). -->
<metadata.forbiddenapis.signaturesFile>${maven.multiModuleProjectDirectory}/tika-parent/forbidden-apis-metadata-string-key-bans.txt</metadata.forbiddenapis.signaturesFile>
+ <!-- TIKA-4848: default-on in every module's main sources (see forbiddenapis-check below) -->
+ <exception.forbiddenapis.signaturesFile>${maven.multiModuleProjectDirectory}/tika-parent/forbidden-apis-exception-reporting-bans.txt</exception.forbiddenapis.signaturesFile>
+ <!-- Per-module extra main-source bans; a module overrides this (e.g. to
+ ${metadata.forbiddenapis.signaturesFile}) instead of overriding the
+ forbiddenapis-check execution, whose child list would REPLACE the parent's
+ and silently drop the default-on file. -->
+ <forbiddenapis.module.signaturesFile>${maven.multiModuleProjectDirectory}/tika-parent/forbidden-apis-none.txt</forbiddenapis.module.signaturesFile>
<!-- dependency versions -->
<!-- change threetenbp exclusion version -->
<google.cloud.version>2.72.0</google.cloud.version>
@@ -1449,14 +1456,21 @@
</configuration>
<executions>
<!-- Split into id'd executions (was one execution running both goals) so that
- individual modules can layer additional main-only signaturesFiles onto
- "forbiddenapis-check" (matching execution id merges config) without also
- applying them to test sources via "forbiddenapis-testcheck". -->
+ main-only signaturesFiles below don't also apply to test sources via
+ "forbiddenapis-testcheck". Modules layer extra main-only bans by overriding
+ the forbiddenapis.module.signaturesFile property, NOT this execution: a
+ child execution's <signaturesFiles> would replace this list wholesale. -->
<execution>
<id>forbiddenapis-check</id>
<goals>
<goal>check</goal>
</goals>
+ <configuration>
+ <signaturesFiles>
+ <signaturesFile>${exception.forbiddenapis.signaturesFile}</signaturesFile>
+ <signaturesFile>${forbiddenapis.module.signaturesFile}</signaturesFile>
+ </signaturesFiles>
+ </configuration>
</execution>
<execution>
<id>forbiddenapis-testcheck</id>
@@ -1783,6 +1797,8 @@
<spotless.header.file>${session.executionRootDirectory}/tika-parent/license-header.txt</spotless.header.file>
<checkstyle.configLocation>${session.executionRootDirectory}/tika-parent/checkstyle.xml</checkstyle.configLocation>
<metadata.forbiddenapis.signaturesFile>${session.executionRootDirectory}/tika-parent/forbidden-apis-metadata-string-key-bans.txt</metadata.forbiddenapis.signaturesFile>
+ <exception.forbiddenapis.signaturesFile>${session.executionRootDirectory}/tika-parent/forbidden-apis-exception-reporting-bans.txt</exception.forbiddenapis.signaturesFile>
+ <forbiddenapis.module.signaturesFile>${session.executionRootDirectory}/tika-parent/forbidden-apis-none.txt</forbiddenapis.module.signaturesFile>
</properties>
</profile>
<!-- Fast profile: ./mvnw install -Pfast -T1C -->
diff --git a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/pom.xml b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/pom.xml
index afdb91a56f..9aeb79e0bf 100644
--- a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/pom.xml
+++ b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/pom.xml
@@ -98,6 +98,11 @@ May the gods of dependency management fix this in the future.
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<!-- because of protobuf 3.25.6. The actual problem is in netcdf4. Might be a false alarm,
@@ -139,21 +144,6 @@ May the gods of dependency management fix this in the future.
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/pom.xml b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/pom.xml
index d016e26a97..7b80e6955a 100644
--- a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/pom.xml
+++ b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/pom.xml
@@ -108,7 +108,12 @@
</dependency>
</dependencies>
- <build>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
+ <build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
@@ -120,21 +125,6 @@
</inputExcludes>
</configuration>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
<profiles>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/crypto/TSDParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/crypto/TSDParserTest.java
index 4af3e885dc..ce16c22781 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/crypto/TSDParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/crypto/TSDParserTest.java
@@ -44,10 +44,10 @@ public class TSDParserTest extends TikaTest {
List<Metadata> list = getRecursiveMetadata("testTSD_broken_pdf.tsd", parseContext);
assertEquals(2, list.size());
assertEquals("application/pdf", list.get(1).get(HttpHeaders.CONTENT_TYPE));
- assertNotNull(list.get(1).get(TikaCoreProperties.EMBEDDED_EXCEPTION));
+ assertNotNull(list.get(1).get(TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM));
// Exception occurs during TSD content extraction (truncated file)
assertContains("EOFException",
- list.get(1).get(TikaCoreProperties.EMBEDDED_EXCEPTION));
+ list.get(1).get(TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM));
}
@Test
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/pom.xml
index 3b23c6d806..dae9b77fc4 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/pom.xml
@@ -41,6 +41,11 @@
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -61,21 +66,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/pom.xml
index 4c7bad5327..bcd905aa18 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/pom.xml
@@ -41,6 +41,11 @@
<version>${vorbis.version}</version>
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -61,21 +66,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/MP4Parser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/MP4Parser.java
index c1b7a3877e..46cd509b28 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/MP4Parser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/MP4Parser.java
@@ -41,6 +41,7 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
+import org.apache.tika.config.ExceptionReporting;
import org.apache.tika.exception.RuntimeSAXException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
@@ -56,6 +57,7 @@ import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.XHTMLContentHandler;
+import org.apache.tika.utils.ExceptionUtils;
import org.apache.tika.utils.StringUtils;
/**
@@ -134,7 +136,7 @@ public class MP4Parser implements Parser {
//the input holds is skipped rather than allocated
long inputLength = tis.hasLength() ? tis.getLength() : -1;
try {
- TikaMp4Reader.extract(tis, boxHandler, maxBoxSize, inputLength);
+ TikaMp4Reader.extract(tis, boxHandler, maxBoxSize, inputLength, metadata, context);
} catch (RuntimeSAXException e) {
throw (SAXException) e.getCause();
}
@@ -149,8 +151,13 @@ public class MP4Parser implements Parser {
metadata.set(HttpHeaders.CONTENT_TYPE, AUDIO_MP4.toString());
}
- for (String m : errorMessages) {
- metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, m);
+ //the library's error strings quote document content: FULL-only, nothing survives redaction
+ ExceptionReporting reporting = ExceptionReporting.get(context);
+ if (reporting.getLevel() == ExceptionReporting.Level.FULL) {
+ for (String m : errorMessages) {
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.truncate(m, reporting.getMaxLength()));
+ }
}
xhtml.endDocument();
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4Reader.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4Reader.java
index 05bbac913c..df31145868 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4Reader.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4Reader.java
@@ -25,6 +25,10 @@ import com.drew.metadata.mp4.Mp4BoxHandler;
import com.drew.metadata.mp4.Mp4Context;
import com.drew.metadata.mp4.Mp4MediaHandler;
+import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
/**
* A size-bounded reimplementation of com.drew.imaging.mp4.Mp4Reader.
* <p>
@@ -55,15 +59,16 @@ final class TikaMp4Reader {
* than allocated (StreamReader.getBytes allocates before reading).
*/
static void extract(InputStream inputStream, Mp4BoxHandler handler, long maxBoxSize,
- long inputLength) {
+ long inputLength, Metadata metadata, ParseContext parseContext) {
StreamReader reader = new StreamReader(inputStream);
reader.setMotorolaByteOrder(true);
- processBoxes(reader, -1, handler, new Mp4Context(), maxBoxSize, inputLength, 0);
+ processBoxes(reader, -1, handler, new Mp4Context(), maxBoxSize, inputLength, 0, metadata,
+ parseContext);
}
private static void processBoxes(StreamReader reader, long atomEnd, Mp4Handler<?> handler,
Mp4Context context, long maxBoxSize, long inputLength,
- int depth) {
+ int depth, Metadata metadata, ParseContext parseContext) {
if (depth > MAX_BOX_DEPTH) {
handler.addError("MP4 box nesting exceeds the maximum depth of " + MAX_BOX_DEPTH);
return;
@@ -90,7 +95,7 @@ final class TikaMp4Reader {
if (acceptContainer(handler, boxType)) {
processBoxes(reader, reader.getPosition() + payloadLength,
processBox(handler, boxType, null, boxSize, context), context,
- maxBoxSize, inputLength, depth + 1);
+ maxBoxSize, inputLength, depth + 1, metadata, parseContext);
} else if (acceptBox(handler, boxType)) {
//StreamReader.getBytes allocates the whole payload up front, so skip
//(a lazy stream advance) any box over the cap, or one that claims more
@@ -116,8 +121,8 @@ final class TikaMp4Reader {
}
}
} catch (IOException e) {
- handler.addError(e.getMessage() == null ? "IOException reading MP4 boxes"
- : e.getMessage());
+ //not handler.addError: the library's error strings bypass the reporting policy
+ EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
index c1b663ff2d..11fbf1bf57 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
@@ -46,6 +46,7 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.TikaTest;
+import org.apache.tika.config.ExceptionReporting;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Audio;
import org.apache.tika.metadata.HttpHeaders;
@@ -260,6 +261,28 @@ public class MP4ParserTest extends TikaTest {
assertEquals("M4A", r.metadata.get(XMPDM.AUDIO_COMPRESSOR));
}
+ // TIKA-4848: the box walk's own IOException is reported through the policy, not as a
+ // bare message copied out of the library's error list
+ @Test
+ public void testTruncatedStreamWarningIsFormatted() throws Exception {
+ String[] warnings = getXML("testMP4_truncated.m4a", new ParseContext()).metadata
+ .getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ assertTrue(Arrays.stream(warnings)
+ .anyMatch(w -> w.startsWith("java.io.") && w.contains("\tat ")),
+ Arrays.toString(warnings));
+
+ ParseContext redacted = new ParseContext();
+ redacted.set(ExceptionReporting.class, new ExceptionReporting(
+ ExceptionReporting.Level.REDACTED, ExceptionReporting.UNLIMITED));
+ String[] redactedWarnings = getXML("testMP4_truncated.m4a", redacted).metadata
+ .getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ assertTrue(Arrays.stream(redactedWarnings).anyMatch(w -> w.startsWith("java.io.")),
+ Arrays.toString(redactedWarnings));
+ for (String w : redactedWarnings) {
+ assertFalse(w.contains("\tat "), w);
+ }
+ }
+
@Test
public void testAudioOnlyMP4() throws Exception {
final XMLResult xmlResult = getXML("testMP4AudioOnly.mp4");
@@ -586,7 +609,7 @@ public class MP4ParserTest extends TikaTest {
//must return without a StackOverflowError
assertDoesNotThrow(() ->
TikaMp4Reader.extract(new ByteArrayInputStream(boxes), handler, 1000L,
- boxes.length));
+ boxes.length, new Metadata(), new ParseContext()));
}
@Test
@@ -617,7 +640,8 @@ public class MP4ParserTest extends TikaTest {
XHTMLContentHandler xhtml = new XHTMLContentHandler(new DefaultHandler(), tikaMetadata);
TikaMp4BoxHandler handler =
new TikaMp4BoxHandler(mp4Metadata, tikaMetadata, xhtml, new ParseContext());
- TikaMp4Reader.extract(new ByteArrayInputStream(boxes), handler, maxBoxSize, boxes.length);
+ TikaMp4Reader.extract(new ByteArrayInputStream(boxes), handler, maxBoxSize, boxes.length,
+ tikaMetadata, new ParseContext());
Mp4Directory dir = mp4Metadata.getFirstDirectoryOfType(Mp4Directory.class);
return dir == null ? null : dir.getString(Mp4Directory.TAG_MAJOR_BRAND);
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/pom.xml
index 7cef8879fd..7025117ea8 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/pom.xml
@@ -45,6 +45,11 @@
<artifactId>jackson-databind</artifactId>
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -65,21 +70,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
<scm>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-crypto-module/src/main/java/org/apache/tika/parser/crypto/TSDParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-crypto-module/src/main/java/org/apache/tika/parser/crypto/TSDParser.java
index 5cd59c7176..5a06161e8b 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-crypto-module/src/main/java/org/apache/tika/parser/crypto/TSDParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-crypto-module/src/main/java/org/apache/tika/parser/crypto/TSDParser.java
@@ -62,12 +62,10 @@ import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.XHTMLContentHandler;
-import org.apache.tika.utils.ExceptionUtils;
/**
* Tika parser for Time Stamped Data Envelope (application/timestamped-data)
@@ -190,7 +188,7 @@ public class TSDParser implements Parser {
}
} catch (IOException e) {
// Truncated file - record exception and work with what we got
- metadata.set(TikaCoreProperties.EMBEDDED_EXCEPTION, ExceptionUtils.format(e, context));
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata, context);
LOG.debug("Error reading TSD content (possibly truncated)", e);
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/pom.xml
index ffa1772ec5..765052569a 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/pom.xml
@@ -38,6 +38,11 @@
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -67,21 +72,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/pom.xml
index 50ac202fee..2c29ef4557 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/pom.xml
@@ -70,6 +70,11 @@
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -99,21 +104,6 @@
</inputExcludes>
</configuration>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/pom.xml
index e27b2f1da7..cb07b139ff 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/pom.xml
@@ -83,6 +83,11 @@
<scope>test</scope>
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<!-- needs an entry or it will fail?! -->
@@ -113,21 +118,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
index af5412b9a9..edcdd7be9f 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
@@ -142,6 +142,11 @@
<scope>test</scope>
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -195,21 +200,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java
index af7bf9cd14..197a920bb3 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java
@@ -431,7 +431,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor {
EmbeddedDocumentUtil.recordException(inner, parentMetadata, context);
}
} catch (IOException e) {
- EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);//swallow
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
}
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessExtractor.java
index 6063ec1450..b243a5ea39 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessExtractor.java
@@ -76,13 +76,11 @@ class JackcessExtractor extends AbstractPOIFSExtractor {
final DateFormat shortDateTimeFormatter;
final Parser htmlParser;
- final ParseContext parseContext;
protected JackcessExtractor(Metadata metadata, ParseContext context, Locale locale) {
super(context, metadata);
currencyFormatter = NumberFormat.getCurrencyInstance(locale);
shortDateTimeFormatter = DateFormat.getDateInstance(DateFormat.SHORT, locale);
- this.parseContext = context;
Parser tmpHtmlParser =
EmbeddedDocumentUtil.tryToFindExistingLeafParser(JSoupParser.class, context);
if (tmpHtmlParser == null) {
@@ -217,10 +215,10 @@ class JackcessExtractor extends AbstractPOIFSExtractor {
String v = toString(obj, c.getType());
if (isRichText(c)) {
BodyContentHandler h = new BodyContentHandler();
- Metadata m = Metadata.newInstance(parseContext);
+ Metadata m = Metadata.newInstance(context);
m.set(HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8");
try (TikaInputStream tis = TikaInputStream.get(v.getBytes(UTF_8))) {
- htmlParser.parse(tis, h, m, parseContext);
+ htmlParser.parse(tis, h, m, context);
handler.characters(h.toString());
} catch (SAXException e) {
WriteLimitReachedException.throwIfWriteLimitReached(e);
@@ -327,7 +325,7 @@ class JackcessExtractor extends AbstractPOIFSExtractor {
try {
tis = TikaInputStream.get(spc.getStream());
} catch (IOException e) {
- EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, parseContext);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
break;
}
if (tis != null) {
@@ -347,7 +345,7 @@ class JackcessExtractor extends AbstractPOIFSExtractor {
try {
ocStream = TikaInputStream.get(oc.getStream());
} catch (IOException e) {
- EmbeddedDocumentUtil.recordException(e, parentMetadata, parseContext);
+ EmbeddedDocumentUtil.recordException(e, parentMetadata, context);
}
try {
handleEmbeddedResource(ocStream, null,//filename
@@ -373,14 +371,14 @@ class JackcessExtractor extends AbstractPOIFSExtractor {
try {
is = cc.getStream();
} catch (IOException e) {
- EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, parseContext);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
return;
}
try {
fileSystem = new POIFSFileSystem(is);
} catch (Exception e) {
- EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, parseContext);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
return;
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java
index 14fd500a50..2e4935016b 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java
@@ -37,6 +37,7 @@ import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.EmbeddedLimitReachedException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.WriteLimitReachedException;
+import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.OneNote;
@@ -173,9 +174,9 @@ public class OneNoteParser implements Parser {
}
}
if (!walked) {
- legacyFallbackDump("OneNote parse failed; falling back to legacy text dump: " +
- failureMessage(structureFailure), structureFailure, metadata,
- xhtml, oneNoteDirectFileResource);
+ legacyFallbackDump("OneNote parse failed; falling back to legacy text dump",
+ structureFailure, metadata, xhtml, oneNoteDirectFileResource,
+ context);
}
} else if (header.isLegacyOrAlternativePackaging()) {
MSOneStorePackage pkg = null;
@@ -195,12 +196,12 @@ public class OneNoteParser implements Parser {
} catch (Exception e) {
rethrowIfLimitReached(e);
legacyFallbackDump(
- "OneNote FSSHTTPB parse failed; falling back to legacy text dump: " +
- failureMessage(e), e, metadata, xhtml,
- oneNoteDirectFileResource);
+ "OneNote FSSHTTPB parse failed; falling back to legacy text dump",
+ e, metadata, xhtml, oneNoteDirectFileResource, context);
pkg = null;
}
- legacyFallbackIfNoContent(pkg, metadata, xhtml, oneNoteDirectFileResource);
+ legacyFallbackIfNoContent(pkg, metadata, xhtml, oneNoteDirectFileResource,
+ context);
} else {
throw new TikaException("Invalid OneStore document - could not parse headers");
}
@@ -223,32 +224,32 @@ public class OneNoteParser implements Parser {
}
}
- private static String failureMessage(Exception e) {
- return e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();
- }
-
// the walk completed but every page dangled - without this a degraded
// file would yield empty output where the dump still finds its text
static void legacyFallbackIfNoContent(MSOneStorePackage pkg, Metadata metadata,
XHTMLContentHandler xhtml,
- OneNoteDirectFileResource oneNoteDirectFileResource)
+ OneNoteDirectFileResource oneNoteDirectFileResource,
+ ParseContext context)
throws TikaException, SAXException {
if (pkg != null && !pkg.hasEmittedContent()) {
legacyFallbackDump("OneNote FSSHTTPB parse produced no content; " +
"falling back to legacy text dump", null, metadata, xhtml,
- oneNoteDirectFileResource);
+ oneNoteDirectFileResource, context);
}
}
private static void legacyFallbackDump(String warning, Exception cause, Metadata metadata,
XHTMLContentHandler xhtml,
- OneNoteDirectFileResource oneNoteDirectFileResource)
+ OneNoteDirectFileResource oneNoteDirectFileResource,
+ ParseContext context)
throws TikaException, SAXException {
LOG.warn(warning);
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning);
if (cause != null) {
LOG.debug("OneNote parse failure", cause);
+ //the failure text is the policy's business, so it goes through format, not here
+ EmbeddedDocumentUtil.recordException(cause, metadata, context);
}
- metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning);
new OneNoteLegacyDumpStrings(oneNoteDirectFileResource, xhtml).dump();
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
index 6c16bca535..f49b1eadcb 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
@@ -107,7 +107,7 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor {
private final EmbeddedDocumentExtractor embeddedExtractor;
- private final ParseContext context;
+ protected final ParseContext context;
protected OfficeParserConfig config;
protected OPCPackage opcPackage;
@@ -128,9 +128,6 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor {
return new SAXBasedMetadataExtractor(opcPackage, context);
}
- ParseContext getParseContext() {
- return context;
- }
/**
* @see
* org.apache.tika.parser.microsoft.ooxml.OOXMLExtractor#getXHTML(ContentHandler, Metadata,
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
index 6eed018678..b91ba18bf9 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
@@ -71,7 +71,6 @@ public class SXSLFPowerPointExtractorDecorator extends AbstractOOXMLExtractor {
//TODO: what else
};
- private final ParseContext context;
private final Metadata metadata;
private final CommentAuthors commentAuthors = new CommentAuthors();
private PackagePart mainDocument = null;
@@ -89,7 +88,6 @@ public class SXSLFPowerPointExtractorDecorator extends AbstractOOXMLExtractor {
XSLFEventBasedPowerPointExtractor extractor) {
super(context, extractor.getPackage());
this.metadata = metadata;
- this.context = context;
for (String contentType : MAIN_STORY_PART_RELATIONS) {
List<PackagePart> pps = opcPackage.getPartsByContentType(contentType);
if (pps.size() > 0) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
index 5d1c7f122b..d72ac8f225 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
@@ -95,7 +95,6 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
};
- private final ParseContext context;
private final Metadata metadata;
private final Map<String, EmbeddedPartMetadata> embeddedPartMetadataMap = new HashMap<>();
@@ -104,7 +103,6 @@ public class SXWPFWordExtractorDecorator extends AbstractOOXMLExtractor {
XWPFEventBasedWordExtractor extractor) {
super(context, extractor.getPackage());
this.metadata = metadata;
- this.context = context;
}
@Override
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java
index 7e671057c2..7fd0953c55 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java
@@ -49,11 +49,8 @@ public class VSDXExtractorDecorator extends AbstractOOXMLExtractor {
private static final String VISIO_PAGE_REL =
"http://schemas.microsoft.com/visio/2010/relationships/page";
- private final ParseContext context;
-
public VSDXExtractorDecorator(ParseContext context, OPCPackage pkg) {
super(context, pkg);
- this.context = context;
}
@Override
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java
index d59ddd6528..e89fdf547d 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFBExcelExtractorDecorator.java
@@ -53,7 +53,6 @@ public class XSSFBExcelExtractorDecorator extends XSSFExcelExtractorDecorator {
throws SAXException, IOException, TikaException {
this.metadata = metadata;
- this.parseContext = context;
metadata.set(Office.PROTECTED_WORKSHEET, false);
super.getXHTML(handler, metadata, context);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
index 14647d566e..08d7c22f57 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
@@ -115,14 +115,11 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
private final Map<String, Set<Integer>> picturePages = new HashMap<>();
protected final Map<String, String> drawingHyperlinks = new HashMap<>();
protected Metadata metadata;
- protected ParseContext parseContext;
public XSSFExcelExtractorDecorator(ParseContext context, OPCPackage pkg,
Locale locale) {
super(context, pkg);
- this.parseContext = context;
-
if (locale == null) {
formatter = new TikaExcelDataFormatter();
} else {
@@ -135,17 +132,11 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
}
}
- @Override
- public MetadataExtractor getMetadataExtractor() {
- return new SAXBasedMetadataExtractor(opcPackage, parseContext);
- }
-
@Override
public void getXHTML(ContentHandler handler, Metadata metadata, ParseContext context)
throws SAXException, IOException, TikaException {
this.metadata = metadata;
- this.parseContext = context;
metadata.set(Office.PROTECTED_WORKSHEET, "false");
super.getXHTML(handler, metadata, context);
@@ -172,15 +163,15 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
// Styles and shared strings are optional — if either part is missing or
// unreadable, log to metadata and continue with degraded extraction.
try {
- stylesShim = new XSSFStylesShim(xssfReader.getStylesData(), parseContext);
+ stylesShim = new XSSFStylesShim(xssfReader.getStylesData(), context);
} catch (Exception e) {
- EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
}
try {
stringsShim = new XSSFSharedStringsShim(xssfReader.getSharedStringsData(),
- config.isConcatenatePhoneticRuns(), parseContext);
+ config.isConcatenatePhoneticRuns(), context);
} catch (Exception e) {
- EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
}
while (true) {
try {
@@ -188,7 +179,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
break;
}
} catch (RuntimeException e) {
- EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
break;
}
SheetTextAsHTML sheetExtractor = new SheetTextAsHTML(config, xhtml);
@@ -197,11 +188,8 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
try {
nextStream = iter.next();
} catch (RuntimeException e) {
- // POI can throw POIXMLException for missing sheet parts (e.g.,
- // truncated workbook references a sheet that isn't in the zip).
- // Break rather than continue — POI's iterator state may not have
- // advanced, which would cause an infinite loop.
- EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
+ // break, not continue: POI's iterator state may not have advanced
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
break;
}
try (InputStream stream = nextStream) {
@@ -226,19 +214,13 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
try {
processSheet(sheetExtractor, commentsShim, stylesShim, stringsShim, stream);
} catch (SAXException e) {
- // Truncated/malformed sheet XML — keep prior sheets and
- // record the failure as a warning.
WriteLimitReachedException.throwIfWriteLimitReached(e);
- EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
- // Balance any <tr>/<td> left open by the partial parse so
- // the </tbody></table></div> emitted below land in the
- // right place.
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
+ // a partial parse can leave <tr>/<td> open; close them so the
+ // </tbody></table></div> emitted below stay balanced
sheetExtractor.closeAnyPending();
} catch (IOException e) {
- // Truncated stream — same risk: partial <tr>/<td> still
- // open. Close them so the surrounding </tbody></table>
- // stays balanced, record the failure, and keep going.
- EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
sheetExtractor.closeAnyPending();
}
try {
@@ -278,7 +260,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
try (InputStream wbData = xssfReader.getWorkbookData()) {
XMLReaderUtils
.parseSAX(wbData, new WorkbookMetadataHandler(),
- parseContext);
+ context);
} catch (InvalidFormatException | TikaException e) {
//swallow
}
@@ -349,7 +331,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
if (pivotCachePart != null) {
PivotCacheHandler handler = new PivotCacheHandler(xhtml);
try (InputStream is = pivotCachePart.getInputStream()) {
- XMLReaderUtils.parseSAX(is, handler, parseContext);
+ XMLReaderUtils.parseSAX(is, handler, context);
}
if (handler.hasExternalData()) {
metadata.set(Office.HAS_EXTERNAL_PIVOT_DATA, true);
@@ -411,7 +393,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
if (externalLinkPart != null) {
ExternalLinkHandler handler = new ExternalLinkHandler(xhtml);
try (InputStream is = externalLinkPart.getInputStream()) {
- XMLReaderUtils.parseSAX(is, handler, parseContext);
+ XMLReaderUtils.parseSAX(is, handler, context);
}
if (handler.hasDdeLink()) {
metadata.set(Office.HAS_DDE_LINKS, true);
@@ -440,7 +422,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
if (connectionsPart != null) {
ConnectionsHandler handler = new ConnectionsHandler(xhtml);
try (InputStream is = connectionsPart.getInputStream()) {
- XMLReaderUtils.parseSAX(is, handler, parseContext);
+ XMLReaderUtils.parseSAX(is, handler, context);
}
if (handler.hasConnections()) {
metadata.set(Office.HAS_DATA_CONNECTIONS, true);
@@ -471,7 +453,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
PackagePart queryTablePart = sheetPart.getRelatedPart(rel);
if (queryTablePart != null) {
try (InputStream is = queryTablePart.getInputStream()) {
- XMLReaderUtils.parseSAX(is, new QueryTableHandler(xhtml), parseContext);
+ XMLReaderUtils.parseSAX(is, new QueryTableHandler(xhtml), context);
}
}
} catch (IOException | TikaException | IllegalArgumentException e) {
@@ -686,7 +668,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
continue;
}
try (InputStream is = threadedCommentPart.getInputStream()) {
- XMLReaderUtils.parseSAX(is, new ThreadedCommentHandler(xhtml), parseContext);
+ XMLReaderUtils.parseSAX(is, new ThreadedCommentHandler(xhtml), context);
}
}
}
@@ -713,7 +695,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
continue;
}
try (InputStream is = personsPart.getInputStream()) {
- XMLReaderUtils.parseSAX(is, new CommentPersonHandler(metadata), parseContext);
+ XMLReaderUtils.parseSAX(is, new CommentPersonHandler(metadata), context);
}
}
}
@@ -788,7 +770,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
try (InputStream is = drawingPart.getInputStream()) {
XMLReaderUtils.parseSAX(is,
new DrawingShapeHandler(xhtml, drawingHyperlinks),
- parseContext);
+ context);
} catch (IOException | TikaException e) {
//swallow
}
@@ -889,7 +871,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
XSSFSheetInterestingPartsCapturer handler = new XSSFSheetInterestingPartsCapturer(
new TikaSheetXMLHandler(stylesShim, commentsShim, stringsShim,
sheetContentsHandler, formatter, false));
- XMLReaderUtils.parseSAX(sheetInputStream, handler, parseContext);
+ XMLReaderUtils.parseSAX(sheetInputStream, handler, context);
sheetInputStream.close();
if (handler.hasProtection) {
@@ -924,7 +906,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
return null;
}
try (InputStream is = commentsPart.getInputStream()) {
- return new XSSFCommentsShim(is, parseContext);
+ return new XSSFCommentsShim(is, context);
}
} catch (InvalidFormatException | IOException | TikaException | SAXException e) {
//swallow — comments are not critical
@@ -1333,7 +1315,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
try {
prc = drawingPart.getRelationshipsByType(PackageRelationshipTypes.IMAGE_PART);
} catch (InvalidFormatException e) {
- EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
return;
}
if (prc == null) {
@@ -1347,7 +1329,7 @@ public class XSSFExcelExtractorDecorator extends AbstractOOXMLExtractor {
try {
imagePart = drawingPart.getRelatedPart(rel);
} catch (InvalidFormatException | IllegalArgumentException e) {
- EmbeddedDocumentUtil.recordException(e, metadata, parseContext);
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
continue;
}
if (imagePart == null) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java
index 308aa0cdd4..e4de2f7f82 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java
@@ -53,14 +53,12 @@ public class XPSExtractorDecorator extends AbstractOOXMLExtractor {
"http://schemas.openxps.org/oxps/v1.0/fixedrepresentation";
private static String XPS_DOCUMENT =
"http://schemas.microsoft.com/xps/2005/06/fixedrepresentation";
- private final ParseContext context;
private final ZipPackage pkg;
Map<String, Metadata> embeddedImages = new LinkedHashMap<>();
public XPSExtractorDecorator(ParseContext context, OPCPackage opcPackage)
throws TikaException {
super(context, opcPackage);
- this.context = context;
if (opcPackage instanceof ZipPackage) {
this.pkg = (ZipPackage) opcPackage;
} else {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/RTFEmbObjHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/RTFEmbObjHandler.java
index 6ef392ec69..1f85439aa6 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/RTFEmbObjHandler.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/RTFEmbObjHandler.java
@@ -186,9 +186,9 @@ class RTFEmbObjHandler {
byte[] bytes = os.toByteArray();
if (state == EMB_STATE.OBJDATA) {
- RTFObjDataParser objParser = new RTFObjDataParser(memoryLimitInKb, context);
+ RTFObjDataParser objParser = new RTFObjDataParser(memoryLimitInKb);
try {
- byte[] objBytes = objParser.parse(bytes, metadata, unknownFilenameCount);
+ byte[] objBytes = objParser.parse(bytes, metadata, context, unknownFilenameCount);
extractObj(objBytes, handler, metadata);
} catch (IOException e) {
EmbeddedDocumentUtil.recordException(e, metadata, context);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/RTFObjDataParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/RTFObjDataParser.java
index 737829be1d..be5c132bbe 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/RTFObjDataParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/RTFObjDataParser.java
@@ -57,35 +57,26 @@ class RTFObjDataParser {
private final static String WIN_ASCII = "WINDOWS-1252";
private final int memoryLimitInKb;
- private final ParseContext context;
- RTFObjDataParser(int memoryLimitInKb, ParseContext context) {
+ RTFObjDataParser(int memoryLimitInKb) {
this.memoryLimitInKb = memoryLimitInKb;
- this.context = context;
}
- /**
- * Parses the embedded object/pict string
- *
- * @param is actual bytes (already converted from the
- * hex pair string stored in the embedded object data into actual bytes or read
- * as raw binary bytes)
- * @return a SimpleRTFEmbObj or null
- * @throws IOException if there are any surprise surprises during parsing
- */
-
private static boolean hasPOIFSHeader(InputStream is) throws IOException {
return FileMagic.valueOf(is) == FileMagic.OLE2;
}
/**
- * @param bytes
+ * Parses the embedded object/pict bytes (already converted from the hex pair string
+ * stored in the embedded object data, or read as raw binary bytes).
+ *
* @param metadata incoming metadata
- * @param unknownFilenameCount
- * @return byte[] for contents of obj data
- * @throws IOException
+ * @param context parse context, for the exception-reporting policy
+ * @param unknownFilenameCount counter for naming embedded objects that have no name
+ * @return byte[] for contents of obj data, or null if this is not an embedded object
*/
- protected byte[] parse(byte[] bytes, Metadata metadata, AtomicInteger unknownFilenameCount)
+ protected byte[] parse(byte[] bytes, Metadata metadata, ParseContext context,
+ AtomicInteger unknownFilenameCount)
throws IOException, TikaException {
UnsynchronizedByteArrayInputStream is = UnsynchronizedByteArrayInputStream.builder().setByteArray(bytes).get();
long version = readUInt(is);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java
index f064b50c4a..d61aa5666e 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java
@@ -19,6 +19,7 @@ package org.apache.tika.parser.microsoft.onenote;
import static org.apache.tika.parser.microsoft.onenote.OneNoteParser.ONE_NOTE_PREFIX;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -44,6 +45,7 @@ import org.junit.jupiter.api.io.TempDir;
import org.xml.sax.ContentHandler;
import org.apache.tika.TikaTest;
+import org.apache.tika.config.ExceptionReporting;
import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.TikaMemoryLimitException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
@@ -59,30 +61,60 @@ public class OneNoteParserTest extends TikaTest {
//test recursive parser wrapper for image files
+ private static final String[][] FUZZ_RESOURCES = {
+ {"testOneNote-fuzz1.one", "Missing dependent revision"},
+ {"testOneNote-fuzz2.one", "unified property count"},
+ {"testOneNote-fuzz3.one", "Invalid GUID string"}
+ };
+
+ private static Metadata parseFuzzResource(String resource, ParseContext context)
+ throws Exception {
+ InputStream input = OneNoteParserTest.class
+ .getResourceAsStream("/test-documents/" + resource);
+ assertNotNull(input, resource);
+ Metadata metadata = new Metadata();
+ try (InputStream stream = input; TikaInputStream tis = TikaInputStream.get(stream)) {
+ new OneNoteParser().parse(tis, new ToTextContentHandler(), metadata, context);
+ }
+ return metadata;
+ }
+
@Test
public void testFuzzerRegressionInputsFallBackToLegacyDump() throws Exception {
- // a structural failure falls back to the legacy string dump; the failure is
- // pinned in the parse-warning metadata
- String[][] resources = {
- {"testOneNote-fuzz1.one", "Missing dependent revision"},
- {"testOneNote-fuzz2.one", "unified property count"},
- {"testOneNote-fuzz3.one", "Invalid GUID string"}
- };
- for (String[] resource : resources) {
- InputStream input = getClass().getResourceAsStream("/test-documents/" + resource[0]);
- assertNotNull(input, resource[0]);
- try (InputStream stream = input;
- TikaInputStream tis = TikaInputStream.get(stream)) {
- Metadata metadata = new Metadata();
- new OneNoteParser().parse(tis, new ToTextContentHandler(), metadata,
- new ParseContext());
- assertTrue(Arrays.stream(
- metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
- .anyMatch(warning -> warning.contains(resource[1])
- && warning.contains("falling back to legacy text dump")),
- () -> resource[0] + ": " + Arrays.toString(metadata.getValues(
- TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)));
+ // a structural failure falls back to the legacy string dump; the human warning names
+ // no exception text -- the throwable is recorded separately, under the policy
+ for (String[] resource : FUZZ_RESOURCES) {
+ Metadata metadata = parseFuzzResource(resource[0], new ParseContext());
+ String[] warnings =
+ metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ assertTrue(Arrays.stream(warnings)
+ .anyMatch(w -> w.contains("falling back to legacy text dump")
+ && !w.contains(resource[1])),
+ () -> resource[0] + ": " + Arrays.toString(warnings));
+ assertTrue(Arrays.stream(warnings)
+ .anyMatch(w -> w.contains(resource[1]) && w.contains("\tat ")),
+ () -> resource[0] + ": " + Arrays.toString(warnings));
+ }
+ }
+
+ @Test
+ public void testFallbackWarningHonorsExceptionReporting() throws Exception {
+ ParseContext context = new ParseContext();
+ context.set(ExceptionReporting.class, new ExceptionReporting(
+ ExceptionReporting.Level.REDACTED, ExceptionReporting.UNLIMITED));
+ for (String[] resource : FUZZ_RESOURCES) {
+ Metadata metadata = parseFuzzResource(resource[0], context);
+ String[] warnings =
+ metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ for (String warning : warnings) {
+ assertFalse(warning.contains(resource[1]),
+ () -> resource[0] + ": " + Arrays.toString(warnings));
+ assertFalse(warning.contains("\tat "),
+ () -> resource[0] + ": " + Arrays.toString(warnings));
}
+ assertTrue(Arrays.stream(warnings)
+ .anyMatch(w -> w.contains("falling back to legacy text dump")),
+ () -> resource[0] + ": " + Arrays.toString(warnings));
}
}
@@ -102,7 +134,7 @@ public class OneNoteParserTest extends TikaTest {
new OneNoteDirectFileResource(file.toFile())) {
// fresh package = the walk completed without emitting anything
OneNoteParser.legacyFallbackIfNoContent(new MSOneStorePackage(), metadata, xhtml,
- resource);
+ resource, new ParseContext());
assertTrue(writer.toString().contains("recoverable legacy dump text"));
assertTrue(metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)
.contains("produced no content"));
@@ -112,7 +144,8 @@ public class OneNoteParserTest extends TikaTest {
XHTMLContentHandler xhtml2 = new XHTMLContentHandler(
new ToTextContentHandler(exceptionPath), metadata, new ParseContext());
xhtml2.startDocument();
- OneNoteParser.legacyFallbackIfNoContent(null, metadata, xhtml2, resource);
+ OneNoteParser.legacyFallbackIfNoContent(null, metadata, xhtml2, resource,
+ new ParseContext());
xhtml2.endDocument();
assertTrue(exceptionPath.toString().isBlank());
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/pom.xml
index 57c54b8996..fc092e8a8f 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/pom.xml
@@ -92,6 +92,11 @@
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<!-- needs an entry or it will fail?! -->
@@ -122,21 +127,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
index 38ed311114..28c00910a3 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
@@ -75,6 +75,11 @@
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<!-- needs an entry or it will fail?! -->
@@ -105,21 +110,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index 3dfcc56870..aa57face74 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -157,7 +157,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
private static final MediaType XFA_MEDIA_TYPE = MediaType.application("vnd.adobe.xdp+xml");
private static final MediaType XMP_MEDIA_TYPE = MediaType.application("rdf+xml");
- final List<IOException> exceptions = new ArrayList<>();
+ final List<Exception> exceptions = new ArrayList<>();
final PDDocument pdDocument;
final XHTMLContentHandler xhtml;
final ParseContext context;
@@ -268,8 +268,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
writeParagraphStart();
}
- private void extractXMPXFA(PDDocument pdfDocument, Metadata parentMetadata,
- ParseContext context) throws IOException, SAXException {
+ private void extractXMPXFA() throws IOException, SAXException {
Set<MediaType> supportedTypes = Collections.EMPTY_SET;
Parser embeddedParser = context.get(Parser.class);
if (embeddedParser != null) {
@@ -282,22 +281,22 @@ class AbstractPDF2XHTML extends PDFTextStripper {
if (supportedTypes.contains(XMP_MEDIA_TYPE)) {
//try the main metadata
- if (pdfDocument.getDocumentCatalog().getMetadata() != null) {
+ if (pdDocument.getDocumentCatalog().getMetadata() != null) {
try (TikaInputStream tis = TikaInputStream.get(
- pdfDocument.getDocumentCatalog().getMetadata().exportXMPMetadata())) {
+ pdDocument.getDocumentCatalog().getMetadata().exportXMPMetadata())) {
extractXMPAsEmbeddedFile(tis, XMP_DOCUMENT_CATALOG_LOCATION);
} catch (IOException e) {
- EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata, context);
}
}
//now iterate through the pages
int pageNumber = 1;
- for (PDPage page : pdfDocument.getPages()) {
+ for (PDPage page : pdDocument.getPages()) {
if (page.getMetadata() != null) {
try (TikaInputStream tis = TikaInputStream.get(page.getMetadata().exportXMPMetadata())) {
extractXMPAsEmbeddedFile(tis, XMP_PAGE_LOCATION_PREFIX + pageNumber);
} catch (IOException e) {
- EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata, context);
}
}
pageNumber++;
@@ -305,8 +304,8 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
//now try the xfa
- if (pdfDocument.getDocumentCatalog().getAcroForm(null) != null &&
- pdfDocument.getDocumentCatalog().getAcroForm(null).getXFA() != null) {
+ if (pdDocument.getDocumentCatalog().getAcroForm(null) != null &&
+ pdDocument.getDocumentCatalog().getAcroForm(null).getXFA() != null) {
Metadata xfaMetadata = Metadata.newInstance(context);
xfaMetadata.set(HttpHeaders.CONTENT_TYPE, XFA_MEDIA_TYPE.toString());
@@ -316,9 +315,9 @@ class AbstractPDF2XHTML extends PDFTextStripper {
supportedTypes.contains(XFA_MEDIA_TYPE)) {
byte[] bytes = null;
try {
- bytes = pdfDocument.getDocumentCatalog().getAcroForm(null).getXFA().getBytes();
+ bytes = pdDocument.getDocumentCatalog().getAcroForm(null).getXFA().getBytes();
} catch (IOException e) {
- EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata, context);
}
if (bytes != null) {
try (TikaInputStream tis = TikaInputStream.get(bytes)) {
@@ -533,12 +532,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
if (config.isCatchIntermediateIOExceptions()) {
-
- String msg = e.getMessage();
- if (msg == null) {
- msg = "IOException, no message";
- }
- metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, msg);
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
exceptions.add(e);
} else {
throw e;
@@ -546,27 +540,14 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
/**
- * Mirrors {@link #handleCatchableIOE(IOException)}'s policy (governed by the same
- * {@code catchIntermediateIOExceptions} config flag) for a single page's OCR call
- * timing out. Without this, a per-page OCR timeout previously propagated as a plain
- * {@link IOException} out of {@link #endPage(PDPage)}, aborting the whole document
- * parse and losing every page after it -- the flagship "one image times out, siblings
- * continue" behavior this timeout model is built around didn't actually hold for the
- * most common OCR path. Recording into {@code exceptions} (rather than swallowing
- * outright) preserves the class's existing contract: remaining pages are still
- * processed and their content still reaches the handler, but the caller
- * (PDF2XHTML/OCR2XHTML/etc.) still surfaces a TikaException wrapping the first
- * recorded failure once every page has been attempted -- exactly like any other
- * per-page IOException already does via {@link #handleCatchableIOE}.
+ * Mirrors {@link #handleCatchableIOE(IOException)} for a per-page OCR timeout: remaining
+ * pages are still processed, and the caller still surfaces a TikaException wrapping the
+ * first recorded failure once every page has been attempted.
*/
void handleCatchableTimeout(TikaTimeoutException e) throws TikaTimeoutException {
if (config.isCatchIntermediateIOExceptions()) {
- String msg = e.getMessage();
- if (msg == null) {
- msg = "TikaTimeoutException, no message";
- }
- metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, msg);
- exceptions.add(new IOException(msg, e));
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
+ exceptions.add(e);
} else {
throw e;
}
@@ -605,7 +586,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
try (TemporaryResources tmp = new TemporaryResources()) {
- try (RenderResult renderResult = renderCurrentPage(pdPage, context, tmp)) {
+ try (RenderResult renderResult = renderCurrentPage(pdPage, tmp)) {
Metadata renderMetadata = renderResult.getMetadata();
try (TikaInputStream tis = renderResult.getInputStream()) {
renderMetadata.set(TikaCoreProperties.CONTENT_TYPE_PARSER_OVERRIDE,
@@ -630,13 +611,12 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
}
- private RenderResult renderCurrentPage(PDPage pdPage, ParseContext parseContext,
- TemporaryResources tmpResources)
+ private RenderResult renderCurrentPage(PDPage pdPage, TemporaryResources tmpResources)
throws IOException, TikaException {
- PDFRenderingState renderingState = parseContext.get(PDFRenderingState.class);
+ PDFRenderingState renderingState = context.get(PDFRenderingState.class);
if (renderingState == null) {
Metadata pageMetadata = getCurrentPageMetadata(pdPage);
- noContextRenderCurrentPage(pageMetadata, parseContext, tmpResources);
+ return noContextRenderCurrentPage(pageMetadata, tmpResources);
}
//if the full document has already been rendered, then reuse that file
//TODO: we need to prevent this if only a portion of the page or portions
@@ -652,7 +632,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
}
Metadata pageMetadata = getCurrentPageMetadata(pdPage);
- Renderer thisRenderer = getPDFRenderer(renderer);
+ Renderer thisRenderer = getPDFRenderer();
//if there's a configured renderer and if the rendering strategy is "all"
if (thisRenderer != null &&
config.getOcr().getRenderingStrategy() == OcrConfig.RenderingStrategy.ALL) {
@@ -662,7 +642,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
//do not do autocloseable. We need to leave the pdDocument open!
TikaInputStream tis = TikaInputStream.get(new byte[0]);
tis.setOpenContainer(pdDocument);
- return thisRenderer.render(tis, pageMetadata, parseContext, pageRangeRequest)
+ return thisRenderer.render(tis, pageMetadata, context, pageRangeRequest)
.getResults().get(0);
} else {
@@ -670,17 +650,17 @@ class AbstractPDF2XHTML extends PDFTextStripper {
if (state == null) {
throw new IllegalArgumentException("RenderingState must not be null");
}
- return thisRenderer.render(state.getTikaInputStream(), pageMetadata, parseContext,
+ return thisRenderer.render(state.getTikaInputStream(), pageMetadata, context,
pageRangeRequest).getResults().get(0);
}
} else {
- return noContextRenderCurrentPage(pageMetadata, parseContext, tmpResources);
+ return noContextRenderCurrentPage(pageMetadata, tmpResources);
}
}
- private Renderer getPDFRenderer(Renderer renderer) {
+ private Renderer getPDFRenderer() {
if (renderer == null) {
- return renderer;
+ return null;
}
if (renderer instanceof CompositeRenderer) {
return ((CompositeRenderer) renderer).getLeafRenderer(PDFParser.MEDIA_TYPE);
@@ -700,7 +680,6 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
private RenderResult noContextRenderCurrentPage(Metadata pageMetadata,
- ParseContext parseContext,
TemporaryResources tmpResources)
throws IOException, TikaException {
PDFRenderer renderer = null;
@@ -722,10 +701,10 @@ class AbstractPDF2XHTML extends PDFTextStripper {
int dpi = config.getOcr().getDpi();
Path tmpFile = null;
- RenderingTracker renderingTracker = parseContext.get(RenderingTracker.class);
+ RenderingTracker renderingTracker = context.get(RenderingTracker.class);
if (renderingTracker == null) {
renderingTracker = new RenderingTracker();
- parseContext.set(RenderingTracker.class, renderingTracker);
+ context.set(RenderingTracker.class, renderingTracker);
}
int id = renderingTracker.getNextId();
@@ -1178,7 +1157,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
handleCatchableIOE(e);
}
- extractXMPXFA(pdf, metadata, context);
+ extractXMPXFA();
//extract acroform data at end of doc
if (config.isExtractAcroFormContent() == true) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java
index 0f6a7d26fc..1c04c23a44 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java
@@ -427,7 +427,7 @@ public class ImageGraphicsEngine extends PDFGraphicsStreamEngine {
EmbeddedDocumentUtil.recordException(e, parentMetadata, parseContext);
return;
} catch (IOException e) {
- EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata, parseContext);
+ EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, parseContext);
return;
}
try (TikaInputStream tis = TikaInputStream.get(buffer.toByteArray())) {
@@ -499,11 +499,7 @@ public class ImageGraphicsEngine extends PDFGraphicsStreamEngine {
throw e;
}
- String msg = e.getMessage();
- if (msg == null) {
- msg = "IOException, no message";
- }
- parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, msg);
+ EmbeddedDocumentUtil.recordException(e, parentMetadata, parseContext);
exceptions.add(e);
} else {
throw e;
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngineTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngineTest.java
new file mode 100644
index 0000000000..832d691c61
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngineTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf.image;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.awt.image.BufferedImage;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.graphics.image.PDImage;
+import org.junit.jupiter.api.Test;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.pdf.PDFParserConfig;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+public class ImageGraphicsEngineTest {
+
+ // TIKA-4848: record on the parent, not the per-image metadata that is discarded here
+ @Test
+ public void testIOExceptionRecordedOnParentMetadata() throws Exception {
+ Metadata parentMetadata = new Metadata();
+ ParseContext parseContext = new ParseContext();
+ XHTMLContentHandler xhtml =
+ new XHTMLContentHandler(new DefaultHandler(), parentMetadata, parseContext);
+ xhtml.startDocument();
+
+ ImageGraphicsEngine engine = new ImageGraphicsEngine(new PDPage(), 1,
+ new NoOpEmbeddedDocumentExtractor(), new PDFParserConfig(), new HashMap<>(),
+ new AtomicInteger(0), xhtml, parentMetadata, parseContext) {
+ //the only PDImage dereference before writeToBuffer, so a null image suffices
+ @Override
+ protected String getSuffix(PDImage pdImage, Metadata metadata) {
+ return "png";
+ }
+
+ @Override
+ protected BufferedImage writeToBuffer(PDImage pdImage, String suffix,
+ boolean directJPEG, OutputStream out)
+ throws IOException {
+ throw new IOException("simulated broken image stream");
+ }
+ };
+
+ engine.processImage(null, 0);
+
+ String recorded =
+ parentMetadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM);
+ assertNotNull(recorded, "image write failure must be recorded on the parent metadata");
+ assertTrue(recorded.contains("simulated broken image stream"), recorded);
+ }
+
+ private static class NoOpEmbeddedDocumentExtractor implements EmbeddedDocumentExtractor {
+ @Override
+ public boolean shouldParseEmbedded(Metadata metadata, ParseContext parseContext) {
+ return true;
+ }
+
+ @Override
+ public void parseEmbedded(TikaInputStream stream, ContentHandler handler,
+ Metadata metadata, ParseContext parseContext,
+ boolean outputHtml) {
+ }
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
index 5b369292ea..58e703d79d 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
@@ -594,7 +594,7 @@ public class ZipParser extends AbstractArchiveParser {
}
private void handleEncryptedEntry(String name, Metadata parentMetadata,
- XHTMLContentHandler xhtml, ParseContext context) throws SAXException {
+ XHTMLContentHandler xhtml, ParseContext context) throws SAXException {
EmbeddedDocumentUtil.recordEmbeddedStreamException(
new EncryptedDocumentException("stream (" + name + ") is encrypted"),
parentMetadata, context);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/pom.xml
index 15e454e8f1..e01907dfd9 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/pom.xml
@@ -59,6 +59,11 @@
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -88,21 +93,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/pom.xml
index bc6908abf4..bb503245d9 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/pom.xml
@@ -35,6 +35,11 @@
<artifactId>commons-codec</artifactId>
</dependency>
</dependencies>
+ <properties>
+ <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
+ <forbiddenapis.module.signaturesFile>${metadata.forbiddenapis.signaturesFile}</forbiddenapis.module.signaturesFile>
+ </properties>
+
<build>
<plugins>
<!-- needs an entry or it will fail?! -->
@@ -65,21 +70,6 @@
</execution>
</executions>
</plugin>
- <!-- TIKA-4816 metadata string-key ban, main sources; see tika-parent -->
- <plugin>
- <groupId>de.thetaphi</groupId>
- <artifactId>forbiddenapis</artifactId>
- <executions>
- <execution>
- <id>forbiddenapis-check</id>
- <configuration>
- <signaturesFiles>
- <signaturesFile>${metadata.forbiddenapis.signaturesFile}</signaturesFile>
- </signaturesFiles>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesClient.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesClient.java
index 340fbf25ef..3988f9c90d 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesClient.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesClient.java
@@ -227,8 +227,11 @@ public class PipesClient implements Closeable {
LOG.error("server initialization failed: {} ", t.getId(), e);
serverManager.markServerForRestart(RestartReason.CRASH, connectionGeneration);
closeConnection();
+ // Config policy, not t.getParseContext(): a request context can never carry
+ // ExceptionReporting (wire-blocked), so it would always report FULL. Applies to
+ // every message this class fabricates below.
return buildFatalResult(t.getId(), t.getEmitKey(), PipesResult.RESULT_STATUS.FAILED_TO_INITIALIZE,
- intermediateResult.get(), e.getMessage());
+ intermediateResult.get(), ExceptionUtils.format(e, pipesConfig.getDefaultExceptionReporting()));
} catch (SecurityException e) {
LOG.error("security exception during initialization: {} ", t.getId());
serverManager.markServerForRestart(RestartReason.CRASH, connectionGeneration);
@@ -244,7 +247,7 @@ public class PipesClient implements Closeable {
t.getId(), e);
closeConnection();
return buildFatalResult(t.getId(), t.getEmitKey(), PipesResult.RESULT_STATUS.FAILED_TO_INITIALIZE,
- intermediateResult.get(), e.getMessage());
+ intermediateResult.get(), ExceptionUtils.format(e, pipesConfig.getDefaultExceptionReporting()));
}
try {
@@ -267,7 +270,7 @@ public class PipesClient implements Closeable {
e.getMessage());
return buildFatalResult(t.getId(), t.getEmitKey(),
PipesResult.RESULT_STATUS.PAYLOAD_LIMIT_EXCEEDED,
- intermediateResult.get(), e.getMessage());
+ intermediateResult.get(), ExceptionUtils.format(e, pipesConfig.getDefaultExceptionReporting()));
} catch (Exception e) {
LOG.error("exception waiting for server to complete task: {} ", t.getId(), e);
serverManager.markServerForRestart(RestartReason.CRASH, connectionGeneration);
@@ -492,7 +495,7 @@ public class PipesClient implements Closeable {
serverManager.markServerForRestart(RestartReason.TIMEOUT, connectionGeneration);
closeConnection();
return buildFatalResult(t.getId(), t.getEmitKey(), TIMEOUT, intermediateResult.get(),
- ExceptionUtils.getStackTrace(e));
+ ExceptionUtils.format(e, pipesConfig.getDefaultExceptionReporting()));
} catch (PayloadLimitExceededException e) {
// Stream is desynchronized (payload bytes were not consumed); close the connection.
LOG.warn("clientId={}: payload too large for id={}: {}", pipesClientId, t.getId(), e.getMessage());
@@ -500,7 +503,7 @@ public class PipesClient implements Closeable {
closeConnection();
return buildFatalResult(t.getId(), t.getEmitKey(),
PipesResult.RESULT_STATUS.PAYLOAD_LIMIT_EXCEEDED,
- intermediateResult.get(), e.getMessage());
+ intermediateResult.get(), ExceptionUtils.format(e, pipesConfig.getDefaultExceptionReporting()));
} catch (SecurityException e) {
throw e;
} catch (Exception e) {
@@ -517,7 +520,7 @@ public class PipesClient implements Closeable {
}
closeConnection();
return buildFatalResult(t.getId(), t.getEmitKey(), status, intermediateResult.get(),
- ExceptionUtils.getStackTrace(e));
+ ExceptionUtils.format(e, pipesConfig.getDefaultExceptionReporting()));
}
}
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesConfig.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesConfig.java
index a694d04b44..ce73c844f4 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesConfig.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesConfig.java
@@ -29,8 +29,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.util.StdConverter;
+import org.apache.tika.config.ExceptionReporting;
import org.apache.tika.config.TimeoutLimits;
import org.apache.tika.config.loader.TikaJsonConfig;
+import org.apache.tika.config.loader.TikaObjectMapperFactory;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.pipes.api.FetchEmitTuple;
import org.apache.tika.pipes.api.ParseMode;
@@ -204,22 +206,37 @@ public class PipesConfig {
if (config == null) {
config = new PipesConfig();
}
- // Carry the config-default limits to the client (for its backstop) without full
- // parse-context resolution -- the client JVM may lack the plugin classes it needs.
- JsonNode limitsNode = tikaJsonConfig.getRootNode().path("parse-context").path("timeout-limits");
- if (!limitsNode.isMissingNode()) {
+ // Carry the config-default limits and reporting policy to the client (for its backstop
+ // and for the results it fabricates itself) without full parse-context resolution --
+ // the client JVM may lack the plugin classes it needs. Bind with Tika's own mapper so
+ // the parent reads these two exactly as the fork does.
+ ObjectMapper mapper = TikaObjectMapperFactory.getMapper();
+ JsonNode parseContextNode = tikaJsonConfig.getRootNode().path("parse-context");
+ JsonNode limitsNode = parseContextNode.path("timeout-limits");
+ //isNull: an explicit JSON null keeps the default rather than binding a null field
+ if (!limitsNode.isMissingNode() && !limitsNode.isNull()) {
try {
- config.defaultTimeoutLimits =
- new ObjectMapper().treeToValue(limitsNode, TimeoutLimits.class);
+ config.defaultTimeoutLimits = mapper.treeToValue(limitsNode, TimeoutLimits.class);
} catch (JsonProcessingException e) {
throw new TikaConfigException("problem parsing parse-context.timeout-limits", e);
}
}
+ JsonNode reportingNode = parseContextNode.path("exception-reporting");
+ if (!reportingNode.isMissingNode() && !reportingNode.isNull()) {
+ try {
+ config.defaultExceptionReporting =
+ mapper.treeToValue(reportingNode, ExceptionReporting.class);
+ } catch (JsonProcessingException e) {
+ throw new TikaConfigException("problem parsing parse-context.exception-reporting", e);
+ }
+ }
return config;
}
private TimeoutLimits defaultTimeoutLimits = new TimeoutLimits();
+ private ExceptionReporting defaultExceptionReporting = new ExceptionReporting();
+
/**
* The config-level {@code parse-context.timeout-limits} defaults -- what the forked
* server enforces when a request carries no {@link TimeoutLimits} of its own.
@@ -229,6 +246,16 @@ public class PipesConfig {
return defaultTimeoutLimits;
}
+ /**
+ * The config-level {@code parse-context.exception-reporting} policy. The parent JVM needs
+ * it for the results it fabricates itself (crash, timeout, failed-to-initialize): the
+ * request's own ParseContext can never carry the policy -- it is wire-blocked.
+ */
+ @JsonIgnore
+ public ExceptionReporting getDefaultExceptionReporting() {
+ return defaultExceptionReporting;
+ }
+
public long getSocketTimeoutMillis() {
return socketTimeoutMillis;
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncEmitter.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncEmitter.java
index e59d6177c9..6e1f9b03c7 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncEmitter.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncEmitter.java
@@ -35,7 +35,6 @@ import org.apache.tika.pipes.api.emitter.EmitData;
import org.apache.tika.pipes.api.emitter.Emitter;
import org.apache.tika.pipes.core.PipesConfig;
import org.apache.tika.pipes.core.emitter.EmitterManager;
-import org.apache.tika.utils.ExceptionUtils;
/**
* Worker thread that takes EmitData off the queue, batches it
@@ -134,7 +133,7 @@ public class AsyncEmitter implements Callable<Integer> {
LOG.warn("emitter id={} failed on instantiation", e.getKey(), ex);
return;
}
- tryToEmit(emitter, e.getValue());
+ tryToEmit(e.getKey(), emitter, e.getValue());
emitted += e.getValue().size();
}
@@ -145,13 +144,13 @@ public class AsyncEmitter implements Callable<Integer> {
lastEmitted = Instant.now();
}
- private void tryToEmit(Emitter emitter, List<? extends EmitData> emitData) {
+ private void tryToEmit(String emitterId, Emitter emitter,
+ List<? extends EmitData> emitData) {
try {
emitter.emit(emitData);
} catch (IOException e) {
- LOG.warn("emitter class ({}): {}", emitter.getClass(),
- ExceptionUtils.getStackTrace(e));
+ LOG.warn("emitter id={} ({}) failed", emitterId, emitter.getClass(), e);
}
}
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
index f7d956882a..f1fe8a7a8a 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
@@ -156,8 +156,10 @@ class EmitHandler {
try {
emitter = emitterManager.getEmitter(emitKey.getEmitterId());
} catch (org.apache.tika.pipes.api.emitter.EmitterNotFoundException e) {
- String noEmitterMsg = getNoEmitterMsg(taskId);
- LOG.warn(noEmitterMsg);
+ String noEmitterMsg = "Emitter '" + emitKey.getEmitterId() + "' not found.";
+ // configured ids are operator-facing; log them, don't return them to the caller
+ LOG.warn("{} The configured emitterManager supports: {}", noEmitterMsg,
+ emitterManager.getSupported());
return new PipesResult(PipesResult.RESULT_STATUS.EMITTER_NOT_FOUND, noEmitterMsg);
} catch (IOException | TikaException e) {
LOG.warn("Couldn't initialize emitter for task id '" + taskId + "'", e);
@@ -306,19 +308,4 @@ class EmitHandler {
}
}
- private String getNoEmitterMsg(String emitterName) {
- StringBuilder sb = new StringBuilder();
- sb.append("Emitter '").append(emitterName).append("'");
- sb.append(" not found.");
- sb.append("\nThe configured emitterManager supports:");
- int i = 0;
- for (String e : emitterManager.getSupported()) {
- if (i++ > 0) {
- sb.append(", ");
- }
- sb.append(e);
- }
- return sb.toString();
- }
-
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/FetchHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/FetchHandler.java
index 5a0820bc00..418054719e 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/FetchHandler.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/FetchHandler.java
@@ -70,8 +70,10 @@ class FetchHandler {
try {
return new FetcherOrResult(fetcherManager.getFetcher(fetcherId), null);
} catch (FetcherNotFoundException e) {
- String noFetcherMsg = getNoFetcherMsg(t.getFetchKey().getFetcherId());
- LOG.warn(noFetcherMsg);
+ String noFetcherMsg = "Fetcher '" + t.getFetchKey().getFetcherId() + "' not found.";
+ // configured ids are operator-facing; log them, don't return them to the caller
+ LOG.warn("{} The configured FetcherManager supports: {}", noFetcherMsg,
+ fetcherManager.getSupported());
return new FetcherOrResult(null, new PipesResult(PipesResult.RESULT_STATUS.FETCHER_NOT_FOUND, noFetcherMsg));
} catch (IOException | TikaException e) {
LOG.warn("Couldn't initialize fetcher for fetch id={}", t.getId(), e);
@@ -81,21 +83,6 @@ class FetchHandler {
}
- private String getNoFetcherMsg(String fetcherId) {
- StringBuilder sb = new StringBuilder();
- sb.append("Fetcher '").append(fetcherId).append("'");
- sb.append(" not found.");
- sb.append("\nThe configured FetcherManager supports:");
- int i = 0;
- for (String f : fetcherManager.getSupported()) {
- if (i++ > 0) {
- sb.append(", ");
- }
- sb.append(f);
- }
- return sb.toString();
- }
-
private record FetcherOrResult(Fetcher fetcher, PipesResult pipesResult) {
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
index 17b16eb7ca..173a7efc6e 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
@@ -200,31 +200,23 @@ public class PipesServer implements AutoCloseable {
DataInputStream dis = new DataInputStream(new BufferedInputStream(socket.getInputStream()));
DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(socket.getOutputStream()));
+ ParseContext configContext = null;
try {
TikaLoader tikaLoader = TikaLoader.load(tikaConfigPath);
- TikaJsonConfig tikaJsonConfig = tikaLoader.getConfig();
- PipesConfig pipesConfig = PipesConfig.load(tikaJsonConfig);
-
- // Set socket timeout from config after loading PipesConfig
- socket.setSoTimeout((int) pipesConfig.getSocketTimeoutMillis());
- socket.setTcpNoDelay(true);
-
- MetadataFilter metadataFilter = tikaLoader.loadMetadataFilters();
- ContentHandlerFactory contentHandlerFactory = tikaLoader.loadContentHandlerFactory();
- MetadataWriteLimiterFactory metadataWriteLimiterFactory = tikaLoader.loadParseContext().get(MetadataWriteLimiterFactory.class);
- PipesServer pipesServer = new PipesServer(pipesClientId, tikaLoader, pipesConfig, socket, dis, dos, metadataFilter, contentHandlerFactory, metadataWriteLimiterFactory);
+ //load before constructing so the catch block below can redact per config
+ configContext = tikaLoader.loadParseContext();
+ PipesServer pipesServer =
+ new PipesServer(pipesClientId, socket, dis, dos, tikaLoader, configContext);
pipesServer.initializeResources();
LOG.debug("PipesServer loaded and ready");
return pipesServer;
} catch (Exception e) {
LOG.error("Failed to start up", e);
try {
- // Config may be what failed to load, so no ExceptionReporting policy is available.
- String msg = ExceptionUtils.format(e, ExceptionReporting.DEFAULT);
+ // a null configContext means the config is what failed: built-in defaults then
+ String msg = ExceptionUtils.format(e, ExceptionReporting.get(configContext));
byte[] bytes = msg.getBytes(StandardCharsets.UTF_8);
PipesMessage.startupFailed(bytes).write(dos);
- // pipesConfig may not have loaded successfully (that may be why we're
- // here); use the built-in default rather than an unreliable reference.
PipesMessage ackMsg = PipesMessage.read(dis);
if (ackMsg.type() != PipesMessageType.ACK) {
LOG.warn("Expected ACK but got: {}", ackMsg.type());
@@ -236,26 +228,26 @@ public class PipesServer implements AutoCloseable {
}
}
- public PipesServer(String pipesClientId, TikaLoader tikaLoader, PipesConfig pipesConfig, Socket socket, DataInputStream in,
- DataOutputStream out, MetadataFilter metadataFilter, ContentHandlerFactory contentHandlerFactory,
- MetadataWriteLimiterFactory metadataWriteLimiterFactory) throws TikaConfigException,
- IOException {
+ private PipesServer(String pipesClientId, Socket socket, DataInputStream in,
+ DataOutputStream out, TikaLoader tikaLoader, ParseContext configContext)
+ throws TikaConfigException, IOException {
this.pipesClientId = pipesClientId;
this.tikaLoader = tikaLoader;
- this.pipesConfig = pipesConfig;
+ this.pipesConfig = PipesConfig.load(tikaLoader.getConfig());
this.socket = socket;
- this.defaultMetadataFilter = metadataFilter;
- this.defaultContentHandlerFactory = contentHandlerFactory;
- this.defaultMetadataWriteLimiterFactory = metadataWriteLimiterFactory;
- this.input = new DataInputStream(in);
- this.output = new DataOutputStream(out);
+ socket.setSoTimeout((int) pipesConfig.getSocketTimeoutMillis());
+ this.defaultMetadataFilter = tikaLoader.loadMetadataFilters();
+ this.defaultContentHandlerFactory = tikaLoader.loadContentHandlerFactory();
+ this.defaultMetadataWriteLimiterFactory = configContext.get(MetadataWriteLimiterFactory.class);
+ this.input = in;
+ this.output = out;
this.heartbeatIntervalMillis = pipesConfig.getHeartbeatIntervalMillis();
validateHeartbeatInterval(pipesConfig);
emitStrategy = pipesConfig.getEmitStrategy().getType();
this.protocolIO = new ServerProtocolIO(input, output, pipesConfig.getMaxIpcPayloadBytes(),
- ExceptionReporting.get(tikaLoader.loadParseContext()));
+ ExceptionReporting.get(configContext));
}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
index 09ba6ed797..f8512bc6bf 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
@@ -85,13 +85,16 @@ public class ServerProtocolIO {
private final int maxIpcPayloadBytes;
private final ExceptionReporting exceptionReporting;
+ /**
+ * @deprecated since 4.1, use the overload taking an {@link ExceptionReporting}
+ */
+ @Deprecated
public ServerProtocolIO(DataInputStream input, DataOutputStream output, int maxIpcPayloadBytes) {
- this(input, output, maxIpcPayloadBytes, ExceptionReporting.DEFAULT);
+ this(input, output, maxIpcPayloadBytes, new ExceptionReporting());
}
public ServerProtocolIO(DataInputStream input, DataOutputStream output, int maxIpcPayloadBytes,
ExceptionReporting exceptionReporting) {
- this.exceptionReporting = exceptionReporting;
if (maxIpcPayloadBytes < MIN_FALLBACK_PAYLOAD_BYTES) {
throw new IllegalArgumentException(String.format(Locale.ROOT,
"maxIpcPayloadBytes %d is below the minimum %d required to carry a PAYLOAD_LIMIT_EXCEEDED response",
@@ -100,6 +103,7 @@ public class ServerProtocolIO {
this.input = input;
this.output = output;
this.maxIpcPayloadBytes = maxIpcPayloadBytes;
+ this.exceptionReporting = exceptionReporting;
}
/**
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java
index 29338b2823..21371d2ca9 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java
+++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java
@@ -75,7 +75,6 @@ public class SharedServerResources {
MetadataWriteLimiterFactory defaultMetadataWriteLimiterFactory,
EmitStrategy emitStrategy, ConfigStore configStore,
ExceptionReporting exceptionReporting) {
- this.exceptionReporting = exceptionReporting;
this.tikaLoader = tikaLoader;
this.pipesConfig = pipesConfig;
this.autoDetectParser = autoDetectParser;
@@ -88,6 +87,7 @@ public class SharedServerResources {
this.defaultMetadataWriteLimiterFactory = defaultMetadataWriteLimiterFactory;
this.emitStrategy = emitStrategy;
this.configStore = configStore;
+ this.exceptionReporting = exceptionReporting;
}
/**
diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
index e90ef06fc1..668bef4ad5 100644
--- a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
+++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.pipes.core;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -62,6 +63,24 @@ public class TikaPipesConfigTest extends TikaTest {
assertEquals(100 * 1024 * 1024, config.getMaxIpcPayloadBytes());
}
+ @Test
+ void testExplicitNullParseContextDefaultsKeepDefaults() throws Exception {
+ // an explicit JSON null must not bind a null policy that NPEs on the crash path
+ String json = """
+ {
+ "pipes": {},
+ "parse-context": {
+ "timeout-limits": null,
+ "exception-reporting": null
+ }
+ }
+ """;
+ PipesConfig config = PipesConfig.load(TikaJsonConfig.load(
+ new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))));
+ assertNotNull(config.getDefaultTimeoutLimits());
+ assertNotNull(config.getDefaultExceptionReporting());
+ }
+
@Test
void testMaxIpcPayloadBytesFromJson() throws Exception {
String json = """
diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java
index bbb456bd4d..2b91a2afb0 100644
--- a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java
+++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java
@@ -124,7 +124,7 @@ class ServerProtocolIOTest {
ServerProtocolIO io = new ServerProtocolIO(
new DataInputStream(serverInPipe),
new DataOutputStream(serverOutPipe),
- maxPayloadBytes);
+ maxPayloadBytes, new ExceptionReporting());
io.writeFinished(toWrite);
clientThread.join(5000);
@@ -295,7 +295,7 @@ class ServerProtocolIOTest {
* sends ACK, and returns the deserialized stack trace string.
*/
private String exchangeCrash(Throwable t, int maxPayloadBytes) throws Exception {
- return exchangeCrash(t, maxPayloadBytes, ExceptionReporting.DEFAULT);
+ return exchangeCrash(t, maxPayloadBytes, new ExceptionReporting());
}
private String exchangeCrash(Throwable t, int maxPayloadBytes, ExceptionReporting reporting)
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/ExceptionReportingPipesTest.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/ExceptionReportingPipesTest.java
index 89094f94f0..140834e8d1 100644
--- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/ExceptionReportingPipesTest.java
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/ExceptionReportingPipesTest.java
@@ -36,33 +36,40 @@ import org.apache.tika.pipes.api.emitter.EmitKey;
import org.apache.tika.pipes.api.fetcher.FetchKey;
/**
- * The forked worker must apply the config's exception-reporting policy to the container
- * exception it records and to the PipesResult message it returns on fetch failure.
+ * The config's exception-reporting policy must reach every message pipes hands a caller:
+ * the container exception and the fetch/emit failures the forked worker reports, and the
+ * results the parent fabricates when the fork never gets far enough to answer.
*/
public class ExceptionReportingPipesTest {
private static final String FETCHER_NAME = "fsf";
+ private static final String EMITTER_NAME = "fse";
private static final String NPE_DOC = "mock-npe.xml";
+ private static final String CLEAN_DOC = "mock_times.xml";
private static final String MESSAGE = "secret null pointer message";
- private PipesClient init(Path tmp, String template) throws Exception {
+ private PipesClient init(Path tmp, String template, String... testDocs) throws Exception {
Path tikaConfigPath = PluginsTestHelper.getFileSystemFetcherConfig(
template, tmp, tmp.resolve("input"), tmp.resolve("output"), false);
- PluginsTestHelper.copyTestFilesToTmpInput(tmp, NPE_DOC);
+ PluginsTestHelper.copyTestFilesToTmpInput(tmp, testDocs);
PipesConfig pipesConfig = PipesConfig.load(TikaJsonConfig.load(tikaConfigPath));
return new PipesClient(pipesConfig, tikaConfigPath);
}
private static FetchEmitTuple tuple(String fetchKey) {
+ return tuple(fetchKey, new EmitKey());
+ }
+
+ private static FetchEmitTuple tuple(String fetchKey, EmitKey emitKey) {
ParseContext parseContext = new ParseContext();
parseContext.set(ParseMode.class, ParseMode.RMETA);
- return new FetchEmitTuple(fetchKey, new FetchKey(FETCHER_NAME, fetchKey), new EmitKey(),
+ return new FetchEmitTuple(fetchKey, new FetchKey(FETCHER_NAME, fetchKey), emitKey,
new Metadata(), parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT);
}
@Test
public void containerExceptionRedacted(@TempDir Path tmp) throws Exception {
- try (PipesClient client = init(tmp, "tika-config-exception-reporting.json")) {
+ try (PipesClient client = init(tmp, "tika-config-exception-reporting.json", NPE_DOC)) {
PipesResult result = client.process(tuple(NPE_DOC));
assertEquals(PipesResult.RESULT_STATUS.PARSE_SUCCESS_WITH_EXCEPTION, result.status());
String trace = result.emitData().getMetadataList().get(0)
@@ -76,7 +83,7 @@ public class ExceptionReportingPipesTest {
@Test
public void containerExceptionFullByDefault(@TempDir Path tmp) throws Exception {
- try (PipesClient client = init(tmp, "tika-config-basic.json")) {
+ try (PipesClient client = init(tmp, "tika-config-basic.json", NPE_DOC)) {
PipesResult result = client.process(tuple(NPE_DOC));
assertEquals(PipesResult.RESULT_STATUS.PARSE_SUCCESS_WITH_EXCEPTION, result.status());
String trace = result.emitData().getMetadataList().get(0)
@@ -87,13 +94,58 @@ public class ExceptionReportingPipesTest {
@Test
public void fetchExceptionRedacted(@TempDir Path tmp) throws Exception {
- try (PipesClient client = init(tmp, "tika-config-exception-reporting.json")) {
+ try (PipesClient client = init(tmp, "tika-config-exception-reporting.json", NPE_DOC)) {
PipesResult result = client.process(tuple("does-not-exist.xml"));
assertEquals(PipesResult.RESULT_STATUS.FETCH_EXCEPTION, result.status());
String msg = result.message();
- assertTrue(msg.contains("Exception"), msg);
+ assertTrue(msg.contains("java.io.FileNotFoundException"), msg);
+ assertTrue(msg.contains("\tat "), msg);
// the fetcher's message names the missing path; the policy must strip it
assertFalse(msg.contains("does-not-exist"), msg);
}
}
+
+ @Test
+ public void emitExceptionRedacted(@TempDir Path tmp) throws Exception {
+ Path outputDir = tmp.resolve("output");
+ try (PipesClient client = init(tmp, "tika-config-exception-reporting-emit-all.json",
+ CLEAN_DOC)) {
+ EmitKey emitKey = new EmitKey(EMITTER_NAME, CLEAN_DOC);
+ assertEquals(PipesResult.RESULT_STATUS.EMIT_SUCCESS,
+ client.process(tuple(CLEAN_DOC, emitKey)).status());
+ // onExists=EXCEPTION: the second emit of the same key fails
+ PipesResult result = client.process(tuple(CLEAN_DOC, emitKey));
+ assertEquals(PipesResult.RESULT_STATUS.EMIT_EXCEPTION, result.status());
+ String msg = result.message();
+ assertTrue(msg.contains("java.io.IOException"), msg);
+ assertTrue(msg.contains("\tat "), msg);
+ // the emitter's message names the absolute output path; the policy must strip it
+ assertFalse(msg.contains(outputDir.toAbsolutePath().toString()), msg);
+ assertFalse(msg.contains("onExists"), msg);
+ }
+ }
+
+ /**
+ * FAILED_TO_INITIALIZE is fabricated in the parent JVM, which has no request that could
+ * carry the policy -- it must read it from the config, or the fork's startup trace
+ * (config path, plugin roots) is returned verbatim.
+ */
+ @Test
+ public void startupFailureRedacted(@TempDir Path tmp) throws Exception {
+ try (PipesClient client = init(tmp,
+ "tika-config-exception-reporting-startup-failure.json")) {
+ FetchEmitTuple t = new FetchEmitTuple(NPE_DOC, new FetchKey("bad-fetcher", NPE_DOC),
+ new EmitKey(), new Metadata(), new ParseContext(),
+ FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT);
+ PipesResult result = client.process(t);
+ assertEquals(PipesResult.RESULT_STATUS.FAILED_TO_INITIALIZE, result.status());
+ String msg = result.message();
+ assertTrue(msg.contains("ServerInitializationException"), msg);
+ assertTrue(msg.contains("\tat "), msg);
+ // a MESSAGE_REDACTED trace carries no message text at all
+ assertFalse(msg.contains("Exception:"), msg);
+ assertFalse(msg.contains("non-existent-fetcher-plugin"), msg);
+ assertFalse(msg.contains(tmp.toAbsolutePath().toString()), msg);
+ }
+ }
}
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/PipesClientTest.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/PipesClientTest.java
index e343efd060..3a1002976d 100644
--- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/PipesClientTest.java
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/PipesClientTest.java
@@ -626,6 +626,9 @@ public class PipesClientTest {
pipesResult.message().contains("not found") ||
pipesResult.message().contains("fetcher"),
"Error message should mention the missing fetcher");
+
+ // the configured fetcher ids must not be disclosed to the caller
+ assertFalse(pipesResult.message().contains("fsf"), pipesResult.message());
}
}
@@ -672,6 +675,9 @@ public class PipesClientTest {
pipesResult.message().contains("not found") ||
pipesResult.message().contains("emitter"),
"Error message should mention the missing emitter");
+
+ // the configured emitter ids must not be disclosed to the caller
+ assertFalse(pipesResult.message().contains("fse"), pipesResult.message());
}
}
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/resources/configs/tika-config-exception-reporting-emit-all.json b/tika-pipes/tika-pipes-integration-tests/src/test/resources/configs/tika-config-exception-reporting-emit-all.json
new file mode 100644
index 0000000000..8c6498f3d7
--- /dev/null
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/resources/configs/tika-config-exception-reporting-emit-all.json
@@ -0,0 +1,41 @@
+{
+ "fetchers": {
+ "fsf": {
+ "file-system-fetcher": {
+ "basePath": "FETCHER_BASE_PATH",
+ "extractFileSystemMetadata": false
+ }
+ }
+ },
+ "emitters": {
+ "fse": {
+ "file-system-emitter": {
+ "basePath": "EMITTER_BASE_PATH",
+ "fileExtension": "json",
+ "onExists": "EXCEPTION"
+ }
+ }
+ },
+ "pipes": {
+ "numClients": 1,
+ "forkedJvmArgs": [
+ "-Xmx256m"
+ ],
+ "emitStrategy": {
+ "type": "EMIT_ALL"
+ }
+ },
+ "parse-context": {
+ "exception-reporting": {
+ "level": "MESSAGE_REDACTED",
+ "maxLength": 10000
+ },
+ "timeout-limits": {
+ "progressTimeoutMillis": 60000
+ }
+ },
+ "auto-detect-parser": {
+ "throwOnZeroBytes": false
+ },
+ "plugin-roots": "PLUGINS_PATHS"
+}
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/resources/configs/tika-config-exception-reporting-startup-failure.json b/tika-pipes/tika-pipes-integration-tests/src/test/resources/configs/tika-config-exception-reporting-startup-failure.json
new file mode 100644
index 0000000000..04c9c77b18
--- /dev/null
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/resources/configs/tika-config-exception-reporting-startup-failure.json
@@ -0,0 +1,29 @@
+{
+ "fetchers": {
+ "bad-fetcher": {
+ "non-existent-fetcher-plugin": {
+ "basePath": "FETCHER_BASE_PATH"
+ }
+ }
+ },
+ "pipes": {
+ "numClients": 1,
+ "forkedJvmArgs": [
+ "-Xmx256m"
+ ],
+ "emitStrategy": {
+ "type": "DYNAMIC",
+ "thresholdBytes": 1000000
+ }
+ },
+ "parse-context": {
+ "exception-reporting": {
+ "level": "MESSAGE_REDACTED",
+ "maxLength": 10000
+ },
+ "timeout-limits": {
+ "progressTimeoutMillis": 5000
+ }
+ },
+ "plugin-roots": "PLUGINS_PATHS"
+}
diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/src/main/java/org/apache/tika/pipes/reporter/fs/FileSystemJsonlReporter.java b/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/src/main/java/org/apache/tika/pipes/reporter/fs/FileSystemJsonlReporter.java
index 0d74ce5445..ff0236134a 100644
--- a/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/src/main/java/org/apache/tika/pipes/reporter/fs/FileSystemJsonlReporter.java
+++ b/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/src/main/java/org/apache/tika/pipes/reporter/fs/FileSystemJsonlReporter.java
@@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.tika.config.ExceptionReporting;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.pipes.api.FetchEmitTuple;
import org.apache.tika.pipes.api.PipesResult;
@@ -180,7 +181,7 @@ public class FileSystemJsonlReporter extends PipesReporterBase {
@Override
public void error(Throwable t) {
- error(ExceptionUtils.getStackTrace(t));
+ error(ExceptionUtils.format(t, new ExceptionReporting()));
}
@Override
diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/src/main/java/org/apache/tika/pipes/reporter/fs/FileSystemStatusReporter.java b/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/src/main/java/org/apache/tika/pipes/reporter/fs/FileSystemStatusReporter.java
index 83a527d0cd..f76c77fc14 100644
--- a/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/src/main/java/org/apache/tika/pipes/reporter/fs/FileSystemStatusReporter.java
+++ b/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/src/main/java/org/apache/tika/pipes/reporter/fs/FileSystemStatusReporter.java
@@ -32,6 +32,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.tika.config.ExceptionReporting;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.pipes.api.FetchEmitTuple;
import org.apache.tika.pipes.api.PipesResult;
@@ -159,7 +160,7 @@ public class FileSystemStatusReporter extends AbstractTikaExtension implements P
public void error(Throwable t) {
crashed = true;
interuptThread();
- crash(ExceptionUtils.getStackTrace(t));
+ crash(ExceptionUtils.format(t, new ExceptionReporting()));
}
@Override
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
index 0048a3753c..592aaed460 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java
@@ -17,7 +17,9 @@
package org.apache.tika.serialization.serdes;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.Map;
import java.util.Set;
@@ -53,6 +55,20 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> {
// Friendly names written from the context map, so we skip them when writing jsonConfigs.
Set<String> serializedNames = new HashSet<>();
+ // Unregistered instances that resolveAll materialized from a jsonConfig (e.g. the
+ // composite behind "metadata-filters") cannot be serialized themselves; their original
+ // JSON below carries them. Registered instances are always written from the instance:
+ // callers may mutate a resolved instance before serialization (parseUnpack does), so
+ // the live object, not the original JSON, is the truth for them.
+ Map<String, JsonConfig> jsonConfigs = parseContext.getJsonConfigs();
+ Set<Object> resolvedFromJson = Collections.newSetFromMap(new IdentityHashMap<>());
+ for (String name : jsonConfigs.keySet()) {
+ Object resolved = parseContext.getResolvedConfig(name);
+ if (resolved != null) {
+ resolvedFromJson.add(resolved);
+ }
+ }
+
// Context-map objects are written flat, by friendly name; read back as lazy jsonConfigs.
Map<String, Object> contextMap = parseContext.getContextMap();
for (Map.Entry<String, Object> entry : contextMap.entrySet()) {
@@ -66,6 +82,10 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> {
// Find the friendly component name — all serializable components must be registered
String keyName = ComponentNameResolver.getFriendlyName(value.getClass());
if (keyName == null) {
+ if (resolvedFromJson.contains(value)) {
+ // unserializable, but its original jsonConfig is written below
+ continue;
+ }
throw new IOException(
"Cannot serialize ParseContext entry: " + value.getClass().getName() +
" is not registered. Components must be registered via " +
@@ -82,7 +102,6 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> {
}
// Then, serialize JSON configs at the top level (skip any already written above)
- Map<String, JsonConfig> jsonConfigs = parseContext.getJsonConfigs();
for (Map.Entry<String, JsonConfig> entry : jsonConfigs.entrySet()) {
if (serializedNames.contains(entry.getKey())) {
continue;
diff --git a/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java b/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingConfigLoadTest.java
similarity index 73%
rename from tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java
rename to tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingConfigLoadTest.java
index 3b60a535ea..3ea59ecf61 100644
--- a/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java
+++ b/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingConfigLoadTest.java
@@ -17,7 +17,6 @@
package org.apache.tika.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertSame;
import org.junit.jupiter.api.Test;
@@ -25,7 +24,7 @@ import org.apache.tika.TikaTest;
import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.parser.ParseContext;
-public class ExceptionReportingTest extends TikaTest {
+public class ExceptionReportingConfigLoadTest extends TikaTest {
@Test
public void testLoadFromConfig() throws Exception {
@@ -34,12 +33,4 @@ public class ExceptionReportingTest extends TikaTest {
ExceptionReporting reporting = ExceptionReporting.get(context);
assertEquals(new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, 500), reporting);
}
-
- @Test
- public void testDefaults() {
- assertSame(ExceptionReporting.DEFAULT, ExceptionReporting.get(null));
- assertSame(ExceptionReporting.DEFAULT, ExceptionReporting.get(new ParseContext()));
- assertEquals(ExceptionReporting.Level.FULL, ExceptionReporting.DEFAULT.getLevel());
- assertEquals(ExceptionReporting.UNLIMITED, ExceptionReporting.DEFAULT.getMaxLength());
- }
}
diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java b/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java
index 32b73d7503..d93e5924b1 100644
--- a/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java
+++ b/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java
@@ -298,6 +298,54 @@ public class TestParseContextSerialization {
assertEquals(MockUpperCaseFilter.class, deserFilter.getFilters().get(1).getClass());
}
+ @Test
+ public void testResolvedContextSerializesOriginalJson() throws Exception {
+ // TIKA-4848 review: a parent JVM (server/grpc) resolves the request's ParseContext
+ // before forwarding it to the fork. The materialized instances (e.g. the unregistered
+ // CompositeMetadataFilter) must not break -- or lossily replace -- serialization;
+ // the original jsonConfigs must be what crosses the wire.
+ ParseContext parseContext = new ParseContext();
+ parseContext.setJsonConfig("metadata-filters", """
+ [
+ "attachment-counting-list-filter",
+ "mock-upper-case-filter"
+ ]
+ """);
+ ParseContextUtils.resolveAll(parseContext, Thread.currentThread().getContextClassLoader());
+ assertEquals(CompositeMetadataFilter.class,
+ parseContext.get(MetadataFilter.class).getClass());
+
+ ObjectMapper mapper = createMapper();
+ String json = mapper.writeValueAsString(parseContext);
+
+ JsonNode filters = mapper.readTree(json).get("metadata-filters");
+ assertNotNull(filters, "original metadata-filters json should survive serialization");
+ assertTrue(filters.isArray());
+ assertEquals("attachment-counting-list-filter", filters.get(0).asText());
+
+ // and the fork side can resolve it again
+ ParseContext deser = mapper.readValue(json, ParseContext.class);
+ ParseContextUtils.resolveAll(deser, Thread.currentThread().getContextClassLoader());
+ CompositeMetadataFilter refilter = (CompositeMetadataFilter) deser.get(MetadataFilter.class);
+ assertEquals(MockUpperCaseFilter.class, refilter.getFilters().get(1).getClass());
+ }
+
+ @Test
+ public void testMutatedResolvedInstanceWinsOverOriginalJson() throws Exception {
+ // The server mutates resolved instances before forwarding (parseUnpack does); the
+ // live registered instance, not the original jsonConfig, must cross the wire.
+ ParseContext parseContext = new ParseContext();
+ parseContext.setJsonConfig("timeout-limits", "{\"progressTimeoutMillis\": 1000}");
+ ParseContextUtils.resolveAll(parseContext, Thread.currentThread().getContextClassLoader());
+ parseContext.get(TimeoutLimits.class).setProgressTimeoutMillis(7777);
+
+ ObjectMapper mapper = createMapper();
+ ParseContext deser = mapper.readValue(mapper.writeValueAsString(parseContext),
+ ParseContext.class);
+ ParseContextUtils.resolveAll(deser, Thread.currentThread().getContextClassLoader());
+ assertEquals(7777, deser.get(TimeoutLimits.class).getProgressTimeoutMillis());
+ }
+
@Test
public void testContextKeyDeserialization() throws Exception {
// Test that components with @TikaComponent(contextKey=...) are stored
diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java b/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java
index bbfebac925..8282c82d4c 100644
--- a/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java
+++ b/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.serialization;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -62,6 +63,12 @@ public class WireRestrictedParseContextTest {
Set<Class<?>> union = new HashSet<>(allowed);
union.addAll(blocked);
assertEquals(all, union, "wire-allowed union wire-blocked must equal CONTEXT_KEY_INTERFACES");
+ Set<Class<?>> blockedConfig = ComponentNameResolver.getWireBlockedConfigKeys();
+ assertFalse(blockedConfig.isEmpty(), "wire-blocked config-DTO list must not be empty");
+ for (Class<?> c : blockedConfig) {
+ assertFalse(all.contains(c),
+ c.getName() + " must not be both a config DTO and a context-key interface");
+ }
}
@Test
@@ -89,8 +96,7 @@ public class WireRestrictedParseContextTest {
@Test
public void restrictedRejectsExceptionReporting() {
- // Operator policy on how much exception detail leaves the server; a caller must not be
- // able to relax it per request.
+ // operator policy: a caller must not be able to relax it per request
String json = "{\"exception-reporting\":{\"level\":\"FULL\"}}";
Exception e = assertThrows(Exception.class,
() -> restrictedMapper().readValue(json, ParseContext.class));
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/CatchAllExceptionMapper.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/CatchAllExceptionMapper.java
new file mode 100644
index 0000000000..7b7efda96d
--- /dev/null
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/CatchAllExceptionMapper.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.core;
+
+import jakarta.ws.rs.WebApplicationException;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.ext.ExceptionMapper;
+import jakarta.ws.rs.ext.Provider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.tika.config.ExceptionReporting;
+import org.apache.tika.utils.ExceptionUtils;
+
+/**
+ * Last-resort mapper for exceptions no more specific mapper handles, so they don't fall
+ * through to CXF's fault chain, which echoes raw exception messages to the caller.
+ * <p>
+ * The {@link WebApplicationException} passthrough is load-bearing, not defensive. CXF returns
+ * a WAE's own response without consulting any mapper only when that response already carries
+ * an entity and the WAE has no cause ({@code support.wae.spec.optimization}). Every other WAE
+ * -- 404 and 405 from routing, an entity-less 204 or 503 a resource throws -- goes to mapper
+ * selection, where {@code default.wae.mapper.least.specific} sorts CXF's own WAE mapper last
+ * and hands it to this {@code ExceptionMapper<Throwable>}. Drop the branch and each of those
+ * becomes a 500 trace.
+ * <p>
+ * CXF's mapper logged what it handled; taking that over means logging here, or 404s and 405s
+ * vanish from the server log.
+ */
+@Provider
+public class CatchAllExceptionMapper implements ExceptionMapper<Throwable> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CatchAllExceptionMapper.class);
+
+ private final ExceptionReporting exceptionReporting;
+
+ public CatchAllExceptionMapper(ExceptionReporting exceptionReporting) {
+ this.exceptionReporting = exceptionReporting;
+ }
+
+ @Override
+ public Response toResponse(Throwable t) {
+ if (t instanceof WebApplicationException) {
+ Response response = ((WebApplicationException) t).getResponse();
+ if (response != null) {
+ if (response.getStatus() >= 500) {
+ LOG.warn("returning {} for {}", response.getStatus(), t.getClass().getName(), t);
+ } else {
+ LOG.debug("returning {} for {}", response.getStatus(), t.getClass().getName(), t);
+ }
+ return response;
+ }
+ }
+ LOG.warn("unmapped exception; returning 500", t);
+ return Response
+ .status(500)
+ .entity(ExceptionUtils.format(t, exceptionReporting))
+ .type("text/plain")
+ .build();
+ }
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerParseExceptionMapper.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerParseExceptionMapper.java
index 8456d72e1e..f82f37c9ad 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerParseExceptionMapper.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerParseExceptionMapper.java
@@ -23,9 +23,6 @@ import jakarta.ws.rs.ext.ExceptionMapper;
import jakarta.ws.rs.ext.Provider;
import org.apache.tika.config.ExceptionReporting;
-import org.apache.tika.exception.EncryptedDocumentException;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.exception.UnsupportedFormatException;
import org.apache.tika.utils.ExceptionUtils;
@Provider
@@ -34,51 +31,33 @@ public class TikaServerParseExceptionMapper implements ExceptionMapper<TikaServe
private final ExceptionReporting exceptionReporting;
public TikaServerParseExceptionMapper() {
- this(ExceptionReporting.DEFAULT);
+ this(new ExceptionReporting());
}
public TikaServerParseExceptionMapper(ExceptionReporting exceptionReporting) {
this.exceptionReporting = exceptionReporting;
}
+ /**
+ * 500 unless a {@link WebApplicationException} is in the cause chain: in 4.x a parse
+ * failure comes back as container-exception metadata from the fork, so what reaches here
+ * is a fetch/spool/IPC failure, never a document's own exception. Third-party
+ * {@code TikaServerResource} providers that wrapped a {@code WebApplicationException}
+ * chose their own response (status, headers); that passes through untouched. The cause is
+ * formatted rather than the wrapper so the body does not open with this server's own
+ * frames.
+ */
public Response toResponse(TikaServerParseException e) {
- if (e.getMessage() != null && e
- .getMessage()
- .equals(Response.Status.UNSUPPORTED_MEDIA_TYPE.toString())) {
- return buildResponse(e, 415);
- }
Throwable cause = e.getCause();
- if (cause == null) {
- return buildResponse(e, Response.Status.INTERNAL_SERVER_ERROR.getStatusCode());
- } else {
- if (cause instanceof EncryptedDocumentException) {
- return buildResponse(cause, 422);
- } else if (cause instanceof TikaException) {
- //unsupported media type
- Throwable causeOfCause = cause.getCause();
- if (causeOfCause instanceof WebApplicationException) {
- return ((WebApplicationException) causeOfCause).getResponse();
- }
- return buildResponse(cause, 422);
- } else if (cause instanceof IllegalStateException) {
- return buildResponse(cause, 422);
- } else if (cause instanceof UnsupportedFormatException) {
- return buildResponse(cause, 422);
- } else if (cause instanceof WebApplicationException) {
- return ((WebApplicationException) e.getCause()).getResponse();
- } else {
- return buildResponse(e, 500);
+ int depth = 0;
+ for (Throwable t = cause; t != null && depth < 8; t = t.getCause(), depth++) {
+ if (t instanceof WebApplicationException wae) {
+ return wae.getResponse();
}
}
- }
-
- private Response buildResponse(Throwable cause, int i) {
- if (cause == null) {
- return Response.status(i).build();
- }
return Response
- .status(i)
- .entity(ExceptionUtils.format(cause, exceptionReporting))
+ .status(500)
+ .entity(ExceptionUtils.format(cause != null ? cause : e, exceptionReporting))
.type("text/plain")
.build();
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
index 264ce728bc..01302d2c1c 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
@@ -433,6 +433,7 @@ public class TikaServerProcess {
writers.add(new TextMessageBodyWriter());
writers.addAll(loadWriterServices());
writers.add(new TikaServerParseExceptionMapper(tikaResource.getExceptionReporting()));
+ writers.add(new CatchAllExceptionMapper(tikaResource.getExceptionReporting()));
writers.add(new BadRequestExceptionMapper());
writers.add(new JSONObjWriter());
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java
index fe6ece30c2..9f140e14b2 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java
@@ -27,10 +27,8 @@ import jakarta.ws.rs.PUT;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
-import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.HttpHeaders;
-import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.MultivaluedMap;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.UriInfo;
@@ -153,13 +151,7 @@ public class MetadataResource {
String containerException = metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION);
if (containerException != null && !containerException.isEmpty()) {
- // Already policy-formatted; wrapping it as an exception message would let the
- // mapper redact it away and append this server's own frames.
- throw new WebApplicationException(Response
- .status(422)
- .entity(containerException)
- .type(MediaType.TEXT_PLAIN)
- .build());
+ throw PipesParsingHelper.containerExceptionResponse(containerException);
}
if (metadata.get(field) == null) {
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
index 13a40e06bc..9c1c1765b8 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
@@ -384,6 +384,19 @@ public class PipesParsingHelper {
}
+ /**
+ * 422 whose body is the already policy-formatted container exception. Re-wrapping it as
+ * an exception message would let the mapper redact it a second time and append this
+ * server's own frames.
+ */
+ public static WebApplicationException containerExceptionResponse(String containerException) {
+ return new WebApplicationException(Response
+ .status(422)
+ .entity(containerException)
+ .type(MediaType.TEXT_PLAIN)
+ .build());
+ }
+
/**
* Maps PipesResult status to HTTP response status. Private so no caller can map a
* status without the Retry-After headers {@link #responseBuilder} attaches.
@@ -627,11 +640,7 @@ public class PipesParsingHelper {
Metadata containerMetadata = metadataList.get(0);
String containerException = containerMetadata.get(TikaCoreProperties.CONTAINER_EXCEPTION);
if (containerException != null) {
- Response response = Response.status(422)
- .entity(containerException)
- .type("text/plain")
- .build();
- throw new WebApplicationException(response);
+ throw containerExceptionResponse(containerException);
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
index aa57ed9ef6..c590a183d6 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
@@ -236,6 +236,12 @@ public class TikaResource {
// Copy jsonConfigs for lazy resolution by parsers (e.g., pdf-parser config)
for (Map.Entry<String, JsonConfig> entry : configuredContext.getJsonConfigs().entrySet()) {
context.setJsonConfig(entry.getKey(), entry.getValue().json());
+ // Keep the jsonConfig->instance association: serialization uses it to send the
+ // original JSON rather than re-serializing the resolved instance.
+ Object resolved = configuredContext.getResolvedConfig(entry.getKey());
+ if (resolved != null) {
+ context.setResolvedConfig(entry.getKey(), resolved);
+ }
LOG.debug("Merged jsonConfig entry {} into context", entry.getKey());
}
}
@@ -360,7 +366,14 @@ public class TikaResource {
String configJson = new String(configAtt.getObject(InputStream.class).readAllBytes(),
StandardCharsets.UTF_8);
LOG.debug("setupMultipartConfig: processing config JSON of length {}", configJson.length());
- mergeParseContextFromConfig(configJson, context);
+ try {
+ mergeParseContextFromConfig(configJson, context);
+ } catch (IOException | TikaConfigException e) {
+ // Caller error (bad JSON, or a wire-blocked component): 400 with the
+ // reason, not a 500 whose reason the reporting policy may redact away.
+ throw new BadRequestException(
+ "Could not resolve the request config: " + e.getMessage(), e);
+ }
}
handedOff = true;
return tis;
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerParseExceptionMapperTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CatchAllExceptionMapperTest.java
similarity index 54%
copy from tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerParseExceptionMapperTest.java
copy to tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CatchAllExceptionMapperTest.java
index 9d9883d94d..7a96d4060e 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerParseExceptionMapperTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CatchAllExceptionMapperTest.java
@@ -20,41 +20,44 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.Test;
import org.apache.tika.config.ExceptionReporting;
-import org.apache.tika.exception.TikaException;
-public class TikaServerParseExceptionMapperTest {
+public class CatchAllExceptionMapperTest {
private static final String SECRET = "secret /path/to/file";
- private static String body(ExceptionReporting reporting) {
- Response r = new TikaServerParseExceptionMapper(reporting)
- .toResponse(new TikaServerParseException(new TikaException(SECRET)));
- assertEquals(422, r.getStatus());
- return (String) r.getEntity();
- }
-
@Test
public void fullByDefault() {
- String body = body(null);
- assertTrue(body.contains("org.apache.tika.exception.TikaException: " + SECRET), body);
+ Response r = new CatchAllExceptionMapper(new ExceptionReporting())
+ .toResponse(new RuntimeException(SECRET));
+ assertEquals(500, r.getStatus());
+ String body = (String) r.getEntity();
+ assertTrue(body.contains("java.lang.RuntimeException: " + SECRET), body);
}
@Test
public void messageRedacted() {
- String body = body(new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, -1));
- assertTrue(body.contains("org.apache.tika.exception.TikaException"), body);
+ Response r = new CatchAllExceptionMapper(
+ new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, -1))
+ .toResponse(new RuntimeException(SECRET));
+ assertEquals(500, r.getStatus());
+ String body = (String) r.getEntity();
+ assertTrue(body.contains("java.lang.RuntimeException"), body);
assertTrue(body.contains("\tat "), body);
assertFalse(body.contains(SECRET), body);
}
@Test
- public void maxLength() {
- String body = body(new ExceptionReporting(ExceptionReporting.Level.FULL, 30));
- assertTrue(body.endsWith("...[truncated]"), body);
- assertFalse(body.contains(SECRET), body);
+ public void webApplicationExceptionPassesThrough() {
+ Response original = Response.status(404).entity("not here").build();
+ Response r = new CatchAllExceptionMapper(
+ new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, -1))
+ .toResponse(new WebApplicationException(original));
+ assertEquals(404, r.getStatus());
+ assertEquals("not here", r.getEntity());
}
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RedactedStackTraceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RedactedStackTraceTest.java
index 0e699a4d1b..d0ffde87f3 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RedactedStackTraceTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RedactedStackTraceTest.java
@@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -34,9 +35,14 @@ import java.util.Map;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.jaxrs.ext.multipart.Attachment;
+import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
import org.junit.jupiter.api.Test;
@@ -55,7 +61,9 @@ import org.apache.tika.server.core.writer.TextMessageBodyWriter;
/**
* With exception-reporting set to MESSAGE_REDACTED in the server config, no channel may leak
- * an exception message: /rmeta metadata, /unpack 422 body, /meta/{field} 422 body.
+ * an exception message: /rmeta metadata, /unpack 422 body, /meta/{field} 422 body, and the
+ * catch-all 500. Also pins the mapper chain: a WebApplicationException keeps its own status,
+ * and a bad request config is a 400 with its reason rather than a redacted 500.
* {@link StackTraceTest} pins the FULL default.
*/
public class RedactedStackTraceTest extends CXFTestBase {
@@ -69,6 +77,11 @@ public class RedactedStackTraceTest extends CXFTestBase {
@TempDir
private static Path unpackTempDir;
+ @Override
+ protected boolean isAllowPerRequestConfig() {
+ return true; // /rmeta/config must reach the config gate to be rejected by it
+ }
+
@Override
protected void setUpResources(JAXRSServerFactoryBean sf) {
List<ResourceProvider> providers = new ArrayList<>();
@@ -76,6 +89,7 @@ public class RedactedStackTraceTest extends CXFTestBase {
providers.add(new SingletonResourceProvider(new RecursiveMetadataResource(tikaResource)));
providers.add(new SingletonResourceProvider(tikaResource));
providers.add(new SingletonResourceProvider(new UnpackerResource(tikaResource)));
+ providers.add(new SingletonResourceProvider(new ThrowingResource()));
sf.setResourceProviders(providers);
}
@@ -83,12 +97,38 @@ public class RedactedStackTraceTest extends CXFTestBase {
protected void setUpProviders(JAXRSServerFactoryBean sf) {
List<Object> providers = new ArrayList<>();
providers.add(new TikaServerParseExceptionMapper(tikaResource.getExceptionReporting()));
+ providers.add(new CatchAllExceptionMapper(tikaResource.getExceptionReporting()));
+ providers.add(new BadRequestExceptionMapper());
providers.add(new JSONMessageBodyWriter());
providers.add(new TextMessageBodyWriter());
providers.add(new MetadataListMessageBodyWriter());
sf.setProviders(providers);
}
+ /** No production route throws a bare RuntimeException; this one stands in for one. */
+ @jakarta.ws.rs.Path("/throwing")
+ public static class ThrowingResource {
+ @PUT
+ @Produces(MediaType.TEXT_PLAIN)
+ public String boom() {
+ throw new IllegalStateException(MESSAGE);
+ }
+ }
+
+ /** The server JVM's own policy: what the exception mappers apply to their bodies. */
+ @Override
+ protected InputStream getTikaConfigInputStream() throws IOException {
+ ObjectNode config = (ObjectNode) MAPPER.readTree(BASIC_CONFIG);
+ redact(config);
+ return new ByteArrayInputStream(
+ MAPPER.writeValueAsString(config).getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static void redact(ObjectNode config) {
+ ((ObjectNode) config.get("parse-context")).putObject("exception-reporting")
+ .put("level", "MESSAGE_REDACTED").put("maxLength", 10000);
+ }
+
@Override
protected InputStream getPipesConfigInputStream() throws IOException {
Map<String, Object> replacements = new HashMap<>();
@@ -98,8 +138,7 @@ public class RedactedStackTraceTest extends CXFTestBase {
replacements.put("TIMEOUT_MILLIS", 60000L);
JsonNode config = JsonConfigHelper.loadFromResource(UNPACK_CONFIG_TEMPLATE,
CXFTestBase.class, replacements);
- ((ObjectNode) config.get("parse-context")).putObject("exception-reporting")
- .put("level", "MESSAGE_REDACTED").put("maxLength", 10000);
+ redact((ObjectNode) config);
return new ByteArrayInputStream(
MAPPER.writeValueAsString(config).getBytes(StandardCharsets.UTF_8));
}
@@ -122,7 +161,7 @@ public class RedactedStackTraceTest extends CXFTestBase {
.put(ClassLoader.getSystemResourceAsStream(TEST_NULL));
assertEquals(200, response.getStatus());
List<Metadata> list = JsonMetadataList.fromJson(
- new java.io.InputStreamReader((InputStream) response.getEntity(),
+ new InputStreamReader((InputStream) response.getEntity(),
StandardCharsets.UTF_8));
assertRedacted(list.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION));
}
@@ -146,4 +185,53 @@ public class RedactedStackTraceTest extends CXFTestBase {
// the body is the container exception itself, not re-wrapped with server frames
assertFalse(body.contains("MetadataResource"), body);
}
+
+ /**
+ * An entity-less WebApplicationException does reach {@link CatchAllExceptionMapper} -- CXF
+ * only short-circuits WAEs that already carry a body -- so its passthrough branch is what
+ * keeps a 404 a 404 rather than a 500 trace. {@code StackTraceTest#testEmptyParser} pins
+ * the same branch for /unpack's 204.
+ */
+ @Test
+ public void unknownPathStillReturns404() throws Exception {
+ Response response = WebClient.create(endPoint + "/no-such-endpoint")
+ .accept("text/plain")
+ .put("hello");
+ assertEquals(404, response.getStatus());
+ }
+
+ /** A resource-thrown non-WAE reaches the catch-all: 500 text/plain under the policy. */
+ @Test
+ public void resourceExceptionIsRedacted500() throws Exception {
+ Response response = WebClient.create(endPoint + "/throwing")
+ .accept("text/plain")
+ .put("hello");
+ assertEquals(500, response.getStatus());
+ assertEquals("text/plain", response.getMediaType().getType() + "/"
+ + response.getMediaType().getSubtype());
+ String body = getStringFromInputStream((InputStream) response.getEntity());
+ assertTrue(body.contains("java.lang.IllegalStateException"), body);
+ assertTrue(body.contains("\tat "), body);
+ assertFalse(body.contains(MESSAGE), body);
+ }
+
+ /**
+ * A request config carrying a wire-blocked component is the caller's error: 400 with the
+ * reason. As a 500 the reason would be exactly what the policy strips.
+ */
+ @Test
+ public void wireBlockedRequestConfigIs400() throws Exception {
+ Attachment file = new Attachment("file", "application/xml",
+ ClassLoader.getSystemResourceAsStream(TEST_NULL));
+ Attachment config = new Attachment("config", "application/json",
+ new ByteArrayInputStream(
+ "{\"exception-reporting\":{\"level\":\"FULL\"}}"
+ .getBytes(StandardCharsets.UTF_8)));
+ Response response = WebClient.create(endPoint + "/rmeta/config")
+ .type("multipart/form-data")
+ .post(new MultipartBody(List.of(file, config)));
+ assertEquals(400, response.getStatus());
+ String body = getStringFromInputStream((InputStream) response.getEntity());
+ assertTrue(body.contains("may not be supplied via a request parseContext"), body);
+ }
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
index c0e5b8226b..80081c850b 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
@@ -86,6 +86,9 @@ public class StackTraceTest extends CXFTestBase {
protected void setUpProviders(JAXRSServerFactoryBean sf) {
List<Object> providers = new ArrayList<>();
providers.add(new TikaServerParseExceptionMapper());
+ // production registers this too, and CXF hands it every WebApplicationException;
+ // without it here the 422/404 paths below are exercised in a different chain
+ providers.add(new CatchAllExceptionMapper(tikaResource.getExceptionReporting()));
providers.add(new JSONMessageBodyWriter());
providers.add(new CSVMessageBodyWriter());
//providers.add(new XMPMessageBodyWriter());
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerParseExceptionMapperTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerParseExceptionMapperTest.java
index 9d9883d94d..9b9d9fa52b 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerParseExceptionMapperTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerParseExceptionMapperTest.java
@@ -20,41 +20,83 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.Test;
import org.apache.tika.config.ExceptionReporting;
-import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.core.PipesException;
+/**
+ * The live shapes: a {@code PipesException} cause (PipesParsingHelper) and a message-only
+ * wrapper (interrupted parse). Both are 500, and the body is the cause, not this server's
+ * wrapper.
+ */
public class TikaServerParseExceptionMapperTest {
private static final String SECRET = "secret /path/to/file";
- private static String body(ExceptionReporting reporting) {
- Response r = new TikaServerParseExceptionMapper(reporting)
- .toResponse(new TikaServerParseException(new TikaException(SECRET)));
- assertEquals(422, r.getStatus());
- return (String) r.getEntity();
+ private static Response respond(ExceptionReporting reporting) {
+ return new TikaServerParseExceptionMapper(reporting)
+ .toResponse(new TikaServerParseException(new PipesException(SECRET)));
}
@Test
public void fullByDefault() {
- String body = body(null);
- assertTrue(body.contains("org.apache.tika.exception.TikaException: " + SECRET), body);
+ Response r = respond(new ExceptionReporting());
+ assertEquals(500, r.getStatus());
+ String body = (String) r.getEntity();
+ assertTrue(body.startsWith("org.apache.tika.pipes.core.PipesException: " + SECRET), body);
+ // the wrapper is not the first line and contributes no frames of its own
+ assertFalse(body.contains("TikaServerParseException:"), body);
}
@Test
public void messageRedacted() {
- String body = body(new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, -1));
- assertTrue(body.contains("org.apache.tika.exception.TikaException"), body);
+ Response r = respond(new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, -1));
+ assertEquals(500, r.getStatus());
+ String body = (String) r.getEntity();
+ assertTrue(body.startsWith("org.apache.tika.pipes.core.PipesException"), body);
assertTrue(body.contains("\tat "), body);
assertFalse(body.contains(SECRET), body);
}
@Test
public void maxLength() {
- String body = body(new ExceptionReporting(ExceptionReporting.Level.FULL, 30));
+ // 50 cuts mid-message: the start of SECRET survives but not all of it
+ Response r = respond(new ExceptionReporting(ExceptionReporting.Level.FULL, 50));
+ String body = (String) r.getEntity();
assertTrue(body.endsWith("...[truncated]"), body);
+ assertTrue(body.contains("secret"), body);
assertFalse(body.contains(SECRET), body);
}
+
+ @Test
+ public void noCauseFallsBackToTheWrapper() {
+ Response r = new TikaServerParseExceptionMapper(new ExceptionReporting())
+ .toResponse(new TikaServerParseException("Parsing interrupted"));
+ assertEquals(500, r.getStatus());
+ String body = (String) r.getEntity();
+ assertTrue(body.contains("Parsing interrupted"), body);
+ }
+
+ @Test
+ public void webApplicationExceptionCausePassesThrough() {
+ // a third-party TikaServerResource provider chose its own response; keep it
+ Response chosen = Response.status(429).header("Retry-After", "7").build();
+ Response r = new TikaServerParseExceptionMapper(new ExceptionReporting())
+ .toResponse(new TikaServerParseException(
+ new WebApplicationException(chosen)));
+ assertEquals(429, r.getStatus());
+ assertEquals("7", r.getHeaderString("Retry-After"));
+ }
+
+ @Test
+ public void nestedWebApplicationExceptionCausePassesThrough() {
+ Response chosen = Response.status(503).build();
+ Response r = new TikaServerParseExceptionMapper(new ExceptionReporting())
+ .toResponse(new TikaServerParseException(
+ new RuntimeException(new WebApplicationException(chosen))));
+ assertEquals(503, r.getStatus());
+ }
}