You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/07/29 16:07:07 UTC
(tika) branch shim-for-tika-eval-for-pre-4.0.0 created (now 3ae1144884)
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a change to branch shim-for-tika-eval-for-pre-4.0.0
in repository https://gitbox.apache.org/repos/asf/tika.git
at 3ae1144884 add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1
This branch includes the following new commits:
new 3ae1144884 add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tika) 01/01: add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch shim-for-tika-eval-for-pre-4.0.0
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 3ae11448842efad649b3baa895efb044caefdb82
Author: tallison <ta...@apache.org>
AuthorDate: Wed Jul 29 12:06:53 2026 -0400
add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1
---
.../org/apache/tika/eval/app/io/ExtractReader.java | 76 ++++++++++++++++++++++
.../apache/tika/eval/app/io/ExtractReaderTest.java | 34 ++++++++++
.../test-dirs/legacy/beta1-style.doc.json | 17 +++++
.../resources/test-dirs/legacy/collision.doc.json | 7 ++
4 files changed, 134 insertions(+)
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
index 26ff7f22cf..bac92b48e4 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
@@ -25,8 +25,10 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -159,6 +161,9 @@ public class ExtractReader {
try {
if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) {
metadataList = JsonMetadataList.fromJson(reader);
+ for (Metadata m : metadataList) {
+ normalizeLegacyKeys(m);
+ }
if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) {
while (metadataList.size() > 1) {
metadataList.remove(metadataList.size() - 1);
@@ -214,6 +219,77 @@ public class ExtractReader {
}
+ // Pre-4.0 extract key -> 4.0 key, for the Tika-native fields tika-eval reads. Digest keys are
+ // handled by the prefix rule in normalizeLegacyKeys; Content-Type/Content-Length are standard
+ // names (unchanged) so they are not listed. New-side keys come from the live constants so this
+ // can't drift from the 4.0 declarations.
+ private static final Map<String, String> LEGACY_KEY_MAP = Map.ofEntries(
+ Map.entry("X-TIKA:content", TikaCoreProperties.TIKA_CONTENT.getName()),
+ Map.entry("X-TIKA:content_handler", TikaCoreProperties.TIKA_CONTENT_HANDLER.getName()),
+ Map.entry("X-TIKA:embedded_depth", TikaCoreProperties.EMBEDDED_DEPTH.getName()),
+ Map.entry("X-TIKA:embedded_resource_path", TikaCoreProperties.EMBEDDED_RESOURCE_PATH.getName()),
+ Map.entry("X-TIKA:final_embedded_resource_path", TikaCoreProperties.FINAL_EMBEDDED_RESOURCE_PATH.getName()),
+ Map.entry("X-TIKA:parse_time_millis", TikaCoreProperties.PARSE_TIME_MILLIS.getName()),
+ Map.entry("X-TIKA:resourceName", TikaCoreProperties.RESOURCE_NAME_KEY.getName()),
+ Map.entry("X-TIKA:detectedEncoding", TikaCoreProperties.DETECTED_ENCODING.getName()),
+ Map.entry("X-TIKA:encodingDetector", TikaCoreProperties.ENCODING_DETECTOR.getName()),
+ Map.entry("Content-Type-Hint", TikaCoreProperties.CONTENT_TYPE_HINT.getName()),
+ Map.entry("embeddedResourceType", TikaCoreProperties.EMBEDDED_RESOURCE_TYPE.getName()),
+ Map.entry("X-TIKA:EXCEPTION:container_exception", TikaCoreProperties.CONTAINER_EXCEPTION.getName()),
+ Map.entry("X-TIKA:EXCEPTION:embedded_exception", TikaCoreProperties.EMBEDDED_EXCEPTION.getName()));
+
+ private static final String LEGACY_DIGEST_PREFIX = TikaCoreProperties.LEGACY_TIKA_META_PREFIX
+ + "digest" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
+
+ /**
+ * Pre-4.0 extracts (e.g. 4.0.0-beta-1) key Tika-native fields under X-TIKA:/camelCase names.
+ * Normalize the fields tika-eval reads to their 4.0 tk: keys so a cross-version compare reflects
+ * real diffs, not the rename. Harmless on 4.0 extracts: the legacy keys are simply absent.
+ */
+ private static void normalizeLegacyKeys(Metadata m) {
+ m.setTrusted(true); // sanctioned trusted transformation: may write reserved tk: keys
+ try {
+ for (Map.Entry<String, String> e : LEGACY_KEY_MAP.entrySet()) {
+ remapLegacyKey(m, e.getKey(), e.getValue());
+ }
+ // digest keys: X-TIKA:digest:<alg> -> tk:digest:<alg> (algorithm unchanged; MD5 drives
+ // embedded-doc matching). names() is a snapshot, so remapping while iterating is safe.
+ for (String name : m.names()) {
+ if (name.startsWith(LEGACY_DIGEST_PREFIX)) {
+ remapLegacyKey(m, name, TikaCoreProperties.TIKA_META_PREFIX
+ + name.substring(TikaCoreProperties.LEGACY_TIKA_META_PREFIX.length()));
+ }
+ }
+ } finally {
+ m.setTrusted(false);
+ }
+ }
+
+ private static void remapLegacyKey(Metadata m, String legacyKey, String modernKey) {
+ String[] legacyVals = m.getValues(legacyKey);
+ if (legacyVals.length == 0) {
+ return;
+ }
+ String[] modernVals = m.getValues(modernKey);
+ if (modernVals.length > 0) {
+ // Both present: safe only if identical. Fail loud rather than silently clobber a value.
+ if (!Arrays.equals(legacyVals, modernVals)) {
+ throw new IllegalStateException("Extract has both legacy key '" + legacyKey
+ + "' and modern key '" + modernKey + "' with different values; legacy-key "
+ + "normalization would clobber. Extract is inconsistent.");
+ }
+ } else {
+ for (int i = 0; i < legacyVals.length; i++) {
+ if (i == 0) {
+ m.set(modernKey, legacyVals[i]);
+ } else {
+ m.add(modernKey, legacyVals[i]);
+ }
+ }
+ }
+ m.remove(legacyKey);
+ }
+
public enum ALTER_METADATA_LIST {
AS_IS, //leave the metadata list as is
FIRST_ONLY, //take only the metadata list for the "container" document
diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java
index 8c33d42ffc..e164bfe4fa 100644
--- a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java
+++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java
@@ -18,6 +18,8 @@ package org.apache.tika.eval.app.io;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.IOException;
import java.nio.file.Path;
@@ -88,6 +90,38 @@ public class ExtractReaderTest extends TikaTest {
.get(TikaCoreProperties.TIKA_CONTENT));
}
+ @Test
+ public void testLegacyKeyNormalization() throws Exception {
+ // a pre-4.0 (4.0.0-beta-1 style) extract: X-TIKA:/camelCase Tika-native keys
+ Path f = getResourceAsFile("/test-dirs/legacy/beta1-style.doc.json").toPath();
+ List<Metadata> list = new ExtractReader().loadExtract(f);
+ assertEquals(2, list.size());
+
+ Metadata container = list.get(0);
+ // readable via the 4.0 Property constants that tika-eval uses
+ assertEquals("the quick brown fox", container.get(TikaCoreProperties.TIKA_CONTENT));
+ assertEquals("12", container.get(TikaCoreProperties.PARSE_TIME_MILLIS));
+ assertEquals("boom", container.get(TikaCoreProperties.CONTAINER_EXCEPTION));
+ assertEquals("abc123", container.get("tk:digest:MD5"));
+ // the legacy keys are gone
+ assertNull(container.get("X-TIKA:content"));
+ assertNull(container.get("X-TIKA:digest:MD5"));
+
+ Metadata embedded = list.get(1);
+ assertEquals("inner.txt", embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
+ assertEquals("1", embedded.get(TikaCoreProperties.EMBEDDED_DEPTH));
+ assertEquals("inner.txt", embedded.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+ assertEquals("ATTACHMENT", embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ assertNull(embedded.get("X-TIKA:embedded_resource_path"));
+ }
+
+ @Test
+ public void testLegacyNormalizationCollisionFailsLoud() throws Exception {
+ // both the legacy and modern content key present with different values -> must not clobber
+ Path f = getResourceAsFile("/test-dirs/legacy/collision.doc.json").toPath();
+ assertThrows(IllegalStateException.class, () -> new ExtractReader().loadExtract(f));
+ }
+
@Test
public void testTextBasic() throws IOException {
ExtractReader extractReader = new ExtractReader();
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json
new file mode 100644
index 0000000000..d8938f4a06
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json
@@ -0,0 +1,17 @@
+[
+ {
+ "Content-Type" : "text/plain",
+ "X-TIKA:content" : "the quick brown fox",
+ "X-TIKA:parse_time_millis" : "12",
+ "X-TIKA:digest:MD5" : "abc123",
+ "X-TIKA:EXCEPTION:container_exception" : "boom"
+ },
+ {
+ "Content-Type" : "text/plain",
+ "X-TIKA:embedded_resource_path" : "inner.txt",
+ "X-TIKA:content" : "attachment contents",
+ "X-TIKA:embedded_depth" : "1",
+ "X-TIKA:resourceName" : "inner.txt",
+ "embeddedResourceType" : "ATTACHMENT"
+ }
+]
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json
new file mode 100644
index 0000000000..47c2b342e1
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json
@@ -0,0 +1,7 @@
+[
+ {
+ "Content-Type" : "text/plain",
+ "X-TIKA:content" : "legacy value",
+ "tk:content" : "modern value"
+ }
+]