You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/04/13 17:28:17 UTC
(tika) branch main updated (4fbd3ec9a6 -> 86a4f3e02b)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
from 4fbd3ec9a6 TIKA-4327: update junrar
new 155a620714 ebcdic gate
new 9ce41f5c70 win-1252 hack
new 41784517b5 sparse-Latin vCard IBM424 false positive test
new d370f54247 bump limit to something realistic
new d28c9b1f71 gate UTF-16 model output on 2-byte column-diversity asymmetry
new 86a4f3e02b Merge remote-tracking branch 'origin/main' into 4x-reg-test-charset-detection-tweaks
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../ml/chardetect/MojibusterEncodingDetector.java | 145 ++++++++++++++++++++-
.../ml/chardetect/StructuralEncodingRules.java | 131 +++++++++++++++++++
.../chardetect/SparseLatinVcardRegressionTest.java | 116 +++++++++++++++++
3 files changed, 388 insertions(+), 4 deletions(-)
create mode 100644 tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/SparseLatinVcardRegressionTest.java
(tika) 05/06: gate UTF-16 model output on 2-byte column-diversity asymmetry
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit d28c9b1f7107192beb3f6ec2fcf0f5bbf5a03502
Author: tallison <ta...@apache.org>
AuthorDate: Sun Apr 12 16:36:48 2026 -0400
gate UTF-16 model output on 2-byte column-diversity asymmetry
---
.../ml/chardetect/MojibusterEncodingDetector.java | 13 +++-
.../ml/chardetect/StructuralEncodingRules.java | 78 ++++++++++++++++++++++
2 files changed, 90 insertions(+), 1 deletion(-)
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index a0c56393c8..3a253f42d3 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -430,8 +430,19 @@ public class MojibusterEncodingDetector implements EncodingDetector {
boolean excludeUtf8 = gates
&& StructuralEncodingRules.checkUtf8(probe) == StructuralEncodingRules.Utf8Result.NOT_UTF8;
+ // UTF-16 structural gate: stride-2 bigram features can misfire on
+ // non-UTF-16 probes with scattered nulls (e.g. Greek plaintext with
+ // 0.3% nulls scoring as UTF-16-LE). Real UTF-16 of any script has a
+ // concentrated byte column paired with a diverse one; scattered nulls
+ // produce ~balanced column diversity. Mask UTF-16 labels when the
+ // column-asymmetry test fails.
+ boolean utf16Plausible = !gates
+ || StructuralEncodingRules.has2ByteColumnAsymmetry(probe);
+ boolean excludeUtf16Be = wideResult.invalidUtf16Be || !utf16Plausible;
+ boolean excludeUtf16Le = wideResult.invalidUtf16Le || !utf16Plausible;
+
List<EncodingResult> results = runModel(probe, excludeUtf8,
- wideResult.invalidUtf16Be, wideResult.invalidUtf16Le, topN);
+ excludeUtf16Be, excludeUtf16Le, topN);
// If the model had no evidence (probe too short or all tokens filtered), fall back to
// windows-1252 at very low confidence rather than returning empty and letting
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
index 101e00484c..beaffc7475 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
@@ -255,6 +255,84 @@ public final class StructuralEncodingRules {
return ebcdicSpace >= sample * 0.03 && ebcdicSpace > asciiSpace * 3;
}
+ /**
+ * Minimum probe length before {@link #has2ByteColumnAsymmetry} produces
+ * meaningful diversity counts. Short probes or probes with limited
+ * vocabulary may have too few distinct byte values per column to compare
+ * reliably; on anything below this threshold we fall back to the pre-gate
+ * behaviour (model + {@link WideUnicodeDetector} positive signal). Set
+ * above the size of typical short probes (a few hundred bytes) so real
+ * CJK UTF-16 text has room to diversify its high-byte column.
+ */
+ public static final int MIN_COLUMN_ASYMMETRY_PROBE = 2048;
+
+ /**
+ * Returns {@code true} if the probe's byte distribution across stride-2
+ * columns is sufficiently asymmetric to be plausible UTF-16 of some script.
+ *
+ * <p>Every UTF-16 variant has one byte column concentrated in a
+ * script-specific Unicode block prefix while the other column is diverse:
+ * UTF-16 Latin pairs to {@code (ascii, 0x00)} so one column is {@code 0x00}
+ * (1 value) vs ASCII range (~70 values); UTF-16 Cyrillic / Greek / Arabic /
+ * Hebrew pair to a single high-byte block prefix ({@code 0x04}, {@code 0x03},
+ * {@code 0x06}, {@code 0x05}); UTF-16 CJK Unified uses {@code 0x4E}-{@code 0x9F}
+ * (~80 distinct high bytes) against ~256 low bytes; Hangul uses
+ * {@code 0xAC}-{@code 0xD7} (~44 high bytes).</p>
+ *
+ * <p>Non-UTF-16 text — including scattered-null binaries and mixed-content
+ * files — has roughly balanced column diversity (both columns saturate
+ * near 256 distinct byte values on long probes).</p>
+ *
+ * <p>This is a <em>negative</em> gate: when it returns {@code false}, the
+ * probe cannot be any UTF-16 variant, and UTF-16 labels should be masked
+ * from model output even when the stride-2 bigram features score them
+ * highly (e.g. a Greek plaintext file with 0.36% scattered nulls being
+ * mis-scored as UTF-16-LE).</p>
+ *
+ * <p>A diversity ratio of 3× (more diverse column has at least 3× as many
+ * distinct values as the more concentrated column) admits all UTF-16
+ * variants including CJK (ratio ~3.2) while rejecting scattered-null
+ * false positives (ratio ~1:1).</p>
+ *
+ * <p>For probes shorter than {@link #MIN_COLUMN_ASYMMETRY_PROBE}, this
+ * method returns {@code true} conservatively — column counts from short
+ * samples are not statistically meaningful, so the caller should rely on
+ * {@link WideUnicodeDetector} positive signal and downstream CharSoup
+ * arbitration rather than masking.</p>
+ *
+ * @param bytes the probe to analyse
+ * @return {@code true} if the probe has UTF-16-compatible column asymmetry
+ * (or is too short to judge); {@code false} if column diversity is
+ * too balanced to be any UTF-16 variant
+ */
+ public static boolean has2ByteColumnAsymmetry(byte[] bytes) {
+ if (bytes == null || bytes.length < MIN_COLUMN_ASYMMETRY_PROBE) {
+ return true;
+ }
+ int sample = Math.min(bytes.length, 4096);
+ boolean[] evenSeen = new boolean[256];
+ boolean[] oddSeen = new boolean[256];
+ int evenDistinct = 0;
+ int oddDistinct = 0;
+ for (int i = 0; i < sample; i++) {
+ int v = bytes[i] & 0xFF;
+ if ((i & 1) == 0) {
+ if (!evenSeen[v]) {
+ evenSeen[v] = true;
+ evenDistinct++;
+ }
+ } else {
+ if (!oddSeen[v]) {
+ oddSeen[v] = true;
+ oddDistinct++;
+ }
+ }
+ }
+ int min = Math.min(evenDistinct, oddDistinct);
+ int max = Math.max(evenDistinct, oddDistinct);
+ return max >= min * 3;
+ }
+
public static boolean checkIbm424(byte[] bytes, int offset, int length) {
if (length < 8) {
return false;
(tika) 03/06: sparse-Latin vCard IBM424 false positive test
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 41784517b5fb987332206a7257a5a01fd23a0334
Author: tallison <ta...@apache.org>
AuthorDate: Sun Apr 12 15:51:20 2026 -0400
sparse-Latin vCard IBM424 false positive test
---
.../chardetect/SparseLatinVcardRegressionTest.java | 116 +++++++++++++++++++++
1 file changed, 116 insertions(+)
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/SparseLatinVcardRegressionTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/SparseLatinVcardRegressionTest.java
new file mode 100644
index 0000000000..16b49346fc
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/SparseLatinVcardRegressionTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.detect.EncodingResult;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * Regression test for the sparse-Latin vCard / config-file detection
+ * class.
+ *
+ * <p>Before the {@link StructuralEncodingRules#isEbcdicLikely(byte[])}
+ * gate and the {@link MojibusterEncodingDetector.Rule#LOSSLESS_WIN1252_CANONICALISATION}
+ * post-rule, a predominantly-ASCII probe with a small number of
+ * Latin-supplement high bytes (e.g. a vCard containing a German
+ * business name) detected as {@code IBM424} (Hebrew EBCDIC) at 0.99
+ * confidence — producing complete mojibake with dice=0 vs the 3.x
+ * baseline.</p>
+ *
+ * <p>After the fixes, the same probe detects as {@code windows-1252},
+ * preserving content fidelity.</p>
+ */
+public class SparseLatinVcardRegressionTest {
+
+ /**
+ * End-to-end regression assertion: the synthetic sparse-Latin vCard
+ * must detect as {@code windows-1252}, not {@code IBM424} or a
+ * byte-equivalent {@code windows-1257 / windows-1254 / x-MacRoman}
+ * sibling.
+ */
+ @Test
+ public void sparseLatinVcardDetectsAsWindows1252() throws Exception {
+ byte[] probe = buildSparseLatinVcard();
+
+ MojibusterEncodingDetector detector = new MojibusterEncodingDetector();
+ try (TikaInputStream tis = TikaInputStream.get(probe)) {
+ List<EncodingResult> results = detector.detect(
+ tis, new Metadata(), new ParseContext());
+ assertFalse(results.isEmpty(),
+ "Detector must return at least one candidate");
+ assertEquals("windows-1252", results.get(0).getCharset().name(),
+ "Sparse-Latin vCard must detect as windows-1252, not "
+ + "IBM424 / windows-1257 / windows-1254 / x-MacRoman");
+ }
+ }
+
+ /**
+ * Synthetic vCard-shaped probe that reproduces the regression class.
+ *
+ * <p>Preserved byte statistics from the original failing file:
+ * <ul>
+ * <li>Length in the 400-600 byte range (long-probe path).</li>
+ * <li>Exactly 3 non-ASCII bytes, all {@code 0xE4} — 'ä' under
+ * ISO-8859-1 / windows-1252 / windows-1257. The extreme-sparse
+ * regime where the flat statistical model was overconfidently
+ * wrong.</li>
+ * <li>Zero C1 bytes ({@code 0x80–0x9F}) so ISO→Windows upgrade
+ * does not fire.</li>
+ * <li>LF line endings only (no CRLF) so CRLF→Windows upgrade
+ * does not fire.</li>
+ * <li>Zero {@code 0x40} bytes so the EBCDIC gate cleanly returns
+ * {@code false}.</li>
+ * </ul>
+ *
+ * <p>Content is a fictitious German bakery at a fictitious address.
+ * No real business or person is represented.</p>
+ */
+ private static byte[] buildSparseLatinVcard() {
+ String vcard =
+ "BEGIN:VCARD\n"
+ + "\t\t\t\t\tVERSION:3.0\n"
+ + "\t\t\t\t\tN:Example B\u00E4ckerei GmbH\n"
+ + "\t\t\t\t\tFN:Example B\u00E4ckerei GmbH\n"
+ + "\t\t\t\t\tORG:Example B\u00E4ckerei GmbH;\n"
+ + "\t\t\t\t\tPHOTO;VALUE=URL;TYPE=jpg:"
+ + "https://example.com/images/logo.jpg\n"
+ + "\t\t\t\t\titem1.EMAIL;TYPE=PREF,INTERNET:\n"
+ + "\t\t\t\t\titem1.X-ABLabel:email\n"
+ + "\t\t\t\t\tTEL;TYPE=WORK,VOICE:\n"
+ + "\t\t\t\t\tTEL;TYPE=WORK,FAX:\n"
+ + "\t\t\t\t\titem2.ADR;TYPE=WORK:"
+ + ";;Teststr. 1;Musterstadt;;12345;Germany;\n"
+ + "\t\t\t\t\titem2.X-ABADR:de\n"
+ + "\t\t\t\t\tLABEL;TYPE=WORK:Teststr. 1 Musterstadt, 12345\n"
+ + "\t\t\t\t\tURL;TYPE=PREF:\n"
+ + "\t\t\t\t\tREV:2026-04-12 12:00:00\n"
+ + "\t\t\t\t\tNOTE:Synthetic test fixture for charset "
+ + "detector regression coverage\n"
+ + "\t\t\t\t\tEND:VCARD\n";
+ return vcard.getBytes(StandardCharsets.ISO_8859_1);
+ }
+}
(tika) 06/06: Merge remote-tracking branch 'origin/main' into 4x-reg-test-charset-detection-tweaks
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 86a4f3e02bd5cf0d1173127e5b8f3ac225f7c399
Merge: d28c9b1f71 4fbd3ec9a6
Author: tallison <ta...@apache.org>
AuthorDate: Mon Apr 13 13:28:00 2026 -0400
Merge remote-tracking branch 'origin/main' into 4x-reg-test-charset-detection-tweaks
tika-parent/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
(tika) 01/06: ebcdic gate
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 155a62071491763c6a07d0d36efdfcd37ce28f2f
Author: tallison <ta...@apache.org>
AuthorDate: Sun Apr 12 08:00:22 2026 -0400
ebcdic gate
---
.../ml/chardetect/MojibusterEncodingDetector.java | 34 ++++++++++++++
.../ml/chardetect/StructuralEncodingRules.java | 53 ++++++++++++++++++++++
2 files changed, 87 insertions(+)
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index 6f6590e38d..b232f884f0 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -422,6 +422,17 @@ public class MojibusterEncodingDetector implements EncodingDetector {
int[] features = extractor.extract(probe);
float[] logits = model.predictLogits(features);
+ // EBCDIC gate: if the probe lacks the EBCDIC word-separator pattern
+ // (0x40 dominant over 0x20), it cannot be any EBCDIC variant. The
+ // statistical model can produce very large logits for EBCDIC labels
+ // on predominantly-ASCII probes whose n-grams happen to align with
+ // training features (observed with 99%-ASCII vCards mis-scored at
+ // IBM424 logit 55 vs windows-1252 logit 26). Mask those labels out
+ // before ranking so downstream arbitration sees only plausible
+ // candidates.
+ boolean excludeEbcdic = enabledRules.contains(Rule.STRUCTURAL_GATES)
+ && !StructuralEncodingRules.isEbcdicLikely(probe);
+
for (int i = 0; i < logits.length; i++) {
String lbl = model.getLabel(i);
if (excludeUtf8 && "UTF-8".equalsIgnoreCase(lbl)) {
@@ -433,6 +444,9 @@ public class MojibusterEncodingDetector implements EncodingDetector {
if (excludeUtf16Le && lbl.equalsIgnoreCase("UTF-16-LE")) {
logits[i] = Float.NEGATIVE_INFINITY;
}
+ if (excludeEbcdic && isEbcdicLabel(lbl)) {
+ logits[i] = Float.NEGATIVE_INFINITY;
+ }
}
List<EncodingResult> results = selectByLogitGap(model, logits, topN);
@@ -706,6 +720,26 @@ public class MojibusterEncodingDetector implements EncodingDetector {
}
}
+ /**
+ * True EBCDIC variants that must be gated by {@link
+ * StructuralEncodingRules#isEbcdicLikely(byte[])}.
+ *
+ * <p>Note: {@code IBM850}, {@code IBM852}, {@code IBM855}, {@code IBM866},
+ * and {@code IBM437} are DOS/OEM code pages, <em>not</em> EBCDIC — they
+ * use {@code 0x20} for space like ASCII and are therefore not gated.</p>
+ */
+ private static boolean isEbcdicLabel(String label) {
+ if (label == null) {
+ return false;
+ }
+ return label.equals("IBM420-ltr") || label.equals("IBM420-rtl")
+ || label.equals("IBM420")
+ || label.equals("IBM424-ltr") || label.equals("IBM424-rtl")
+ || label.equals("IBM424")
+ || label.equals("IBM500")
+ || label.equals("IBM1047");
+ }
+
private static byte[] readProbe(TikaInputStream is, int maxBytes) throws IOException {
is.mark(maxBytes);
try {
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
index a7114527b0..101e00484c 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
@@ -202,6 +202,59 @@ public final class StructuralEncodingRules {
return checkIbm424(bytes, 0, bytes.length);
}
+ /**
+ * Returns {@code true} if the probe is plausibly EBCDIC based on the
+ * word-separator distribution. In every EBCDIC variant (IBM420, IBM424,
+ * IBM500, IBM1047) the space character is {@code 0x40}, not {@code 0x20};
+ * a stretch of EBCDIC text therefore has {@code 0x40} as its single most
+ * common byte, at roughly 10–20% of the sample. Conversely, any ASCII
+ * or ISO-8859-X / windows-12XX / DOS / Mac / CJK text uses {@code 0x20}
+ * (or {@code 0x09 / 0x0A}) as its whitespace and has {@code 0x40} only
+ * as the rare {@code @} character (typically less than 0.1% of bytes).
+ *
+ * <p>This is a <em>negative</em> gate: when it returns {@code false}, the
+ * probe cannot be any EBCDIC variant, and downstream scoring should
+ * exclude EBCDIC labels from consideration even if the statistical model
+ * ranks them highly.</p>
+ *
+ * <p>Threshold rationale: we require both (a) {@code 0x40} at least 3%
+ * of the sample and (b) {@code 0x40} at least 3× more frequent
+ * than {@code 0x20}. Gate (b) alone is not sufficient because sparse
+ * binary content can have neither byte; gate (a) alone is not sufficient
+ * because some text formats (CSV with {@code @}-separated fields,
+ * e-mail address lists) can exceed 3% {@code 0x40} while clearly being
+ * ASCII-spaced. Both gates together match real EBCDIC text reliably
+ * across IBM420/424/500/1047 variants.</p>
+ *
+ * @param bytes the probe to analyse
+ * @return {@code true} if the probe's whitespace distribution is
+ * consistent with EBCDIC; {@code false} if it is clearly ASCII-spaced
+ */
+ public static boolean isEbcdicLikely(byte[] bytes) {
+ if (bytes == null || bytes.length < 8) {
+ return false;
+ }
+ int sample = Math.min(bytes.length, 4096);
+ int ebcdicSpace = 0;
+ int asciiSpace = 0;
+ int prev = -1;
+ for (int i = 0; i < sample; i++) {
+ int b = bytes[i] & 0xFF;
+ if (b == 0x40) {
+ // Guard against Shift_JIS trail bytes that happen to equal 0x40.
+ boolean isShiftJisTrail = (prev >= 0x81 && prev <= 0x9F)
+ || (prev >= 0xE0 && prev <= 0xFC);
+ if (!isShiftJisTrail) {
+ ebcdicSpace++;
+ }
+ } else if (b == 0x20) {
+ asciiSpace++;
+ }
+ prev = b;
+ }
+ return ebcdicSpace >= sample * 0.03 && ebcdicSpace > asciiSpace * 3;
+ }
+
public static boolean checkIbm424(byte[] bytes, int offset, int length) {
if (length < 8) {
return false;
(tika) 02/06: win-1252 hack
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 9ce41f5c70f8cc349d39e9a3fc1e76e859fe06ab
Author: tallison <ta...@apache.org>
AuthorDate: Sun Apr 12 11:56:14 2026 -0400
win-1252 hack
---
.../ml/chardetect/MojibusterEncodingDetector.java | 90 +++++++++++++++++++++-
1 file changed, 89 insertions(+), 1 deletion(-)
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index b232f884f0..9cc1f7cf32 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -106,7 +106,29 @@ public class MojibusterEncodingDetector implements EncodingDetector {
* line ending) does <em>not</em> trigger this rule. Mirrors the legacy
* {@code UniversalEncodingListener.report()} heuristic.</p>
*/
- CRLF_TO_WINDOWS
+ CRLF_TO_WINDOWS,
+ /**
+ * Lossless canonicalisation to {@code windows-1252}: if the top
+ * candidate is a single-byte Latin-family charset (not CJK, EBCDIC,
+ * Hebrew, Arabic, Cyrillic, or Greek) and decoding the probe with
+ * both that charset and {@code windows-1252} produces character-for-
+ * character identical strings, relabel the result as
+ * {@code windows-1252}.
+ *
+ * <p>Why: the statistical model is often confident in a sibling
+ * Latin-family charset (windows-1254 Turkish, windows-1257 Baltic,
+ * x-MacRoman, ISO-8859-X) when the probe contains only bytes that
+ * decode to the same characters under {@code windows-1252}. For
+ * the bytes actually present in the probe, the label difference is
+ * cosmetic. Canonicalising to {@code windows-1252} matches the
+ * WHATWG default, matches pre-4.x Tika behaviour for ASCII-adjacent
+ * Western text, and does not alter the decoded output. When the
+ * charsets disagree on any byte actually present (e.g. real
+ * Japanese bytes under Shift_JIS, or Cyrillic under KOI8-R),
+ * the decoded strings differ and the rule does not fire, preserving
+ * the model's decision.</p>
+ */
+ LOSSLESS_WIN1252_CANONICALISATION
}
private static final long serialVersionUID = 1L;
@@ -476,10 +498,76 @@ public class MojibusterEncodingDetector implements EncodingDetector {
&& StructuralEncodingRules.hasGb18030FourByteSequence(probe)) {
results = upgradeGbToGb18030(results);
}
+ if (enabledRules.contains(Rule.LOSSLESS_WIN1252_CANONICALISATION)) {
+ results = losslessWin1252Canonicalise(probe, results);
+ }
// Trim to topN after all rules have fired, not before.
return results.subList(0, Math.min(topN, results.size()));
}
+ /**
+ * Labels for which {@link Rule#LOSSLESS_WIN1252_CANONICALISATION} may fire.
+ * Only single-byte Latin-family code pages are candidates; CJK, Cyrillic,
+ * Greek, Hebrew, Arabic, and Thai charsets are excluded because differing
+ * labels there imply different scripts, not cosmetic variation.
+ *
+ * <p>windows-1252 itself is included so it is a no-op when already chosen.
+ * ISO-8859-1 is included because it is a strict subset of windows-1252 for
+ * the 0xA0-0xFF range (they differ only in 0x80-0x9F, where ISO-8859-1
+ * defines C1 control codes). ISO-8859-15 differs from 1252 at 8 code
+ * points used for Euro / OE / S-caron / Z-caron; the byte-level equality
+ * check catches those automatically.</p>
+ */
+ private static final java.util.Set<String> WIN1252_CANONICALISABLE_LABELS =
+ java.util.Set.of(
+ "windows-1252", "ISO-8859-1", "ISO-8859-15",
+ "windows-1250", "ISO-8859-2",
+ "windows-1254", "ISO-8859-9",
+ "windows-1257", "ISO-8859-4", "ISO-8859-13",
+ "ISO-8859-3", "ISO-8859-16",
+ "x-MacRoman");
+
+ /**
+ * If the top result's label is in {@link #WIN1252_CANONICALISABLE_LABELS}
+ * and decoding the probe with that charset produces the same string as
+ * decoding with {@code windows-1252}, replace the top result with a
+ * windows-1252 result at the same confidence. See {@link
+ * Rule#LOSSLESS_WIN1252_CANONICALISATION}.
+ */
+ private static List<EncodingResult> losslessWin1252Canonicalise(byte[] probe,
+ List<EncodingResult> results) {
+ if (results.isEmpty()) {
+ return results;
+ }
+ EncodingResult top = results.get(0);
+ String topLabel = top.getLabel();
+ if (topLabel == null || !WIN1252_CANONICALISABLE_LABELS.contains(topLabel)) {
+ return results;
+ }
+ if ("windows-1252".equals(topLabel)) {
+ return results;
+ }
+ Charset topCharset = top.getCharset();
+ Charset win1252;
+ try {
+ win1252 = Charset.forName("windows-1252");
+ } catch (IllegalArgumentException e) {
+ return results;
+ }
+ String decodedTop = new String(probe, topCharset);
+ String decoded1252 = new String(probe, win1252);
+ if (!decodedTop.equals(decoded1252)) {
+ return results;
+ }
+ List<EncodingResult> out = new ArrayList<>(results.size());
+ out.add(new EncodingResult(win1252, top.getConfidence(),
+ "windows-1252", top.getResultType()));
+ for (int i = 1; i < results.size(); i++) {
+ out.add(results.get(i));
+ }
+ return out;
+ }
+
/**
* Maximum confidence assigned to a STATISTICAL model result. Kept strictly
* below 1.0 so that statistical results are never mistaken for STRUCTURAL or
(tika) 04/06: bump limit to something realistic
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit d370f5424711096db2eaa7ad91355fe4f9499ee3
Author: tallison <ta...@apache.org>
AuthorDate: Sun Apr 12 16:26:14 2026 -0400
bump limit to something realistic
---
.../org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index 9cc1f7cf32..a0c56393c8 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -169,8 +169,12 @@ public class MojibusterEncodingDetector implements EncodingDetector {
LABEL_TO_JAVA_NAME = Collections.unmodifiableMap(m);
}
- /** Default number of bytes read from the stream for detection. */
- public static final int MAX_PROBE_BYTES = 4096;
+ /**
+ * Default number of bytes read from the stream for detection.
+ * Set generously so HTML/XML probes reach body text past
+ * ASCII-heavy head / script sections.
+ */
+ public static final int MAX_PROBE_BYTES = 32768;
/**
* JSON-deserializable configuration for {@link MojibusterEncodingDetector}.