You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/03/05 23:13:04 UTC
(tika) 09/09: chardet -
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch chardet-work
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 534c72f36938c5be548c0b7386011ac1c58dc786
Author: tballison <ti...@elastic.co>
AuthorDate: Thu Mar 5 18:12:35 2026 -0500
chardet -
---
.../java/org/apache/tika/detect/BOMDetector.java | 18 +-
.../tika/detect/CompositeEncodingDetector.java | 3 +-
.../org/apache/tika/detect/EncodingDetector.java | 11 +-
.../tika/detect/EncodingDetectorContext.java | 7 +
.../org/apache/tika/detect/EncodingResult.java | 92 +++++++--
.../tika/detect/MetadataCharsetDetector.java | 132 ++++++++++++
.../tika/detect/OverrideEncodingDetector.java | 3 +-
.../org.apache.tika.detect.EncodingDetector | 10 +
.../charsoup/CharSoupEncodingDetector.java | 66 +++---
.../charsoup/CharSoupEncodingDetectorTest.java | 8 +-
.../tika/parser/html/HtmlEncodingDetector.java | 3 +-
.../StandardHtmlEncodingDetector.java | 105 ++++++----
.../tika-encoding-detector-icu4j/pom.xml | 12 ++
.../tika/parser/txt/CharsetDetectorTest.java | 1 +
.../configs/tika-config-ignore-charset.json | 13 ++
.../resources/test-documents/multi-language.txt | 58 ++++++
.../src/test/resources/test-documents/resume.html | 99 +++++++++
.../resources/test-documents/testIgnoreCharset.txt | 4 +
.../resources/test-documents/testTXT_win-1252.txt | 1 +
.../test-documents/test_ignore_IBM420.html | Bin 0 -> 1869 bytes
.../ml/chardetect/MojibusterEncodingDetector.java | 223 ++++++++++++++++-----
.../ml/chardetect/StructuralEncodingRules.java | 29 +++
.../tika/ml/chardetect/chardetect-ebcdic.bin | Bin 5232 -> 7312 bytes
.../org/apache/tika/ml/chardetect/chardetect.bin | Bin 508522 -> 410106 bytes
.../tika/ml/chardetect/EbcdicRoutingTest.java | 132 ++++++++++++
.../ml/chardetect/ZipFilenameDetectionTest.java | 142 ++++++-------
.../tika-encoding-detector-universal/pom.xml | 6 +
.../tika/parser/txt/UniversalEncodingDetector.java | 2 +-
.../charsoup/CharSoupLanguageDetector.java | 83 +++++---
.../chardetect/ByteNgramFeatureExtractorTest.java | 32 +--
.../apache/tika/parser/html/HtmlParserTest.java | 16 +-
.../microsoft/POIContainerExtractionTest.java | 3 +-
.../java/org/apache/tika/parser/pkg/ZipParser.java | 1 +
.../tika-parser-text-module/pom.xml | 14 +-
.../tika/parser/csv/TextAndCSVParserTest.java | 13 +-
.../org/apache/tika/parser/txt/TXTParserTest.java | 49 +++--
.../tika/config/TikaEncodingDetectorTest.java | 61 ++++--
.../apache/tika/parser/AutoDetectParserTest.java | 4 +-
.../tika/parser/microsoft/rtf/RTFParserTest.java | 4 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 2 +-
.../apache/tika/parser/pkg/PackageParserTest.java | 2 -
.../TIKA-2485-encoding-detector-mark-limits.json | 9 +-
42 files changed, 1122 insertions(+), 351 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/detect/BOMDetector.java b/tika-core/src/main/java/org/apache/tika/detect/BOMDetector.java
index 322e307187..21e9ca08a7 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/BOMDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/BOMDetector.java
@@ -32,16 +32,18 @@ import org.apache.tika.parser.ParseContext;
/**
* Encoding detector that identifies the character set from a byte-order mark
- * (BOM) at the start of the stream. Returns a single result with confidence
- * {@link EncodingResult#CONFIDENCE_DEFINITIVE} when a BOM is found.
+ * (BOM) at the start of the stream. Returns a single {@link EncodingResult.ResultType#DECLARATIVE}
+ * result when a BOM is found — a BOM is an explicit in-band declaration of encoding
+ * and takes priority over all statistical or structural inference.
*
- * <p>Not SPI-loaded by default — add explicitly to your encoding-detector
- * chain when needed. UTF-16/32 content without a BOM is detected by
- * {@code MojibusterEncodingDetector} via stride-2 byte n-gram features.</p>
+ * <p>SPI-loaded first in the default encoding-detector chain so that BOM evidence
+ * reaches {@code CharSoupEncodingDetector} before any statistical detector runs.
+ * {@code MojibusterEncodingDetector} strips the BOM from its own probe independently
+ * to ensure consistent model inference (BOMs are excluded from training data).</p>
*
* @since Apache Tika 0.x (moved to org.apache.tika.detect in 4.0)
*/
-@TikaComponent(spi = false)
+@TikaComponent
public class BOMDetector implements EncodingDetector {
private static final ByteOrderMark[] BOMS =
@@ -88,8 +90,8 @@ public class BOMDetector implements EncodingDetector {
for (int i = 0; i < BOMS.length; i++) {
ByteOrderMark bom = BOMS[i];
if (startsWith(bom, bytes) && CHARSETS[i] != null) {
- return List.of(new EncodingResult(CHARSETS[i],
- EncodingResult.CONFIDENCE_DEFINITIVE));
+ return List.of(new EncodingResult(CHARSETS[i], 1.0f,
+ CHARSETS[i].name(), EncodingResult.ResultType.DECLARATIVE));
}
}
return Collections.emptyList();
diff --git a/tika-core/src/main/java/org/apache/tika/detect/CompositeEncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/CompositeEncodingDetector.java
index 10285ba44a..fc8b0ab038 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/CompositeEncodingDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/CompositeEncodingDetector.java
@@ -194,7 +194,8 @@ public class CompositeEncodingDetector implements EncodingDetector, Serializable
sb.append(", ");
}
sb.append(r.getDetectorName()).append("->").append(r.getCharset().name());
- if (r.getConfidence() < EncodingResult.CONFIDENCE_DEFINITIVE) {
+ sb.append("[").append(r.getResultType()).append("]");
+ if (r.getResultType() == EncodingResult.ResultType.STATISTICAL) {
sb.append(String.format(java.util.Locale.ROOT, "(%.2f)", r.getConfidence()));
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetector.java
index ff02a0bcb5..7522003c76 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetector.java
@@ -31,11 +31,12 @@ import org.apache.tika.parser.ParseContext;
* metadata or the first few bytes of the document stream.
*
* <p>Detectors return a ranked list of {@link EncodingResult}s in descending
- * confidence order. An empty list means no opinion. A single result with
- * confidence {@link EncodingResult#CONFIDENCE_DEFINITIVE} (1.0) indicates a
- * structural detection that requires no further arbitration. Multiple results
- * or lower confidence values invite arbitration by a
- * {@link MetaEncodingDetector}.</p>
+ * confidence order. An empty list means no opinion. Results carry a
+ * {@link EncodingResult.ResultType} indicating the nature of the evidence:
+ * {@code DECLARATIVE} (BOM, HTML meta charset), {@code STRUCTURAL} (byte-grammar
+ * proof), or {@code STATISTICAL} (probabilistic model). A
+ * {@link MetaEncodingDetector} uses these types to arbitrate when detectors
+ * disagree.</p>
*
* @since Apache Tika 0.4
*/
diff --git a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java
index 81d870e599..6957601e2c 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java
@@ -144,6 +144,13 @@ public class EncodingDetectorContext {
return encodingResults.get(0).getConfidence();
}
+ /**
+ * The {@link EncodingResult.ResultType} of the top-ranked result from this detector.
+ */
+ public EncodingResult.ResultType getResultType() {
+ return encodingResults.get(0).getResultType();
+ }
+
public String getDetectorName() {
return detectorName;
}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/EncodingResult.java b/tika-core/src/main/java/org/apache/tika/detect/EncodingResult.java
index 135e81240c..55724aefc7 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/EncodingResult.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/EncodingResult.java
@@ -19,20 +19,50 @@ package org.apache.tika.detect;
import java.nio.charset.Charset;
/**
- * A charset detection result pairing a {@link Charset} with a confidence score.
+ * A charset detection result pairing a {@link Charset} with a confidence score
+ * and a {@link ResultType} indicating the nature of the evidence.
*
- * <p>Confidence is in the range {@code [0.0, 1.0]}. A score of {@code 1.0}
- * indicates a definitive structural detection (e.g. UTF-16/32 from null-byte
- * patterns, or a declared {@code charset} attribute in an HTML meta tag) that
- * requires no further arbitration. Lower scores reflect statistical estimates
- * where arbitration by a {@link MetaEncodingDetector} may improve accuracy.</p>
+ * <h3>Result types</h3>
+ * <ul>
+ * <li>{@link ResultType#DECLARATIVE} — the document explicitly stated its
+ * encoding (BOM, HTML {@code <meta charset>}). These are authoritative
+ * claims about author intent and get preference over inferred results
+ * <em>when consistent with the actual bytes</em>.</li>
+ * <li>{@link ResultType#STRUCTURAL} — byte-grammar proof (ISO-2022 escape
+ * sequences, UTF-8 multibyte validation). The encoding is proven by the
+ * byte structure itself, independent of any declaration.</li>
+ * <li>{@link ResultType#STATISTICAL} — probabilistic inference from a
+ * statistical model. The {@code confidence} float is meaningful here
+ * for ranking among candidates; for DECLARATIVE and STRUCTURAL results
+ * it is conventionally {@code 1.0} but carries no additional information.</li>
+ * </ul>
*
* @since Apache Tika 4.0
*/
public class EncodingResult {
- /** Confidence value indicating a definitive, structural detection. */
- public static final float CONFIDENCE_DEFINITIVE = 1.0f;
+ /**
+ * The nature of the evidence that produced this result.
+ */
+ public enum ResultType {
+ /**
+ * The document explicitly declared its encoding (BOM, HTML meta charset).
+ * Authoritative about author intent; preferred over inferred results when
+ * consistent with the actual bytes.
+ */
+ DECLARATIVE,
+ /**
+ * The encoding is proven by byte-grammar structure (ISO-2022 escape
+ * sequences, UTF-8 multibyte validation). Not a guess — the byte
+ * patterns are only valid in this encoding.
+ */
+ STRUCTURAL,
+ /**
+ * Probabilistic inference from a statistical model. The confidence
+ * float is meaningful for ranking among candidates.
+ */
+ STATISTICAL
+ }
private final Charset charset;
private final float confidence;
@@ -47,28 +77,53 @@ public class EncodingResult {
* prediction without going through {@code Charset.name()}.
*/
private final String label;
+ private final ResultType resultType;
/**
+ * Constructs a STATISTICAL result. Existing detectors that do not yet
+ * classify their evidence type default to statistical (probabilistic)
+ * treatment, which is the safe, arbitratable assumption.
+ *
* @param charset the detected charset; must not be {@code null}
* @param confidence detection confidence in {@code [0.0, 1.0]}
*/
public EncodingResult(Charset charset, float confidence) {
- this(charset, confidence, charset.name());
+ this(charset, confidence, charset.name(), ResultType.STATISTICAL);
}
/**
+ * Constructs a STATISTICAL result with a detector-specific label.
+ *
* @param charset the detected charset; must not be {@code null}
* @param confidence detection confidence in {@code [0.0, 1.0]}
* @param label the detector's original label (e.g. {@code "IBM420-ltr"});
* if {@code null}, defaults to {@code charset.name()}
*/
public EncodingResult(Charset charset, float confidence, String label) {
+ this(charset, confidence, label, ResultType.STATISTICAL);
+ }
+
+ /**
+ * Constructs a result with an explicit {@link ResultType}.
+ *
+ * @param charset the detected charset; must not be {@code null}
+ * @param confidence detection confidence in {@code [0.0, 1.0]}
+ * @param label the detector's original label; if {@code null},
+ * defaults to {@code charset.name()}
+ * @param resultType the nature of the evidence; must not be {@code null}
+ */
+ public EncodingResult(Charset charset, float confidence, String label,
+ ResultType resultType) {
if (charset == null) {
throw new IllegalArgumentException("charset must not be null");
}
+ if (resultType == null) {
+ throw new IllegalArgumentException("resultType must not be null");
+ }
this.charset = charset;
this.confidence = Math.max(0f, Math.min(1f, confidence));
this.label = (label != null) ? label : charset.name();
+ this.resultType = resultType;
}
public Charset getCharset() {
@@ -76,13 +131,25 @@ public class EncodingResult {
}
/**
- * Detection confidence in {@code [0.0, 1.0]}.
- * {@code 1.0} means definitive; lower values invite arbitration.
+ * Detection confidence in {@code [0.0, 1.0]}. Meaningful for ranking
+ * among {@link ResultType#STATISTICAL} candidates. For
+ * {@link ResultType#DECLARATIVE} and {@link ResultType#STRUCTURAL} results
+ * the value is conventionally {@code 1.0} but carries no additional
+ * information beyond the type itself.
*/
public float getConfidence() {
return confidence;
}
+ /**
+ * The nature of the evidence that produced this result.
+ *
+ * @see ResultType
+ */
+ public ResultType getResultType() {
+ return resultType;
+ }
+
/**
* The detector's original label for this result. Usually identical to
* {@link #getCharset()}{@code .name()}, but preserved when the detector
@@ -97,6 +164,7 @@ public class EncodingResult {
public String toString() {
String cs = charset.name();
String lbl = label.equals(cs) ? cs : label + "(" + cs + ")";
- return lbl + "@" + String.format(java.util.Locale.ROOT, "%.2f", confidence);
+ return lbl + "@" + String.format(java.util.Locale.ROOT, "%.2f", confidence)
+ + "[" + resultType + "]";
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/MetadataCharsetDetector.java b/tika-core/src/main/java/org/apache/tika/detect/MetadataCharsetDetector.java
new file mode 100644
index 0000000000..385f3edfbe
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/detect/MetadataCharsetDetector.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.tika.config.TikaComponent;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * Encoding detector that extracts a declared charset from Tika metadata without
+ * reading any bytes from the stream. Returns a single
+ * {@link EncodingResult.ResultType#DECLARATIVE} result when a charset is found.
+ *
+ * <p>Two metadata keys are consulted in order:
+ * <ol>
+ * <li>{@link Metadata#CONTENT_TYPE} — the {@code charset} parameter of the
+ * HTTP/MIME Content-Type header (e.g. {@code text/html; charset=UTF-8}).</li>
+ * <li>{@link Metadata#CONTENT_ENCODING} — a bare charset label set by parsers
+ * such as {@code RFC822Parser}, which splits Content-Type into a bare
+ * media-type key and a separate charset key.</li>
+ * </ol>
+ *
+ * <p>This detector is SPI-loaded in {@code tika-core} and therefore always present
+ * in the default encoding-detector chain. Its DECLARATIVE result is visible to
+ * {@code CharSoupEncodingDetector}, which can weigh it against structural or
+ * statistical evidence from other detectors.</p>
+ *
+ * @since Apache Tika 4.0
+ */
+@TikaComponent(name = "metadata-charset-detector")
+public class MetadataCharsetDetector implements EncodingDetector {
+
+ @Override
+ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
+ ParseContext context) throws IOException {
+ Charset cs = charsetFromContentType(metadata);
+ if (cs == null) {
+ cs = charsetFromContentEncoding(metadata);
+ }
+ if (cs == null) {
+ return Collections.emptyList();
+ }
+ return List.of(new EncodingResult(cs, 1.0f, cs.name(),
+ EncodingResult.ResultType.DECLARATIVE));
+ }
+
+ /**
+ * Returns the charset named in the {@code charset} parameter of the
+ * {@link Metadata#CONTENT_TYPE} value, or {@code null} if absent or unparseable.
+ */
+ public static Charset charsetFromContentType(Metadata metadata) {
+ String contentType = metadata.get(Metadata.CONTENT_TYPE);
+ if (contentType == null) {
+ return null;
+ }
+ MediaType mediaType = MediaType.parse(contentType);
+ if (mediaType == null) {
+ return null;
+ }
+ String label = mediaType.getParameters().get("charset");
+ return parseCharset(label);
+ }
+
+ /**
+ * Returns the charset named in {@link Metadata#CONTENT_ENCODING}, or
+ * {@code null} if absent or unparseable. This key is used by
+ * {@code RFC822Parser} to expose the charset declared in MIME body-part
+ * headers when the bare media type is stored separately in
+ * {@link Metadata#CONTENT_TYPE}.
+ */
+ public static Charset charsetFromContentEncoding(Metadata metadata) {
+ return parseCharset(metadata.get(Metadata.CONTENT_ENCODING));
+ }
+
+ private static Charset parseCharset(String label) {
+ if (label == null || label.isBlank()) {
+ return null;
+ }
+ Charset cs;
+ try {
+ cs = Charset.forName(label.trim());
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ return normalizeWhatwg(cs);
+ }
+
+ /**
+ * Applies the critical WHATWG encoding-label normalizations that are universally
+ * applicable regardless of content type. The WHATWG encoding spec
+ * (https://encoding.spec.whatwg.org/) maps {@code ISO-8859-1}, {@code US-ASCII},
+ * and their aliases to {@code windows-1252} because real-world content labeled
+ * with these names is almost always actually windows-1252.
+ */
+ private static Charset normalizeWhatwg(Charset cs) {
+ if (cs == null) {
+ return null;
+ }
+ String name = cs.name();
+ if (StandardCharsets.ISO_8859_1.name().equals(name)
+ || StandardCharsets.US_ASCII.name().equals(name)) {
+ try {
+ return Charset.forName("windows-1252");
+ } catch (IllegalArgumentException e) {
+ return cs;
+ }
+ }
+ return cs;
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/OverrideEncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/OverrideEncodingDetector.java
index 3c3ddfa627..e300adccac 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/OverrideEncodingDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/OverrideEncodingDetector.java
@@ -85,7 +85,8 @@ public class OverrideEncodingDetector implements EncodingDetector {
@Override
public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
ParseContext parseContext) throws IOException {
- return List.of(new EncodingResult(charset, EncodingResult.CONFIDENCE_DEFINITIVE));
+ return List.of(new EncodingResult(charset, 1.0f, charset.name(),
+ EncodingResult.ResultType.DECLARATIVE));
}
public Charset getCharset() {
diff --git a/tika-core/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector b/tika-core/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector
index 4a812de77e..1c321a2921 100644
--- a/tika-core/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector
+++ b/tika-core/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector
@@ -13,3 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Both detectors are in the org.apache.tika.detect.* namespace and sort before
+# all other detectors, ensuring DECLARATIVE evidence from in-band signals (BOM)
+# and out-of-band declarations (HTTP/MIME headers) reaches CharSoupEncodingDetector
+# before any statistical detector runs.
+#
+# Within the namespace, class-name order is: BOMDetector < MetadataCharsetDetector
+# so BOM evidence (highest confidence) is recorded first.
+org.apache.tika.detect.BOMDetector
+org.apache.tika.detect.MetadataCharsetDetector
+
diff --git a/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java
index 3a60e551ba..35ead7bb8a 100644
--- a/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java
@@ -71,8 +71,8 @@ public class CharSoupEncodingDetector implements MetaEncodingDetector {
* Symmetric confusable peer groups: within each group, encoding variants
* (e.g. ISO-8859-6 vs windows-1256) produce different decoded text for the
* same byte sequence (unlike ISO-8859-1 vs windows-1252 which are functional
- * supersets). When the language-quality winner and a CONFIDENCE_DEFINITIVE
- * declaration are in the same peer group, the language model cannot reliably
+ * supersets). When the language-quality winner and a DECLARATIVE result
+ * are in the same peer group, the language model cannot reliably
* distinguish them — it merely reflects which variant happens to produce
* Arabic (or Cyrillic, …) n-grams its training data favoured.
* In that case we prefer the explicit declaration.
@@ -158,25 +158,31 @@ public class CharSoupEncodingDetector implements MetaEncodingDetector {
CharSoupLanguageDetector langDetector = new CharSoupLanguageDetector();
Charset bestCharset = langDetector.compareLanguageSignal(candidates);
if (bestCharset == null) {
+ // Language signal inconclusive. When a DECLARATIVE result (HTML meta charset,
+ // BOM, HTTP Content-Type) exists and decodes the bytes at least as cleanly as
+ // the statistical fallback, trust the declaration. This covers:
+ // • Pure-ASCII probe (both decodings identical) — prefer the declared charset.
+ // • Probe with high bytes that are valid in BOTH charsets (e.g. Cyrillic in a
+ // page that starts with ASCII JavaScript) — the bytes look "clean" in both
+ // windows-1252 (decoded as Latin Extended) and windows-1251 (decoded as
+ // Cyrillic), so junkRatio cannot distinguish them; trust the declaration.
Charset fallback = firstResult.getCharset();
String fallbackDecoded = candidates.get(fallback);
float fallbackJunk = fallbackDecoded != null
? CharSoupLanguageDetector.junkRatio(fallbackDecoded) : 1f;
- // If the fallback charset produces garbled output (replacement chars) but
- // a definitive declaration decodes the bytes cleanly, the probe was likely
- // too short or ASCII-only. Trust the explicit declaration in that case.
Charset cleanerDeclared = null;
- if (fallbackJunk > 0f) {
- for (EncodingDetectorContext.Result r : context.getResults()) {
- if (r.getConfidence() >= EncodingResult.CONFIDENCE_DEFINITIVE) {
- String declaredDecoded = candidates.get(r.getCharset());
- float declaredJunk = declaredDecoded != null
- ? CharSoupLanguageDetector.junkRatio(declaredDecoded) : 1f;
- if (declaredJunk < fallbackJunk / 2) {
- cleanerDeclared = r.getCharset();
- break;
- }
+ for (EncodingDetectorContext.Result r : context.getResults()) {
+ if (r.getResultType() == EncodingResult.ResultType.DECLARATIVE) {
+ String declaredDecoded = candidates.get(r.getCharset());
+ float declaredJunk = declaredDecoded != null
+ ? CharSoupLanguageDetector.junkRatio(declaredDecoded) : 1f;
+ // Trust the declaration when it decodes at least as cleanly as
+ // the statistical fallback (≤ junk). A declaration that produces
+ // MORE junk than the fallback is likely wrong (e.g. a lying BOM).
+ if (declaredJunk <= fallbackJunk) {
+ cleanerDeclared = r.getCharset();
+ break;
}
}
}
@@ -188,22 +194,22 @@ public class CharSoupEncodingDetector implements MetaEncodingDetector {
bestCharset = fallback;
}
- // If a structurally-declared charset (CONFIDENCE_DEFINITIVE, e.g. HTML meta tag)
- // decodes the bytes to the same string as the language-quality winner, prefer
- // the declaration. This validates the HTML header against the actual bytes:
- // if they are functionally equivalent, trust the author's stated encoding.
- // If they produce different text (a real conflict), the bytes win.
+ // If a DECLARATIVE result (e.g. HTML meta charset) decodes the bytes to the same
+ // string as the language-quality winner, prefer the declaration. This validates the
+ // declared encoding against the actual bytes: if they are functionally equivalent,
+ // trust the author's stated encoding. If they produce different text (a real conflict
+ // — e.g. a lying BOM or a wrong meta tag), the bytes win and the language scorer's
+ // choice stands.
//
- // Additionally, when the winner and the declared charset are in the same
- // confusable peer group (e.g. ISO-8859-6 vs windows-1256) and the declared
- // charset decodes cleanly (low junk ratio), the language model cannot
- // reliably distinguish them — they both produce valid same-script text.
- // In that case, prefer the explicit declaration over the model's guess.
+ // Additionally, when the winner and a DECLARATIVE charset are in the same confusable
+ // peer group (e.g. ISO-8859-6 vs windows-1256) and the declared charset decodes
+ // cleanly (low junk ratio), the language model cannot reliably distinguish them —
+ // they both produce valid same-script text. Prefer the explicit declaration.
String winnerDecoded = candidates.get(bestCharset);
float winnerJunk = winnerDecoded != null ? CharSoupLanguageDetector.junkRatio(winnerDecoded) : 1f;
if (winnerDecoded != null) {
for (EncodingDetectorContext.Result r : context.getResults()) {
- if (r.getConfidence() >= EncodingResult.CONFIDENCE_DEFINITIVE
+ if (r.getResultType() == EncodingResult.ResultType.DECLARATIVE
&& !r.getCharset().equals(bestCharset)) {
Charset declared = r.getCharset();
String declaredDecoded = candidates.get(declared);
@@ -214,11 +220,9 @@ public class CharSoupEncodingDetector implements MetaEncodingDetector {
context.setArbitrationInfo("scored-prefer-declared");
return declared;
}
- // When the winner and the declared charset are in the same confusable
- // peer group (e.g. ISO-8859-6 vs windows-1256), and the declared
- // charset decodes at least as cleanly as the winner (not junkier),
- // prefer the explicit declaration — the language model cannot reliably
- // distinguish same-script encoding variants.
+ // Same-script peer group: language model cannot distinguish variants
+ // (e.g. ISO-8859-6 vs windows-1256 both produce valid Arabic text).
+ // Prefer the declaration when it decodes at least as cleanly as the winner.
float declaredJunk = CharSoupLanguageDetector.junkRatio(declaredDecoded);
if (arePeers(bestCharset, declared) && declaredJunk <= winnerJunk) {
context.setArbitrationInfo("scored-prefer-declared-peer");
diff --git a/tika-encoding-detectors/tika-encoding-detector-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetectorTest.java b/tika-encoding-detectors/tika-encoding-detector-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetectorTest.java
index 832e195d07..f4f24307cf 100644
--- a/tika-encoding-detectors/tika-encoding-detector-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetectorTest.java
+++ b/tika-encoding-detectors/tika-encoding-detector-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetectorTest.java
@@ -191,10 +191,14 @@ public class CharSoupEncodingDetectorTest {
assertEquals(0.25f,
CharSoupLanguageDetector.junkRatio("abc\u0080"), 0.001f);
- // Mixed: \r\n are control chars too
- assertEquals(2f / 13f,
+ // \r and \n are ordinary whitespace — not junk
+ assertEquals(0f,
CharSoupLanguageDetector.junkRatio("hello world\r\n"), 0.001f);
+ // Non-whitespace C1 control char mixed with ordinary whitespace
+ assertEquals(1f / 14f,
+ CharSoupLanguageDetector.junkRatio("hello world\r\n\u0080"), 0.001f);
+
// Empty/null
assertEquals(0f, CharSoupLanguageDetector.junkRatio(""), 0.001f);
assertEquals(0f, CharSoupLanguageDetector.junkRatio(null), 0.001f);
diff --git a/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
index 9487cad414..c2fe6dac76 100644
--- a/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
@@ -171,7 +171,8 @@ public class HtmlEncodingDetector implements EncodingDetector {
if (charset == null) {
return Collections.emptyList();
}
- return List.of(new EncodingResult(charset, EncodingResult.CONFIDENCE_DEFINITIVE));
+ return List.of(new EncodingResult(charset, 1.0f, charset.name(),
+ EncodingResult.ResultType.DECLARATIVE));
}
//returns null if no charset was found
diff --git a/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/charsetdetector/StandardHtmlEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/charsetdetector/StandardHtmlEncodingDetector.java
index 9c7ff115eb..8c7ca1536f 100644
--- a/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/charsetdetector/StandardHtmlEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/charsetdetector/StandardHtmlEncodingDetector.java
@@ -16,8 +16,6 @@
*/
package org.apache.tika.parser.html.charsetdetector;
-import static org.apache.tika.parser.html.charsetdetector.CharsetAliases.getCharsetByLabel;
-
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
@@ -29,54 +27,53 @@ import org.apache.commons.io.input.BoundedInputStream;
import org.apache.tika.config.TikaComponent;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.detect.EncodingResult;
+import org.apache.tika.detect.MetadataCharsetDetector;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
/**
- * An encoding detector that tries to respect the spirit of the HTML spec
- * part 12.2.3 "The input byte stream", or at least the part that is compatible with
- * the implementation of tika.
- * <p>
- * https://html.spec.whatwg.org/multipage/parsing.html#the-input-byte-stream
- * <p>
- * If a resource was fetched over HTTP, then HTTP headers should be added to tika metadata
- * when using {@link #detect}, especially {@link Metadata#CONTENT_TYPE}, as it may contain
- * charset information.
- * <p>
- * This encoding detector may return null if no encoding is detected.
- * It is meant to be used inside a {@link org.apache.tika.detect.CompositeEncodingDetector}.
- * For instance:
- * <pre> {@code
- * EncodingDetector detector = new CompositeEncodingDetector(
- * Arrays.asList(
- * new StandardHtmlEncodingDetector(),
- * new Icu4jEncodingDetector()));
- * }</pre>
- * <p>
+ * An encoding detector that respects the HTML5 encoding-sniff algorithm
+ * (https://html.spec.whatwg.org/multipage/parsing.html#the-input-byte-stream):
+ * BOM → HTTP Content-Type header → {@code <meta charset>} / {@code <meta http-equiv>} tag.
+ *
+ * <p>When used standalone (outside a {@link org.apache.tika.detect.CompositeEncodingDetector}
+ * chain) this detector handles the full spec algorithm including BOM detection.
+ *
+ * <p>When used inside the default Tika chain (with {@code BOMDetector} and
+ * {@code MetadataCharsetDetector} already present), set {@code skipBOM=true} so that
+ * this detector focuses exclusively on the HTML {@code <meta>} scan. That lets
+ * {@code CharSoupEncodingDetector} arbitrate between a BOM declaration and a
+ * contradicting {@code <meta>} declaration instead of silently suppressing one.
+ *
+ * <p>HTTP/MIME Content-Type and Content-Encoding metadata are always read here for
+ * standalone compatibility; in the chain they will already have been returned by
+ * {@code MetadataCharsetDetector} and {@code CharSoup} will handle the duplication
+ * gracefully (identical DECLARATIVE results agree, so no harm done).
*/
@TikaComponent(name = "standard-html-encoding-detector")
public final class StandardHtmlEncodingDetector implements EncodingDetector {
- private static final int META_TAG_BUFFER_SIZE = 8192;
+ /**
+ * Default number of bytes to scan for a {@code <meta charset>} declaration.
+ * 65536 is large enough to cover typical {@code <script>} or {@code <style>}
+ * blocks in the {@code <head>} without significant overhead (encoding detection
+ * already buffers the stream). Users who need to handle even deeper declarations
+ * can raise this via {@link #setMarkLimit(int)}.
+ */
+ private static final int META_TAG_BUFFER_SIZE = 65536;
private int markLimit = META_TAG_BUFFER_SIZE;
/**
- * Extracts a charset from a Content-Type HTTP header.
+ * When {@code true}, the BOM check is skipped and the detector goes directly to
+ * the Content-Type header and {@code <meta>} scan. Use this when
+ * {@code BOMDetector} is already present in the chain so that
+ * {@code CharSoupEncodingDetector} can arbitrate between a BOM declaration and a
+ * contradicting {@code <meta charset>} rather than having the BOM silently win.
*
- * @param metadata parser metadata
- * @return a charset if there is one specified, or null
+ * <p>Default: {@code false} (HTML5 spec-compliant standalone behaviour).</p>
*/
- private static Charset charsetFromContentType(Metadata metadata) {
- String contentType = metadata.get(Metadata.CONTENT_TYPE);
- MediaType mediatype = MediaType.parse(contentType);
- if (mediatype == null) {
- return null;
- }
- String charsetLabel = mediatype.getParameters().get("charset");
- return getCharsetByLabel(charsetLabel);
- }
+ private boolean skipBOM = false;
@Override
public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
@@ -87,10 +84,19 @@ public final class StandardHtmlEncodingDetector implements EncodingDetector {
.setInputStream(tis).setMaxCount(limit).get();
PreScanner preScanner = new PreScanner(limitedStream);
- // Priority: 1. BOM 2. Content-Type HTTP header 3. HTML <meta> tag
- Charset detectedCharset = preScanner.detectBOM();
+ Charset detectedCharset = null;
+
+ if (!skipBOM) {
+ // HTML5 spec: BOM overrides everything. When used standalone this
+ // detector is responsible for BOM detection; when used in the chain with
+ // BOMDetector, setting skipBOM=true lets CharSoup arbitrate.
+ detectedCharset = preScanner.detectBOM();
+ }
+ if (detectedCharset == null) {
+ detectedCharset = MetadataCharsetDetector.charsetFromContentType(metadata);
+ }
if (detectedCharset == null) {
- detectedCharset = charsetFromContentType(metadata);
+ detectedCharset = MetadataCharsetDetector.charsetFromContentEncoding(metadata);
}
if (detectedCharset == null) {
detectedCharset = preScanner.scan();
@@ -100,7 +106,8 @@ public final class StandardHtmlEncodingDetector implements EncodingDetector {
if (detectedCharset == null) {
return Collections.emptyList();
}
- return List.of(new EncodingResult(detectedCharset, EncodingResult.CONFIDENCE_DEFINITIVE));
+ return List.of(new EncodingResult(detectedCharset, 1.0f,
+ detectedCharset.name(), EncodingResult.ResultType.DECLARATIVE));
}
public int getMarkLimit() {
@@ -108,10 +115,24 @@ public final class StandardHtmlEncodingDetector implements EncodingDetector {
}
/**
- * How far into the stream to read for charset detection.
- * Default is 8192.
+ * How far into the stream to scan for a {@code <meta charset>} declaration.
+ * Default is {@value #META_TAG_BUFFER_SIZE} bytes.
*/
public void setMarkLimit(int markLimit) {
this.markLimit = markLimit;
}
+
+ public boolean isSkipBOM() {
+ return skipBOM;
+ }
+
+ /**
+ * When {@code true}, skip the BOM check and rely on {@code BOMDetector} in the
+ * chain. This allows {@code CharSoupEncodingDetector} to arbitrate between a
+ * BOM and a contradicting {@code <meta charset>} declaration.
+ * Default is {@code false}.
+ */
+ public void setSkipBOM(boolean skipBOM) {
+ this.skipBOM = skipBOM;
+ }
}
diff --git a/tika-encoding-detectors/tika-encoding-detector-icu4j/pom.xml b/tika-encoding-detectors/tika-encoding-detector-icu4j/pom.xml
index c96b3a7ff0..985202bf36 100644
--- a/tika-encoding-detectors/tika-encoding-detector-icu4j/pom.xml
+++ b/tika-encoding-detectors/tika-encoding-detector-icu4j/pom.xml
@@ -41,6 +41,12 @@
<artifactId>tika-core</artifactId>
<version>${revision}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-annotation-processor</artifactId>
+ <version>${revision}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
@@ -48,6 +54,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-encoding-detector-universal</artifactId>
+ <version>${revision}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-serialization</artifactId>
diff --git a/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java
index 3cf4b435c7..04666b1726 100644
--- a/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java
+++ b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java
@@ -129,6 +129,7 @@ public class CharsetDetectorTest extends TikaTest {
assertEquals("UTF-8", detector.detect().getName());
}
+ @org.junit.jupiter.api.Disabled("Integration test requiring TXT parser — run via tika-parser-text-module")
@Test
public void testIgnoreCharset() throws Exception {
//TIKA-3516, TIKA-3525, TIKA-1236
diff --git a/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/configs/tika-config-ignore-charset.json b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/configs/tika-config-ignore-charset.json
new file mode 100644
index 0000000000..82442e13a2
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/configs/tika-config-ignore-charset.json
@@ -0,0 +1,13 @@
+{
+ "parsers": [
+ "default-parser"
+ ],
+ "encoding-detectors": [
+ {
+ "icu4j-encoding-detector": {
+ "ignoreCharsets": ["IBM420", "IBM424"]
+ }
+ },
+ "universal-encoding-detector"
+ ]
+}
diff --git a/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/multi-language.txt b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/multi-language.txt
new file mode 100644
index 0000000000..ab78d1c2f6
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/multi-language.txt
@@ -0,0 +1,58 @@
+English: ABC Category
+
+en ABC adhoc internship
+en ABC agent uno well connected
+en ABC agent uno good connections
+en ABC really need uno agent
+en ABC special retainer
+en ABC let's get this through
+en ABC let's push this through
+en ABC Token of gratitude
+en ABC do not expense
+en ABC don't expense
+en ABC want to keep you happy
+en ABC win win payment
+en ABC win win hire
+en ABC donation uno business
+
+Portuguese: ABC Category
+
+Port ABC pagamento de sucesso
+Port ABC pagamento de sucesso
+Port ABC presentinho uno agradecimento
+Port ABC taxa especial
+Port ABC taxa especial
+Port ABC realmente preciso uno agente
+Port ABC vamos empurrar isso
+Port ABC vamos empurrar isso
+Port ABC Vamos acabar com isso
+Port ABC te deixar feliz
+Port ABC te deixar feliz
+Port ABC nao contabilize
+Port ABC n�o contabilize
+Port ABC nao contabilize
+Port ABC n�o contabilize
+Port ABC doa��o uno neg�cios
+Port ABC doacao uno negocios
+Port ABC agente uno bem conectado
+Port ABC agente uno bons contatos
+Port ABC estagio adhoc
+Port ABC contratacao de sucesso�
+Port ABC contrata��o de sucesso�
+
+Spanish: ABC Category
+
+espa�ol ABC adhoc pr�cticas
+espa�ol ABC intermediario uno bien conectado
+espa�ol ABC agente uno buenas conecciones
+espa�ol ABC realmente necesito uno un agente
+espa�ol ABC anticipo especial
+espa�ol ABC hay que pasar
+espa�ol ABC hay que forzar
+espa�ol ABC muestra de gratitud
+espa�ol ABC no registre gasto
+espa�ol ABC no reporte gasto
+espa�ol ABC mantenerte contento
+espa�ol ABC todos ganan con el anticipo
+espa�ol ABC donacion uno negocios
+espa�ol ABC todos ganan con la contratacion
diff --git a/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/resume.html b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/resume.html
new file mode 100644
index 0000000000..3e55dcd116
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/resume.html
@@ -0,0 +1,99 @@
+<div class="js-helper">
+ <style type="text/css">#style_13209008630000000884_BODY{background-color:#FFFFFF;color:#000000;MARGIN:0px 1px;font-family:Tahoma,Arial,Verdana,Sans-Serif}#style_13209008630000000884 TD{font-size:13px;font-family:Tahoma,Arial,Verdana,Sans-Serif;vertical-align:top}#style_13209008630000000884 CAPTION{font-size:13px;font-weight:bold;text-align:left}#style_13209008630000000884 TR.style_13209008630000000884thead TD{font-weight:bold;text-align:center; padding-bottom:6px;padding-top:6px;padd [...]
+
+ </style>
+ <div class="mr_read__body" id="style_13209008630000000884">
+ <base href="http://e.mail.ru/cgi-bin/" target="_self"/>
+
+ <div id="style_13209008630000000884_BODY">
+
+
+ <style type="text/css"></style>
+
+
+ <table border="0" cellpadding="0" cellspacing="0" height="100%" width="100%">
+ <tr>
+ <td>
+
+ </td>
+ </tr>
+ <tr>
+ <td height="100%" style="padding:5px">
+ Здравствуйте, !<br>
+ <br>
+ Предлагаем Вам ознакомиться со списком зарегистрированных компаний, представители которых
+ просмотрели Ваше резюме за последние сутки.<br>
+ <br>
+ <li><a href="/cgi-bin/link?check=1&cnf=710139&url=http%3A%2F%2;0,0" target="_blank">Компании,
+ просмотревшие резюме № .</a> Новые: <b>1.</b></li>
+ <br>
+ <br>
+ Эти сведения предоставляются Вам исключительно для информации. Вы можете оперативно отслеживать,
+ какие именно компании нашли в базе данных Superjob Ваше резюме и заинтересовались им.<br>
+ <br>
+ Если Ваше резюме размещено в закрытом доступе, то его могут просматривать только те
+ работодатели, которым Вы отправили его самостоятельно.<br>
+ Историю отправки своего резюме Вы можете посмотреть по ссылке «История рассылки резюме».<br>
+ <br>
+ <br>
+ <b>Внимание!</b><br>
+ В процессе поиска работы Вы можете столкнуться с такими предложениями работодателей или кадровых
+ агентств, в которых Вас будут просить внести оплату (за предварительное обучение, за оформление
+ документов, за оформление обязательной страховки, на закупку первой партии продукции компании,
+ предназначенной для продажи и т.п.) или предоставить отсканированные копии документов (паспорта,
+ военного билета, трудовой книжки, водительских прав, пенсионного удостоверния и т.п.) для якобы
+ предварительного оформления или подтверждения данных, указанных в Вашем резюме.<br>
+ Это один из признаков мошенничества! Мы рекомендуем Вам очень осторожно относиться к таким
+ предложениям и по возможности избегать собеседований с подобными работодателями.<br>
+ <br>
+ Также мы настоятельно не рекомендуем отправлять платные SMS-сообщения на короткие номера для
+ получения контактов или другой информации о вакансии или же для получения результатов
+ тестирования. С организациями, которые оказывают подобные услуги, мы не сотрудничаем и
+ предупреждаем, что это тоже один из приемов мошенничества.<br>
+ <br>
+ <br>
+ <em>x</em> <a href="/cgi-bin/link?check=1&cnf=8d972a&url=http%3A%2F%2Fwww.sup;0,0"
+ target="_blank">Отключить
+ уведомления о новых просмотрах моих резюме</a><br>
+ <br>
+ По ссылкам в этом письме можно войти в систему без ввода пароля.
+ <br><br>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <span class="style_13209008630000000884noprint"><br><br>Если у Вас есть пожелания и идеи по улучшению сервиса Superjob, пожалуйста, <a
+ href="/cgi-bin/link?check=1;0,0" target="_blank">напишите нам</a>.<br><br></span>
+ <table border="0" cellpadding="10" cellspacing="0" class="style_13209008630000000884noprint"
+ width="100%">
+ <tr>
+ <td align="center" style="border-top:1px solid #BACBD7;">
+ <a href="/cgi-bin/link?check=1&cnf=8fa2f9&url=http%3A%2F%2Fwww.;0,0"
+ target="_blank"><big>Superjob —
+ Работа должна доставлять удовольствие!</big></a>
+ </td>
+ </tr>
+ </table>
+ <table border="0" cellpadding="0" cellspacing="1" class="style_13209008630000000884noprint"
+ width="100%">
+ <tr>
+ <td align="center" style="padding:5px">
+ <span style="color:#999999;font-size:8pt;">Письмо отправлено: xx.xx.xxxx xx:xx:xx</span>
+ </td>
+ </tr>
+ </table>
+
+ </td>
+ </tr>
+ </table>
+
+
+ </div>
+
+
+ <base href="http://e.mail.ru/cgi-bin/" target="_self"/>
+ </div>
+</div>
+
+
+
diff --git a/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/testIgnoreCharset.txt b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/testIgnoreCharset.txt
new file mode 100644
index 0000000000..4673e04852
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/testIgnoreCharset.txt
@@ -0,0 +1,4 @@
+
+ACTIVE AGE
+
+BALM
diff --git a/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/testTXT_win-1252.txt b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/testTXT_win-1252.txt
new file mode 100644
index 0000000000..519c95565a
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/testTXT_win-1252.txt
@@ -0,0 +1 @@
+These smart quotes are the trigger for CharsetRecog_sbcs to think this is a �windows� encoding
\ No newline at end of file
diff --git a/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/test_ignore_IBM420.html b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/test_ignore_IBM420.html
new file mode 100644
index 0000000000..2aecab221d
Binary files /dev/null and b/tika-encoding-detectors/tika-encoding-detector-icu4j/src/test/resources/test-documents/test_ignore_IBM420.html differ
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index 7a2b434d63..d920388072 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -94,7 +94,23 @@ public class MojibusterEncodingDetector implements EncodingDetector {
* 0x40–0xFE), so their presence is definitive proof that a GB18030 codec
* is required to avoid replacement characters.
*/
- GB_FOUR_BYTE_UPGRADE
+ GB_FOUR_BYTE_UPGRADE,
+ /**
+ * Upgrade an ISO-8859-X result to its Windows-12XX equivalent when the
+ * probe contains at least one CRLF pair ({@code 0x0D 0x0A}) but no C1
+ * bytes ({@code 0x80–0x9F}).
+ *
+ * <p>Files originating on Windows use CRLF line endings. The presence
+ * of a {@code \r\n} pair in a probe that is otherwise 7-bit ASCII (or
+ * has only high bytes above {@code 0x9F}) is weak evidence of Windows
+ * origin and therefore of a Windows code page. {@link Rule#ISO_TO_WINDOWS}
+ * already handles the C1-byte case definitively; this rule covers the
+ * weaker case where C1 bytes have not been seen but CRLF line endings
+ * suggest Windows origin. A bare {@code 0x0D} (old Mac Classic CR-only
+ * line ending) does <em>not</em> trigger this rule. Mirrors the legacy
+ * {@code UniversalEncodingListener.report()} heuristic.</p>
+ */
+ CRLF_TO_WINDOWS
}
private static final long serialVersionUID = 1L;
@@ -278,28 +294,28 @@ public class MojibusterEncodingDetector implements EncodingDetector {
}
byte[] probe = readProbe(input, maxProbeBytes);
+ // Strip BOM bytes before feature extraction. BOM detection is handled
+ // by BOMDetector (which runs earlier in the chain and returns DECLARATIVE).
+ // BOMs are excluded from training data; stripping ensures consistent
+ // model inference. A lying BOM (e.g. UTF-8 BOM on a Windows-1252 file)
+ // is caught by CharSoup comparing BOMDetector's DECLARATIVE claim against
+ // Mojibuster's byte-content analysis.
probe = stripBom(probe);
- if (probe.length == 0) {
- return singleResult(StandardCharsets.UTF_8.name(), EncodingResult.CONFIDENCE_DEFINITIVE, Integer.MAX_VALUE);
- }
-
+ // An empty probe (e.g. empty file, or a file that was only a BOM) falls
+ // through to detectAll where isPureAscii returns true for a zero-length
+ // array, yielding the same windows-1252 default as any other pure-ASCII probe.
return detectAll(probe, Integer.MAX_VALUE);
}
/**
- * Structural gates: deterministic early exits before the general model runs.
- * Only bulletproof, zero-false-positive checks belong here.
- * EBCDIC discrimination is intentionally absent — it is handled by the sub-model.
- */
- /**
- * Applies structural encoding rules that produce CONFIDENCE_DEFINITIVE results.
- * Returns non-null only when a byte-level pattern unambiguously identifies the charset
- * (e.g. ISO-2022 escape sequences, sparse valid UTF-8 multibyte sequences).
- * Pure ASCII is deliberately excluded here — ASCII is compatible with virtually all
- * single-byte encodings, so it is NOT definitive. Use {@link #applyAsciiHeuristic}
- * for the ASCII case.
+ * Applies structural encoding rules that produce {@link EncodingResult.ResultType#STRUCTURAL}
+ * results. Returns non-null only when a byte-level pattern unambiguously identifies the
+ * charset (ISO-2022 escape sequences, sparse valid UTF-8 multibyte sequences).
+ *
+ * Pure ASCII is deliberately excluded — ASCII is compatible with virtually all
+ * single-byte encodings and is not structurally definitive.
*/
- private Charset applyDefinitiveStructuralRules(byte[] probe) {
+ private Charset applyStructuralRules(byte[] probe) {
// ISO-2022 before ASCII: all three variants are 7-bit so checkAscii fires first.
Charset iso2022 = StructuralEncodingRules.detectIso2022(probe);
if (iso2022 != null) {
@@ -327,11 +343,11 @@ public class MojibusterEncodingDetector implements EncodingDetector {
}
/**
- * Returns UTF-8 if the probe is pure 7-bit ASCII (no bytes ≥ 0x80, no null bytes).
- * ASCII is a strict subset of UTF-8 and of every single-byte encoding, so this is
- * a heuristic only — the confidence returned by the caller must be below
- * CONFIDENCE_DEFINITIVE to allow downstream detectors (e.g. HTML meta charset) to
- * override it without ambiguity.
+ * Returns true if the probe is pure 7-bit ASCII (no bytes ≥ 0x80, no null bytes).
+ * ASCII is compatible with virtually every single-byte encoding, so this is a
+ * heuristic — we report US-ASCII to honestly reflect what the probe showed.
+ * CharSoup will upgrade to a declared encoding (e.g. ISO-8859-15) when the document
+ * contains an explicit declaration consistent with the ASCII bytes.
*/
private static boolean isPureAscii(byte[] probe) {
return StructuralEncodingRules.checkAscii(probe) && !hasNullBytes(probe);
@@ -396,17 +412,24 @@ public class MojibusterEncodingDetector implements EncodingDetector {
boolean gates = enabledRules.contains(Rule.STRUCTURAL_GATES);
if (gates) {
- // Definitive structural rules (BOM, ISO-2022, sparse UTF-8) → CONFIDENCE_DEFINITIVE.
- Charset definitive = applyDefinitiveStructuralRules(probe);
- if (definitive != null) {
- return singleResult(definitive.name(), EncodingResult.CONFIDENCE_DEFINITIVE, topN);
+ // Structural rules: byte-grammar proof (ISO-2022, sparse UTF-8).
+ Charset structural = applyStructuralRules(probe);
+ if (structural != null) {
+ return singleResult(structural.name(), 1.0f,
+ EncodingResult.ResultType.STRUCTURAL, topN);
}
- // ASCII heuristic: pure 7-bit ASCII is valid in virtually every single-byte
- // encoding, so we report UTF-8 with a sub-definitive confidence. This allows
- // an HTML meta-charset declaration (which IS CONFIDENCE_DEFINITIVE) to override
- // the ASCII heuristic without non-deterministic tie-breaking.
+ // Pure ASCII: no high bytes seen in the probe. We default to windows-1252 —
+ // the WHATWG-canonical "Western Latin, I saw only ASCII bytes" encoding.
+ // HTML5 explicitly defines ISO-8859-1 as an alias for windows-1252, making
+ // windows-1252 the right default: it is the correct superset, it avoids the
+ // ambiguity between ISO-8859-1 and windows-1252 in the 0x80–0x9F range, and
+ // it keeps the no-hint path consistent with the HTML-spec path (where a stated
+ // "charset=iso-8859-1" is normalized to windows-1252 by StandardHtmlEncodingDetector).
+ // CharSoup will further upgrade to any compatible DECLARATIVE encoding
+ // (e.g. an HTML meta charset=UTF-8) when one is present and consistent.
if (isPureAscii(probe)) {
- return singleResult(StandardCharsets.UTF_8.name(), 0.75f, topN);
+ return singleResult("windows-1252", 0.5f,
+ EncodingResult.ResultType.STATISTICAL, topN);
}
}
@@ -420,14 +443,12 @@ public class MojibusterEncodingDetector implements EncodingDetector {
return runEbcdicSubModel(probe, topN);
}
- // Grammar filtering can leave the list empty on very short probes (e.g. a single
- // high byte that is a valid CJK lead but has an invalid trail byte). Fall back to
- // UTF-8 rather than returning empty and causing AutoDetectReader to throw.
+ // If the model had no evidence (probe too short or all tokens filtered), fall back to
+ // windows-1252 at very low confidence rather than returning empty and letting
+ // AutoDetectReader throw. CharSoup will override this with any DECLARATIVE hint.
if (results.isEmpty()) {
- return singleResult(StandardCharsets.UTF_8.name(),
- EncodingResult.CONFIDENCE_DEFINITIVE / 2, Integer.MAX_VALUE);
+ return singleResult("windows-1252", 0.1f, EncodingResult.ResultType.STATISTICAL, topN);
}
-
return results;
}
@@ -443,11 +464,29 @@ public class MojibusterEncodingDetector implements EncodingDetector {
}
}
- List<EncodingResult> results = selectByLogitGap(model, logits, topN);
+ // For short probes the model has limited signal; widen the candidate set so
+ // that CharSoup's language arbitration can rescue the correct answer even when
+ // the gap between competitors exceeds LOGIT_GAP.
+ List<EncodingResult> results;
+ if (probe.length < 50) {
+ results = selectTopN(model, logits, 3);
+ } else if (probe.length < 100) {
+ results = selectTopN(model, logits, 2);
+ } else {
+ results = selectByLogitGap(model, logits, topN);
+ }
if (enabledRules.contains(Rule.ISO_TO_WINDOWS) && StructuralEncodingRules.hasC1Bytes(probe)) {
results = upgradeIsoToWindows(results);
}
+ // CRLF_TO_WINDOWS: when C1 bytes were absent (ISO_TO_WINDOWS didn't fire) but
+ // CRLF pairs suggest Windows line endings, apply the same ISO→Windows upgrade as
+ // weak evidence of Windows file origin. If ISO_TO_WINDOWS already fired, the
+ // results are already Windows-12XX and upgradeIsoToWindows is a no-op.
+ // Bare CR (old Mac Classic line endings) does NOT trigger this rule.
+ if (enabledRules.contains(Rule.CRLF_TO_WINDOWS) && StructuralEncodingRules.hasCrlfBytes(probe)) {
+ results = upgradeIsoToWindows(results);
+ }
if (enabledRules.contains(Rule.CJK_GRAMMAR)) {
results = refineCjkResults(probe, results);
}
@@ -475,6 +514,78 @@ public class MojibusterEncodingDetector implements EncodingDetector {
* hiding plausible alternatives that downstream arbitrators (e.g. CharSoup)
* should evaluate. Linear confidence within the gap preserves the signal.</p>
*/
+ /**
+ * Maximum confidence assigned to a STATISTICAL model result. Kept strictly
+ * below 1.0 so that statistical results are never mistaken for STRUCTURAL or
+ * DECLARATIVE evidence by downstream arbitrators (e.g. CharSoupEncodingDetector).
+ * The top result from the logit-gap window always maps to this value.
+ */
+ private static final float MAX_STATISTICAL_CONFIDENCE = 0.99f;
+
+ /**
+ * Return the top {@code n} single-byte/CJK candidates by logit rank, regardless of gap.
+ * Used for short probes where the model has limited signal and we want
+ * CharSoup's language arbitration to have multiple candidates to compare.
+ * <p>
+ * Wide encodings (UTF-16/32) are excluded: stride-2 features can spuriously
+ * boost them on very short probes that lack the null-byte density that
+ * genuinely characterises UTF-16/32. A 9-byte filename without a BOM is
+ * never UTF-16/32 in practice.
+ * <p>
+ * Confidence is still scaled relative to the logit-gap window so that
+ * results remain in the statistical range below DECLARATIVE/STRUCTURAL.
+ */
+ private static List<EncodingResult> selectTopN(LinearModel m, float[] logits, int n) {
+ // Collect all positive-logit, non-excluded candidates with their array index.
+ // We sort by RAW LOGIT (not sigmoid) so that the model's actual ranking is
+ // preserved even when all logits are large-positive (sigmoid ≈ 1.0 for all).
+ // Example: GB18030=43, EUC-JP=28, Big5=21 — all sigmoid≈0.99 — would tie on
+ // sigmoid and fall back to label-insertion order; sorting by logit keeps GB18030 first.
+ List<int[]> candidates = new ArrayList<>(); // [label-index]
+ for (int i = 0; i < logits.length; i++) {
+ // logit ≤ 0 means sigmoid ≤ 0.5 — the model actively disfavours this encoding.
+ if (logits[i] <= 0) {
+ continue;
+ }
+ String lbl = m.getLabel(i);
+ if (isExcludedFromShortProbe(lbl)) {
+ continue;
+ }
+ if (labelToCharset(lbl) == null) {
+ continue;
+ }
+ candidates.add(new int[]{i});
+ }
+ // Sort descending by logit so model ranking is preserved.
+ candidates.sort((a, b) -> Float.compare(logits[b[0]], logits[a[0]]));
+
+ // Take the top N and assign sigmoid confidence so downstream code has a meaningful score.
+ List<EncodingResult> result = new ArrayList<>(Math.min(n, candidates.size()));
+ for (int rank = 0; rank < Math.min(n, candidates.size()); rank++) {
+ int i = candidates.get(rank)[0];
+ String lbl = m.getLabel(i);
+ Charset cs = labelToCharset(lbl);
+ float conf = (1f / (1f + (float) Math.exp(-logits[i]))) * MAX_STATISTICAL_CONFIDENCE;
+ result.add(new EncodingResult(cs, conf, lbl, EncodingResult.ResultType.STATISTICAL));
+ }
+ return result;
+ }
+
+ /**
+ * Returns true for encodings that should be excluded from short-probe top-N selection.
+ * <ul>
+ * <li>Wide encodings (UTF-16/32): stride-2 features spuriously boost them on short
+ * probes that lack the null-byte density that genuinely characterises UTF-16/32.</li>
+ * <li>EBCDIC family (IBM4xx, IBM500, "EBCDIC" routing label): EBCDIC has its own
+ * dedicated sub-model pipeline and should never surface as a candidate for
+ * short single-byte or CJK content.</li>
+ * </ul>
+ */
+ private static boolean isExcludedFromShortProbe(String label) {
+ return label.startsWith("UTF-16") || label.startsWith("UTF-32")
+ || label.startsWith("IBM") || label.equals("EBCDIC");
+ }
+
private static List<EncodingResult> selectByLogitGap(LinearModel m, float[] logits, int topN) {
float maxLogit = Float.NEGATIVE_INFINITY;
for (float l : logits) {
@@ -486,11 +597,14 @@ public class MojibusterEncodingDetector implements EncodingDetector {
List<EncodingResult> results = new ArrayList<>();
for (int i = 0; i < logits.length; i++) {
if (logits[i] >= floor) {
- float conf = (logits[i] - floor) / LOGIT_GAP;
+ // Scale to [0, MAX_STATISTICAL_CONFIDENCE] so no statistical result
+ // reaches 1.0, keeping the range unambiguously below STRUCTURAL/DECLARATIVE.
+ float conf = ((logits[i] - floor) / LOGIT_GAP) * MAX_STATISTICAL_CONFIDENCE;
String lbl = m.getLabel(i);
Charset cs = labelToCharset(lbl);
if (cs != null) {
- results.add(new EncodingResult(cs, conf, lbl));
+ results.add(new EncodingResult(cs, conf, lbl,
+ EncodingResult.ResultType.STATISTICAL));
}
}
}
@@ -552,7 +666,11 @@ public class MojibusterEncodingDetector implements EncodingDetector {
return results;
}
- // Score every CJK charset in the result list.
+ // Grammar-filter CJK charsets: drop those that produce invalid byte sequences
+ // (score == 0 means the grammar walker found bad bytes — the model was wrong).
+ // Charsets that pass grammar keep their model confidence unchanged so that
+ // all candidates remain on the same sigmoid scale for CharSoup to compare.
+ // Non-CJK charsets pass through unchanged.
List<EncodingResult> refined = new ArrayList<>(results.size());
for (EncodingResult er : results) {
if (!CjkEncodingRules.isCjk(er.getCharset())) {
@@ -561,17 +679,13 @@ public class MojibusterEncodingDetector implements EncodingDetector {
}
int score = CjkEncodingRules.match(probe, er.getCharset());
if (score == 0) {
- // grammar rejects this charset — drop entirely
- } else if (score >= CjkEncodingRules.CLEAN_SHORT_PROBE_CONFIDENCE) {
- // structurally clean (bad == 0) — use grammar confidence
- refined.add(new EncodingResult(er.getCharset(), score / 100f));
- } else {
- // some bad bytes within tolerance — keep model confidence
- refined.add(er);
+ // grammar rejects this charset entirely — drop it
+ continue;
}
+ // Grammar passes: keep the model's sigmoid confidence so everything
+ // is on the same scale when CharSoup compares candidates.
+ refined.add(er);
}
-
- refined.sort((a, b) -> Float.compare(b.getConfidence(), a.getConfidence()));
return refined;
}
@@ -617,7 +731,8 @@ public class MojibusterEncodingDetector implements EncodingDetector {
return upgraded;
}
- private static List<EncodingResult> singleResult(String label, float confidence, int topN) {
+ private static List<EncodingResult> singleResult(String label, float confidence,
+ EncodingResult.ResultType type, int topN) {
if (topN <= 0) {
return Collections.emptyList();
}
@@ -625,7 +740,7 @@ public class MojibusterEncodingDetector implements EncodingDetector {
if (cs == null) {
return Collections.emptyList();
}
- return List.of(new EncodingResult(cs, confidence, label));
+ return List.of(new EncodingResult(cs, confidence, label, type));
}
/**
@@ -669,6 +784,10 @@ public class MojibusterEncodingDetector implements EncodingDetector {
return model;
}
+ public LinearModel getEbcdicModel() {
+ return ebcdicModel;
+ }
+
public EnumSet<Rule> getEnabledRules() {
return EnumSet.copyOf(enabledRules.isEmpty() ? EnumSet.noneOf(Rule.class) : enabledRules);
}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
index a87dd71410..40d409cae6 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
@@ -373,6 +373,35 @@ public final class StructuralEncodingRules {
*/
private static final double IBM500_LATIN_THRESHOLD = 0.25;
+ /**
+ * Returns {@code true} if the probe contains at least one CRLF pair
+ * ({@code 0x0D 0x0A}).
+ *
+ * <p>Files originating on Windows use CRLF as the line separator.
+ * The presence of a {@code 0x0D 0x0A} pair in a probe that is otherwise
+ * 7-bit ASCII is weak evidence that the file was created on Windows and
+ * therefore more likely to use a Windows code page (e.g. windows-1252)
+ * than a Unix-origin ISO-8859-X encoding for any high-byte content
+ * beyond the probe window.</p>
+ *
+ * <p>A bare {@code 0x0D} without a following {@code 0x0A} is <em>not</em>
+ * counted: classic Mac OS used bare CR as its line ending, and that is a
+ * different case that does not imply Windows origin.</p>
+ */
+ public static boolean hasCrlfBytes(byte[] bytes) {
+ return hasCrlfBytes(bytes, 0, bytes.length);
+ }
+
+ public static boolean hasCrlfBytes(byte[] bytes, int offset, int length) {
+ int end = offset + length;
+ for (int i = offset; i < end - 1; i++) {
+ if (bytes[i] == 0x0D && bytes[i + 1] == 0x0A) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Returns {@code true} if the probe contains any byte in the C1 control
* range {@code 0x80–0x9F}.
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/chardetect-ebcdic.bin b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/chardetect-ebcdic.bin
index f76be8560d..191cdaeedf 100644
Binary files a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/chardetect-ebcdic.bin and b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/chardetect-ebcdic.bin differ
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/chardetect.bin b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/chardetect.bin
index 31dd657c92..70421e6bb8 100644
Binary files a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/chardetect.bin and b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/chardetect.bin differ
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/EbcdicRoutingTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/EbcdicRoutingTest.java
new file mode 100644
index 0000000000..60c0803d43
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/EbcdicRoutingTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.detect.EncodingResult;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.ml.LinearModel;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * Verifies the two-phase EBCDIC detection pipeline:
+ * <ol>
+ * <li>General model emits {@code "EBCDIC"} routing label for EBCDIC-family bytes.</li>
+ * <li>{@code MojibusterEncodingDetector} routes to the EBCDIC sub-model, which
+ * returns a specific IBM variant (IBM500, IBM420, IBM855, etc.) — never the
+ * bare {@code "EBCDIC"} routing label.</li>
+ * </ol>
+ */
+public class EbcdicRoutingTest {
+
+ private static MojibusterEncodingDetector detector;
+
+ // Representative English prose encoded in IBM500 (International EBCDIC).
+ // Generated via: text.getBytes(Charset.forName("IBM500"))
+ private static final byte[] IBM500_BYTES = makeEbcdic("IBM500",
+ "The quick brown fox jumps over the lazy dog. " +
+ "This sentence contains every letter of the English alphabet. " +
+ "EBCDIC encoding is used on IBM mainframe systems. " +
+ "Fields are often fixed-width and space-padded in EBCDIC files.");
+
+ // Russian text encoded in IBM855 (Cyrillic EBCDIC).
+ private static final byte[] IBM855_BYTES = makeEbcdic("IBM855",
+ "\u041f\u0440\u0438\u0432\u0435\u0442 \u043c\u0438\u0440! " + // Привет мир!
+ "\u042d\u0442\u043e \u0442\u0435\u043a\u0441\u0442 \u043d\u0430 " + // Это текст на
+ "\u0440\u0443\u0441\u0441\u043a\u043e\u043c \u044f\u0437\u044b\u043a\u0435. " + // русском языке.
+ "\u041a\u043e\u0434\u0438\u0440\u043e\u0432\u043a\u0430 IBM855 " + // Кодировка IBM855
+ "\u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442\u0441\u044f " + // используется
+ "\u043d\u0430 \u043c\u0435\u0439\u043d\u0444\u0440\u0435\u0439\u043c\u0430\u0445."); // на мейнфреймах.
+
+ private static byte[] makeEbcdic(String charsetName, String text) {
+ try {
+ return text.getBytes(Charset.forName(charsetName));
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot encode test data as " + charsetName, e);
+ }
+ }
+
+ @BeforeAll
+ static void setUp() {
+ detector = new MojibusterEncodingDetector();
+ }
+
+ /**
+ * The general model must have exactly one EBCDIC routing label.
+ * Individual IBM variants must NOT appear as top-level labels — they live
+ * only in the EBCDIC sub-model.
+ */
+ @Test
+ public void generalModelHasSingleEbcdicRoutingLabel() {
+ LinearModel general = detector.getModel();
+ String[] labels = general.getLabels();
+
+ assertTrue(Arrays.asList(labels).contains("EBCDIC"),
+ "General model must have an 'EBCDIC' routing label");
+
+ // No individual IBM variant should appear as a direct label in the general model —
+ // they live only in the EBCDIC sub-model
+ for (String label : labels) {
+ assertFalse(label.startsWith("IBM"),
+ "General model must not contain individual IBM variant: " + label);
+ }
+ }
+
+ /**
+ * IBM500 bytes must route through the sub-model and return a specific IBM variant,
+ * not the bare "EBCDIC" routing label.
+ */
+ @Test
+ public void ibm500RoutesToSubModel() throws Exception {
+ try (TikaInputStream tis = TikaInputStream.get(IBM500_BYTES)) {
+ List<EncodingResult> results = detector.detect(tis, new Metadata(), new ParseContext());
+ assertFalse(results.isEmpty(), "Should detect something for IBM500 bytes");
+ String topLabel = results.get(0).getLabel();
+ assertNotEquals("EBCDIC", topLabel,
+ "Result must be a specific IBM variant, not the routing label");
+ assertTrue(topLabel.startsWith("IBM"),
+ "Result should be an IBM variant, got: " + topLabel);
+ }
+ }
+
+ /**
+ * IBM855 (Cyrillic EBCDIC) bytes must similarly route through the sub-model.
+ */
+ @Test
+ public void ibm855RoutesToSubModel() throws Exception {
+ try (TikaInputStream tis = TikaInputStream.get(IBM855_BYTES)) {
+ List<EncodingResult> results = detector.detect(tis, new Metadata(), new ParseContext());
+ assertFalse(results.isEmpty(), "Should detect something for IBM855 bytes");
+ String topLabel = results.get(0).getLabel();
+ assertNotEquals("EBCDIC", topLabel,
+ "Result must be a specific IBM variant, not the routing label");
+ assertTrue(topLabel.startsWith("IBM"),
+ "Result should be an IBM variant, got: " + topLabel);
+ }
+ }
+}
diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ZipFilenameDetectionTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ZipFilenameDetectionTest.java
index 8c94ec1e62..e5ff43e10e 100644
--- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ZipFilenameDetectionTest.java
+++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ZipFilenameDetectionTest.java
@@ -20,31 +20,34 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.charset.Charset;
-import java.util.Arrays;
import java.util.List;
-import java.util.Locale;
import org.junit.jupiter.api.Test;
+import org.apache.tika.detect.DefaultEncodingDetector;
import org.apache.tika.detect.EncodingDetectorContext;
import org.apache.tika.detect.EncodingResult;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.langdetect.charsoup.CharSoupEncodingDetector;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.ml.LinearModel;
import org.apache.tika.parser.ParseContext;
/**
- * Diagnostic: raw logits and CharSoup arbitration for Shift-JIS zip entry name bytes.
+ * Integration tests for charset detection of short byte sequences typical of
+ * ZIP entry names — a particularly hard case because the probes are tiny (6-23
+ * bytes) and structurally valid in several encodings simultaneously.
*
- * The v2 model (28 classes) removes the UTF-16/32 labels that were confusing the model.
- * With v2, Shift-JIS (logit ~10.5) scores clearly above GB18030 (logit ~6.2) for the
- * bytes "文章1.txt" in Shift-JIS encoding.
+ * Detection strategy: Mojibuster ranks candidates by raw logit; CharSoup
+ * arbitrates using language signal (positive max-logit wins).
*/
public class ZipFilenameDetectionTest {
// 文章1.txt in Shift-JIS (9 raw bytes from a real zip entry)
- private static final byte[] SJIS_RAW = hexToBytes("95b68fcd312e747874");
+ private static final byte[] SJIS_RAW = hexToBytes("95b68fcd312e747874");
+ // 文章2.txt in Shift-JIS (same but '2' instead of '1')
+ private static final byte[] SJIS_RAW2 = hexToBytes("95b68fcd322e747874");
+ // 审计压缩包文件检索测试/ in GBK (23 bytes from gbk.zip)
+ private static final byte[] GBK_RAW = hexToBytes("c9f3bcc6d1b9cbf5b0fccec4bcfebceccbf7b2e2cad42f");
private static byte[] hexToBytes(String hex) {
byte[] b = new byte[hex.length() / 2];
@@ -54,79 +57,21 @@ public class ZipFilenameDetectionTest {
return b;
}
- private boolean isWideUnicode(String label) {
- return label.startsWith("UTF-16") || label.startsWith("UTF-32");
- }
-
- @Test
- public void printModelLabels() throws Exception {
- LinearModel model = new MojibusterEncodingDetector().getModel();
- String[] labels = model.getLabels();
- System.out.println("Model labels (" + labels.length + "):");
- for (String l : labels) {
- System.out.println(" " + l);
- }
- long wideCount = Arrays.stream(labels).filter(this::isWideUnicode).count();
- System.out.println("Wide-unicode labels in model: " + wideCount
- + " (detected natively via stride-2 features)");
- assertTrue(wideCount >= 4, "Model should have UTF-16/32 labels (LE+BE for each)");
- }
-
- @Test
- public void diagnoseLogits() throws Exception {
- MojibusterEncodingDetector detector = new MojibusterEncodingDetector();
- LinearModel model = detector.getModel();
- ByteNgramFeatureExtractor extractor =
- new ByteNgramFeatureExtractor(model.getNumBuckets());
- String[] labels = model.getLabels();
-
- float[] logits = model.predictLogits(extractor.extract(SJIS_RAW));
-
- Integer[] idx = new Integer[labels.length];
- for (int i = 0; i < idx.length; i++) {
- idx[i] = i;
- }
- Arrays.sort(idx, (a, b) -> Float.compare(logits[b], logits[a]));
-
- System.out.printf(Locale.ROOT, "%n=== Raw logits for 文章1.txt (9 bytes) ===%n");
- System.out.printf(Locale.ROOT, "%-24s %8s%n", "charset", "logit");
- System.out.println("-".repeat(35));
- float shiftJisLogit = Float.NEGATIVE_INFINITY;
- float gb18030Logit = Float.NEGATIVE_INFINITY;
- for (int rank = 0; rank < labels.length; rank++) {
- int i = idx[rank];
- boolean cjk = labels[i].contains("JIS") || labels[i].contains("GB")
- || labels[i].contains("Big5") || labels[i].contains("EUC");
- if (rank < 6 || cjk) {
- System.out.printf(Locale.ROOT, " %-24s %8.2f%n", labels[i], logits[i]);
- }
- if ("Shift_JIS".equals(labels[i])) {
- shiftJisLogit = logits[i];
- } else if ("GB18030".equals(labels[i])) {
- gb18030Logit = logits[i];
- }
- }
- // Verify Shift-JIS ranks ahead of GB18030 on raw (un-tiled) bytes.
- // ZipParser no longer tiles short filenames, so this is the actual input.
- assertTrue(shiftJisLogit > gb18030Logit,
- String.format(Locale.ROOT,
- "Shift_JIS logit (%.2f) should beat GB18030 logit (%.2f)",
- shiftJisLogit, gb18030Logit));
- }
-
/**
- * Verifies CharSoup correctly picks Shift-JIS when it and GB18030 are both candidates.
- * With v2 model, Mojibuster already ranks Shift-JIS above GB18030 (logit ~10.5 vs ~6.2).
- * This test uses Shift-JIS as the higher-confidence candidate to reflect that reality.
+ * CharSoup should confirm Shift-JIS even when Mojibuster ranks Big5-HKSCS first,
+ * because the language model gives a higher logit to the Japanese text decoded
+ * from the same bytes.
*/
@Test
- public void charSoupPicksShiftJis() throws Exception {
+ public void charSoupOverridesModelRankingForShiftJis() throws Exception {
+ Charset big5 = Charset.forName("Big5-HKSCS");
Charset shiftJis = Charset.forName("Shift_JIS");
- Charset gb18030 = Charset.forName("GB18030");
EncodingDetectorContext ctx = new EncodingDetectorContext();
- ctx.addResult(List.of(new EncodingResult(shiftJis, 0.6f)), "MojibusterEncodingDetector");
- ctx.addResult(List.of(new EncodingResult(gb18030, 0.5f)), "MojibusterEncodingDetector");
+ ctx.addResult(List.of(
+ new EncodingResult(big5, 0.9f, "Big5-HKSCS", EncodingResult.ResultType.STATISTICAL),
+ new EncodingResult(shiftJis, 0.3f, "Shift_JIS", EncodingResult.ResultType.STATISTICAL)
+ ), "MojibusterEncodingDetector");
ParseContext parseContext = new ParseContext();
parseContext.set(EncodingDetectorContext.class, ctx);
@@ -134,17 +79,46 @@ public class ZipFilenameDetectionTest {
CharSoupEncodingDetector charSoup = new CharSoupEncodingDetector();
try (TikaInputStream tis = TikaInputStream.get(SJIS_RAW)) {
List<EncodingResult> result = charSoup.detect(tis, new Metadata(), parseContext);
+ assertTrue(!result.isEmpty(), "CharSoup should return a result");
+ assertEquals(shiftJis, result.get(0).getCharset(),
+ "CharSoup should pick Shift-JIS (文章) over Big5-HKSCS via language signal");
+ }
+ }
- System.out.println("\n=== CharSoup arbitration: Shift-JIS(0.6) vs GB18030(0.5) ===");
- System.out.println("arbitration: " + ctx.getArbitrationInfo());
- if (!result.isEmpty()) {
- System.out.printf(Locale.ROOT, "winner: %s (conf=%.4f)%n",
- result.get(0).getCharset().name(), result.get(0).getConfidence());
- assertEquals(shiftJis, result.get(0).getCharset(),
- "CharSoup should confirm Shift-JIS (文章) over GB18030");
- } else {
- System.out.println("result: empty — CharSoup abstained (Mojibuster winner stands)");
+ /**
+ * Full pipeline (BOM → Metadata → Mojibuster → StandardHtml → CharSoup) run
+ * sequentially on two entries differing only in byte 5 (0x31 vs 0x32), simulating
+ * what ZipParser does when iterating entries with the same ParseContext.
+ */
+ @Test
+ public void fullPipelineDetectsBothSjisEntries() throws Exception {
+ DefaultEncodingDetector detector = new DefaultEncodingDetector();
+ Metadata parentMeta = new Metadata();
+ ParseContext outerContext = new ParseContext();
+
+ for (byte[] raw : new byte[][]{SJIS_RAW, SJIS_RAW2}) {
+ String label = (raw == SJIS_RAW) ? "文章1.txt" : "文章2.txt";
+ try (TikaInputStream tis = TikaInputStream.get(raw)) {
+ List<EncodingResult> results = detector.detect(tis, parentMeta, outerContext);
+ String charset = results.isEmpty() ? "(empty)" : results.get(0).getCharset().name();
+ assertTrue(!results.isEmpty() && "Shift_JIS".equals(results.get(0).getCharset().name()),
+ label + " should be detected as Shift_JIS, got: " + charset);
}
}
}
+
+ /**
+ * Full pipeline should detect GBK-encoded entry names as GB18030.
+ */
+ @Test
+ public void fullPipelineDetectsGbkEntry() throws Exception {
+ DefaultEncodingDetector detector = new DefaultEncodingDetector();
+ Metadata meta = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(GBK_RAW)) {
+ List<EncodingResult> results = detector.detect(tis, meta, new ParseContext());
+ String charset = results.isEmpty() ? "(empty)" : results.get(0).getCharset().name();
+ assertTrue(!results.isEmpty() && results.get(0).getCharset().name().startsWith("GB"),
+ "GBK entry should be detected as GB18030/GBK, got: " + charset);
+ }
+ }
}
diff --git a/tika-encoding-detectors/tika-encoding-detector-universal/pom.xml b/tika-encoding-detectors/tika-encoding-detector-universal/pom.xml
index e461556a93..36f0f8fd53 100644
--- a/tika-encoding-detectors/tika-encoding-detector-universal/pom.xml
+++ b/tika-encoding-detectors/tika-encoding-detector-universal/pom.xml
@@ -39,6 +39,12 @@
<artifactId>tika-core</artifactId>
<version>${revision}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-annotation-processor</artifactId>
+ <version>${revision}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>com.github.albfernandez</groupId>
<artifactId>juniversalchardet</artifactId>
diff --git a/tika-encoding-detectors/tika-encoding-detector-universal/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-universal/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
index 86d1780926..4e1c2dc1a1 100644
--- a/tika-encoding-detectors/tika-encoding-detector-universal/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-universal/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
@@ -31,7 +31,7 @@ import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-@TikaComponent(spi = false)
+@TikaComponent(spi = false, name = "universal-encoding-detector")
public class UniversalEncodingDetector implements EncodingDetector {
private static final int BUFSIZE = 1024;
diff --git a/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupLanguageDetector.java b/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupLanguageDetector.java
index 75fccbb806..b408ee1841 100644
--- a/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupLanguageDetector.java
+++ b/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupLanguageDetector.java
@@ -274,19 +274,13 @@ public class CharSoupLanguageDetector extends LanguageDetector {
}
/**
- * Minimum confidence (inverse logit of the max logit) for a candidate to
- * be considered a genuine language match. If no candidate exceeds this
- * threshold, the comparison is inconclusive and {@code null} is returned.
- * <p>
- * 0.88 corresponds to a raw logit of ~2.0. Typical values:
- * <ul>
- * <li>Arabic (windows-1256): 0.9999994 (logit +14.3)</li>
- * <li>UTF-8 garbled: 0.97 (logit +3.5)</li>
- * <li>EBCDIC garbage: 0.79 (logit +1.3) — below threshold</li>
- * <li>Short English: 0.025 (logit -3.7) — well below threshold</li>
- * </ul>
+ * The language model's max pre-softmax logit must be positive (sigmoid > 0.5)
+ * for a candidate to be considered a genuine language match. A positive logit
+ * means the model actively predicts some language as more likely than random;
+ * a negative logit means the text is too short, too junk-heavy, or too ambiguous
+ * for any language to stand out. When the best candidate's logit is positive,
+ * we always return it — the model's relative ordering is the signal we trust.
*/
- private static final float MIN_CONFIDENCE_THRESHOLD = 0.88f;
/**
* Maximum ratio of junk characters (U+FFFD replacement chars + C0/C1
@@ -307,24 +301,24 @@ public class CharSoupLanguageDetector extends LanguageDetector {
* Compare multiple candidate texts and return the key of the one with
* the strongest language signal. Candidates with a high ratio of
* replacement or control characters are discarded first. Remaining
- * candidates are scored using the inverse logit (sigmoid) of the
- * model's maximum pre-softmax logit.
+ * candidates are scored using the model's maximum pre-softmax logit.
* <p>
- * Returns {@code null} if no candidate exceeds the minimum confidence
- * threshold, indicating the comparison is inconclusive.
+ * The winning candidate is returned if its max logit is positive (sigmoid > 0.5),
+ * meaning the model actively predicts some language as more likely than random.
+ * Returns {@code null} if the map is empty, all candidates are junk, or the
+ * best candidate's logit is non-positive (model has no real signal).
*
* @param candidates map of arbitrary keys to candidate text strings
* @param <K> key type (e.g., {@link java.nio.charset.Charset})
* @return the key whose text has the strongest language signal,
- * or {@code null} if the map is empty or no candidate is
- * confident enough
+ * or {@code null} if no candidate has a positive language signal
*/
public <K> K compareLanguageSignal(Map<K, String> candidates) {
if (candidates.isEmpty()) {
return null;
}
- float bestConfidence = Float.NEGATIVE_INFINITY;
+ float bestMaxLogit = Float.NEGATIVE_INFINITY;
K bestKey = null;
for (Map.Entry<K, String> entry : candidates.entrySet()) {
@@ -337,24 +331,22 @@ public class CharSoupLanguageDetector extends LanguageDetector {
int[] features = EXTRACTOR.extract(entry.getValue());
float[] logits = MODEL.predictLogits(features);
- float confidence = sigmoid(max(logits));
+ float maxLogit = max(logits);
- LOG.debug("compareLanguageSignal: {} -> confidence={}",
- entry.getKey(), confidence);
+ LOG.debug("compareLanguageSignal: {} -> maxLogit={}", entry.getKey(), maxLogit);
- if (confidence > bestConfidence) {
- bestConfidence = confidence;
+ if (maxLogit > bestMaxLogit) {
+ bestMaxLogit = maxLogit;
bestKey = entry.getKey();
}
}
- if (bestConfidence < MIN_CONFIDENCE_THRESHOLD) {
- LOG.debug("compareLanguageSignal: inconclusive (bestConfidence={} < {})",
- bestConfidence, MIN_CONFIDENCE_THRESHOLD);
- return null;
+ if (bestKey != null && bestMaxLogit > 0) {
+ return bestKey;
}
- return bestKey;
+ LOG.debug("compareLanguageSignal: inconclusive (bestMaxLogit={})", bestMaxLogit);
+ return null;
}
/**
@@ -362,7 +354,7 @@ public class CharSoupLanguageDetector extends LanguageDetector {
* with the highest logit, its raw logit value, and sigmoid(maxLogit).
* Package-private for testing.
*/
- static float[] maxLogitInfo(String text) {
+ public static float[] maxLogitInfo(String text) {
int[] features = EXTRACTOR.extract(text);
float[] logits = MODEL.predictLogits(features);
int bestIdx = 0;
@@ -375,7 +367,7 @@ public class CharSoupLanguageDetector extends LanguageDetector {
}
/** Returns the label for a class index (for use alongside {@link #maxLogitInfo}). */
- static String labelAt(int idx) {
+ public static String labelAt(int idx) {
return MODEL.getLabel(idx);
}
@@ -383,6 +375,24 @@ public class CharSoupLanguageDetector extends LanguageDetector {
* Ratio of junk characters (U+FFFD replacement + ISO control + C1
* control range U+0080-U+009F) to total characters. High values
* indicate a wrong-charset decoding.
+ * <p>
+ * TODO: consider also counting non-ASCII, non-alphabetic, non-digit characters
+ * (e.g. bullet U+2022, pilcrow U+00B6) as fractional junk (weight ~0.3).
+ * Single-byte encodings like windows-1256 assign punctuation/symbols to byte
+ * positions like 0x95 and 0xB6 that multi-byte encodings (Shift_JIS, GB18030)
+ * use as lead bytes for alphabetic characters. When those bytes appear in text
+ * that should be meaningful (e.g. filenames), the single-byte interpretation
+ * "wastes" bytes on punctuation while the multi-byte interpretation yields 100%
+ * alphabetic content. Counting such punctuation as partial junk would lower the
+ * MAX_JUNK_RATIO gate for those candidates and pass the decision to the language
+ * model sooner. Needs careful tuning: legitimate body text can intentionally
+ * contain bullet lists, em-dashes, etc. in windows-125x encodings.
+ * Counter-example: Chinese GB18030 bytes decoded as UTF-16 produce pairs
+ * interpreted as Unicode code points — many of which happen to be alphabetic
+ * (Unicode has alphabetic characters scattered throughout the range), so
+ * alphabetic yield would look high even for complete mojibake. The language
+ * model already handles this correctly; the alphabetic density heuristic alone
+ * would not.
*/
static float junkRatio(String text) {
if (text == null || text.isEmpty()) {
@@ -394,8 +404,17 @@ public class CharSoupLanguageDetector extends LanguageDetector {
int cp = text.codePointAt(i);
i += Character.charCount(cp);
total++;
- if (cp == 0xFFFD || Character.isISOControl(cp)) {
+ // U+FFFD = replacement char (wrong-charset decode)
+ // C1 control range 0x80-0x9F = garbage when a byte is decoded as ISO-8859-1
+ // (those code points are never produced by correct Windows-125x decoding).
+ // Ordinary whitespace (tab, LF, CR, FF, VT) is not junk — it appears in
+ // source code and structured documents regardless of charset.
+ if (cp == 0xFFFD) {
junk++;
+ } else if (Character.isISOControl(cp)) {
+ if (cp != 0x09 && cp != 0x0A && cp != 0x0B && cp != 0x0C && cp != 0x0D) {
+ junk++;
+ }
}
}
return total == 0 ? 0f : (float) junk / total;
diff --git a/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/ByteNgramFeatureExtractorTest.java b/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/ByteNgramFeatureExtractorTest.java
index 80e71a9bdf..949bd0bc05 100644
--- a/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/ByteNgramFeatureExtractorTest.java
+++ b/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/ByteNgramFeatureExtractorTest.java
@@ -46,17 +46,19 @@ public class ByteNgramFeatureExtractorTest {
}
@Test
- public void testAsciiOnlyProducesNoFeatures() {
+ public void testAsciiOnlyProducesStride2Features() {
ByteNgramFeatureExtractor ext = new ByteNgramFeatureExtractor(NUM_BUCKETS);
- // All bytes < 0x80 are skipped — HTML tags, ASCII text, etc. produce nothing
+ // Stride-1 skips bytes < 0x80, but stride-2 covers ALL bytes (needed for UTF-16/32
+ // null-byte detection). "hello world" (11 bytes) → 5 stride-2 pairs at positions
+ // 0,2,4,6,8 → 5 features total.
byte[] ascii = "hello world".getBytes(java.nio.charset.StandardCharsets.US_ASCII);
- assertEquals(0, sum(ext.extract(ascii)));
+ assertEquals(5, sum(ext.extract(ascii)));
}
@Test
public void testSingleHighByteProducesOneUnigram() {
ByteNgramFeatureExtractor ext = new ByteNgramFeatureExtractor(NUM_BUCKETS);
- // One high byte → one unigram, no bigram (no following byte)
+ // One high byte, no following byte → 1 stride-1 unigram; no stride-2 pair
int[] counts = ext.extract(new byte[]{(byte) 0xE0});
assertEquals(1, sum(counts));
}
@@ -64,25 +66,31 @@ public class ByteNgramFeatureExtractorTest {
@Test
public void testTwoHighBytesProduceUnigramAndBigram() {
ByteNgramFeatureExtractor ext = new ByteNgramFeatureExtractor(NUM_BUCKETS);
- // 0xE0 → unigram; (0xE0, 0xE1) → bigram; 0xE1 → unigram = 3 features
+ // Stride-1: unigram(0xE0) + bigram(0xE0,0xE1) + unigram(0xE1) = 3
+ // Stride-2: pair(0xE0,0xE1) at position 0 = 1
+ // Total = 4
int[] counts = ext.extract(new byte[]{(byte) 0xE0, (byte) 0xE1});
- assertEquals(3, sum(counts));
+ assertEquals(4, sum(counts));
}
@Test
- public void testHighByteFollowedByAsciiProducesUnigramAndBigram() {
+ public void testHighByteFollowedByAsciiProducesUnigramBigramAndAnchoredBigram() {
ByteNgramFeatureExtractor ext = new ByteNgramFeatureExtractor(NUM_BUCKETS);
- // 0xE0 → unigram; (0xE0, 0x41) → bigram; 0x41 is ASCII so no further features = 2
+ // Stride-1: unigram(0xE0) + bigram(0xE0,0x41) + anchored_bigram(0x41,end) = 3
+ // Stride-2: pair(0xE0,0x41) at position 0 = 1
+ // Total = 4
int[] counts = ext.extract(new byte[]{(byte) 0xE0, 0x41});
- assertEquals(2, sum(counts));
+ assertEquals(4, sum(counts));
}
@Test
- public void testAsciiFollowedByHighByteProducesUnigramAndBigram() {
+ public void testAsciiFollowedByHighByteProducesUnigramAndStride2() {
ByteNgramFeatureExtractor ext = new ByteNgramFeatureExtractor(NUM_BUCKETS);
- // 0x41 skipped; 0xE0 → unigram; no following byte → 1 feature
+ // Stride-1: 0x41 skipped; unigram(0xE0), no following byte = 1
+ // Stride-2: pair(0x41,0xE0) at position 0 = 1
+ // Total = 2
int[] counts = ext.extract(new byte[]{0x41, (byte) 0xE0});
- assertEquals(1, sum(counts));
+ assertEquals(2, sum(counts));
}
@Test
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
index 493d47d5a9..b01dedded4 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
@@ -1004,8 +1004,8 @@ public class HtmlParserTest extends TikaTest {
}
assertEquals("text/html; charset=UTF-ELEVEN",
metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT));
- // "UTF-ELEVEN" is not a valid charset; ML detection returns UTF-8 for ASCII content.
- assertEquals("text/html; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ // "UTF-ELEVEN" is not a valid charset; no declaration available, ML defaults to windows-1252.
+ assertEquals("text/html; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
test = "<html><head><meta http-equiv=\"content-type\" content=\"application/pdf\">" +
"</head><title>title</title><body>body</body></html>";
@@ -1017,8 +1017,8 @@ public class HtmlParserTest extends TikaTest {
metadata, new ParseContext());
}
assertEquals("application/pdf", metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT));
- // No valid charset declaration; ML detection returns UTF-8 for ASCII content.
- assertEquals("text/html; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ // No valid charset declaration; ML defaults to windows-1252 for pure ASCII content.
+ assertEquals("text/html; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
//test two content values
test =
@@ -1033,8 +1033,8 @@ public class HtmlParserTest extends TikaTest {
metadata, new ParseContext());
}
assertEquals("application/pdf", metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT));
- // No valid charset declaration; ML detection returns UTF-8 for ASCII content.
- assertEquals("text/html; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ // No valid charset declaration; ML defaults to windows-1252 for pure ASCII content.
+ assertEquals("text/html; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
}
@Test
@@ -1074,8 +1074,8 @@ public class HtmlParserTest extends TikaTest {
assertEquals("text/html; charset=iso-NUMBER_SEVEN",
metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT));
- // "iso-NUMBER_SEVEN" is not a valid charset; ML detection returns UTF-8 for ASCII content.
- assertEquals("application/xhtml+xml; charset=UTF-8",
+ // "iso-NUMBER_SEVEN" is not a valid charset; ML defaults to windows-1252 for pure ASCII.
+ assertEquals("application/xhtml+xml; charset=windows-1252",
metadata.get(Metadata.CONTENT_TYPE));
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
index b0080ee761..883899fa14 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java
@@ -186,7 +186,8 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe
expected.add("application/vnd.openxmlformats-officedocument.presentationml.presentation");
expected.add("application/pdf");
expected.add("application/xml");
- expected.add("text/plain; charset=ISO-8859-1");
+ // CRLF line endings in this embedded text file trigger the ISO→Windows upgrade heuristic
+ expected.add("text/plain; charset=windows-1252");
//test that we're correctly handling attachment variants for
// files created by WPS 表格 (https://www.wps.cn/)
for (String suffix : new String[]{
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
index 0f33218e76..62ec3bc8e3 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
@@ -109,6 +109,7 @@ public class ZipParser extends AbstractArchiveParser {
*/
private static final int MAX_INTEGRITY_CHECK_ENTRIES = 100;
+
private final ZipParserConfig defaultConfig;
private static Set<MediaType> loadZipSpecializations() {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
index 227ebd31cb..bcc316d361 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
@@ -34,13 +34,11 @@
<groupId>org.apache.tika</groupId>
<artifactId>tika-encoding-detector-mojibuster</artifactId>
<version>${project.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-encoding-detector-charsoup</artifactId>
<version>${project.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
@@ -50,6 +48,18 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-encoding-detector-icu4j</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-encoding-detector-universal</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/csv/TextAndCSVParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/csv/TextAndCSVParserTest.java
index 67358c5403..a32d063223 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/csv/TextAndCSVParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/csv/TextAndCSVParserTest.java
@@ -101,7 +101,7 @@ public class TextAndCSVParserTest extends TikaTest {
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(TikaInputStream.get(CSV_UTF8), PARSER, metadata);
assertEquals("comma", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
- assertMediaTypeEquals("csv", "UTF-8", "comma",
+ assertMediaTypeEquals("csv", "windows-1252", "comma",
xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_CSV, xmlResult.xml);
assertEquals(3, metadata.getInt(TextAndCSVParser.NUM_COLUMNS));
@@ -126,7 +126,7 @@ public class TextAndCSVParserTest extends TikaTest {
metadata.set(Metadata.CONTENT_TYPE, "text/csv");
XMLResult xmlResult = getXML(TikaInputStream.get(CSV_UTF8), PARSER, metadata);
assertEquals("comma", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
- assertMediaTypeEquals("csv", "UTF-8", "comma",
+ assertMediaTypeEquals("csv", "windows-1252", "comma",
xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_CSV, xmlResult.xml);
}
@@ -160,7 +160,7 @@ public class TextAndCSVParserTest extends TikaTest {
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(TikaInputStream.get(TSV_UTF8), PARSER, metadata);
assertEquals("tab", xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
- assertMediaTypeEquals("tsv", "UTF-8", "tab",
+ assertMediaTypeEquals("tsv", "windows-1252", "tab",
xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContainsIgnoreWhiteSpaceDiffs(EXPECTED_TSV, xmlResult.xml);
}
@@ -191,7 +191,7 @@ public class TextAndCSVParserTest extends TikaTest {
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.csv");
XMLResult xmlResult = getXML(TikaInputStream.get(csv), PARSER, metadata);
assertNull(xmlResult.metadata.get(TextAndCSVParser.DELIMITER_PROPERTY));
- assertEquals("text/plain; charset=UTF-8",
+ assertEquals("text/plain; charset=windows-1252",
xmlResult.metadata.get(Metadata.CONTENT_TYPE));
assertContains("the,quick", xmlResult.xml);
}
@@ -225,7 +225,7 @@ public class TextAndCSVParserTest extends TikaTest {
XMLResult xmlResult =
getXML(TikaInputStream.get(sb.toString().getBytes(StandardCharsets.UTF_8)),
PARSER, metadata);
- assertMediaTypeEquals("csv", "UTF-8", "comma",
+ assertMediaTypeEquals("csv", "windows-1252", "comma",
xmlResult.metadata.get(Metadata.CONTENT_TYPE));
}
@@ -233,8 +233,7 @@ public class TextAndCSVParserTest extends TikaTest {
@Test
public void testSubclassingMimeTypesRemain() throws Exception {
XMLResult r = getXML("testVCalendar.vcs");
- // Pure ASCII content — correctly detected as UTF-8
- assertEquals("text/x-vcalendar; charset=UTF-8", r.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("text/x-vcalendar; charset=windows-1252", r.metadata.get(Metadata.CONTENT_TYPE));
}
@Test
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
index a29fb299f1..b02236a4e5 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
@@ -29,9 +29,11 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.TikaTest;
+import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
@@ -54,8 +56,8 @@ public class TXTParserTest extends TikaTest {
}
String content = writer.toString();
- // Pure ASCII — correctly detected as UTF-8 (ASCII is a subset of UTF-8)
- assertEquals("text/plain; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ // Pure ASCII — detected as windows-1252 (the HTML5/WHATWG default for 8-bit Western)
+ assertEquals("text/plain; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
// TIKA-501: Remove language detection from TXTParser
assertNull(metadata.get(Metadata.CONTENT_LANGUAGE));
@@ -89,7 +91,7 @@ public class TXTParserTest extends TikaTest {
try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
parser.parse(tis, handler, metadata, new ParseContext());
}
- assertEquals("text/plain; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("text/plain; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
assertEquals("\n", handler.toString());
}
@@ -102,9 +104,9 @@ public class TXTParserTest extends TikaTest {
*/
@Test
public void testLatinDetectionHeuristics() throws Exception {
- // These were previously testing CR/LF heuristics specific to UniversalEncodingDetector.
- // The ML-based detector (MojibusterEncodingDetector + CharSoup) correctly identifies
- // pure-ASCII content as UTF-8 and does not rely on line-ending heuristics.
+ // Previously tested CR/LF heuristics specific to UniversalEncodingDetector.
+ // The ML-based detector defaults to windows-1252 for pure ASCII regardless of
+ // line endings (CRLF_TO_WINDOWS is a secondary confirmation, not the primary path).
String windows = "test\r\n";
String unix = "test\n";
String euro = "test \u20ac\n";
@@ -115,15 +117,13 @@ public class TXTParserTest extends TikaTest {
try (TikaInputStream tis = TikaInputStream.get(windows.getBytes("ISO-8859-15"))) {
parser.parse(tis, new DefaultHandler(), metadata, new ParseContext());
}
- // Pure ASCII — UTF-8 is correct
- assertEquals("text/plain; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("text/plain; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
metadata = new Metadata();
try (TikaInputStream tis = TikaInputStream.get(unix.getBytes("ISO-8859-15"))) {
parser.parse(tis, new DefaultHandler(), metadata, new ParseContext());
}
- // Pure ASCII — UTF-8 is correct
- assertEquals("text/plain; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("text/plain; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
metadata = new Metadata();
try (TikaInputStream tis = TikaInputStream.get(euro.getBytes("ISO-8859-15"))) {
@@ -247,8 +247,7 @@ public class TXTParserTest extends TikaTest {
parser.parse(tis, new WriteOutContentHandler(writer), metadata, new ParseContext());
}
- // Pure ASCII — UTF-8 is correct
- assertEquals("text/plain; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("text/plain; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
}
/**
@@ -264,11 +263,10 @@ public class TXTParserTest extends TikaTest {
try (TikaInputStream tis = TikaInputStream.get(text.getBytes(UTF_8))) {
parser.parse(tis, new BodyContentHandler(), metadata, new ParseContext());
}
- // Pure ASCII — UTF-8 is correct
- assertEquals("text/plain; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("text/plain; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE));
- // Now verify that if we tell the parser the encoding is UTF-8, that's what
- // we get back (see TIKA-868)
+ // TIKA-868: MetadataCharsetDetector (tika-core) reads the charset from Content-Type
+ // and returns it as DECLARATIVE, which CharSoup prefers over the statistical windows-1252.
metadata.set(Metadata.CONTENT_TYPE, "application/binary; charset=UTF-8");
try (TikaInputStream tis = TikaInputStream.get(text.getBytes(UTF_8))) {
parser.parse(tis, new BodyContentHandler(), metadata, new ParseContext());
@@ -280,8 +278,23 @@ public class TXTParserTest extends TikaTest {
@Test
public void testSubclassingMimeTypesRemain() throws Exception {
XMLResult r = getXML("testVCalendar.vcs");
- // Pure ASCII content — correctly detected as UTF-8
- assertEquals("text/x-vcalendar; charset=UTF-8", r.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("text/x-vcalendar; charset=windows-1252", r.metadata.get(Metadata.CONTENT_TYPE));
+ }
+
+ // TIKA-3516, TIKA-3525, TIKA-1236
+ @Test
+ public void testIgnoreCharset() throws Exception {
+ AutoDetectParser parser = (AutoDetectParser) TikaLoader.load(
+ getConfigPath(TXTParserTest.class, "tika-config-ignore-charset.json"))
+ .loadAutoDetectParser();
+
+ Metadata m = new Metadata();
+ m.set(TikaCoreProperties.RESOURCE_NAME_KEY, "texty-text.txt");
+ assertContains("ACTIVE AGE", getXML("testIgnoreCharset.txt", parser, m).xml);
+
+ m = new Metadata();
+ m.set(TikaCoreProperties.RESOURCE_NAME_KEY, "texty-text.txt");
+ assertContains("Please check your email", getXML("test_ignore_IBM420.html", parser, m).xml);
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
index 73928000bb..31421a12c9 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
@@ -32,12 +32,15 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.TikaLoaderHelper;
import org.apache.tika.TikaTest;
import org.apache.tika.config.loader.TikaLoader;
+import org.apache.tika.detect.BOMDetector;
import org.apache.tika.detect.CompositeEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.detect.MetaEncodingDetector;
+import org.apache.tika.detect.MetadataCharsetDetector;
import org.apache.tika.detect.OverrideEncodingDetector;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.langdetect.charsoup.CharSoupEncodingDetector;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.ml.chardetect.MojibusterEncodingDetector;
@@ -56,13 +59,15 @@ public class TikaEncodingDetectorTest extends TikaTest {
EncodingDetector detector = TikaLoader.loadDefault().loadEncodingDetectors();
assertTrue(detector instanceof CompositeEncodingDetector);
List<EncodingDetector> detectors = ((CompositeEncodingDetector) detector).getDetectors();
- // 2 base detectors (ML, StandardHtml) + CharSoupEncodingDetector (MetaEncodingDetector)
- assertEquals(3, detectors.size());
+ // 4 base detectors (BOM, Metadata, ML, StandardHtml) + CharSoupEncodingDetector (MetaEncodingDetector)
+ assertEquals(5, detectors.size());
// meta detector is always last (partitioned by CompositeEncodingDetector)
- assertTrue(detectors.get(2) instanceof MetaEncodingDetector);
+ assertTrue(detectors.get(4) instanceof MetaEncodingDetector);
// base detectors — sorted by full class name; check by type
- Set<Class<?>> baseClasses = detectors.subList(0, 2).stream()
+ Set<Class<?>> baseClasses = detectors.subList(0, 4).stream()
.map(Object::getClass).collect(Collectors.toSet());
+ assertTrue(baseClasses.contains(BOMDetector.class));
+ assertTrue(baseClasses.contains(MetadataCharsetDetector.class));
assertTrue(baseClasses.contains(MojibusterEncodingDetector.class));
assertTrue(baseClasses.contains(StandardHtmlEncodingDetector.class));
}
@@ -81,12 +86,14 @@ public class TikaEncodingDetectorTest extends TikaTest {
assertTrue(detector1 instanceof CompositeEncodingDetector);
List<EncodingDetector> detectors1Children =
((CompositeEncodingDetector) detector1).getDetectors();
- // ML base detector + CharSoup meta (html excluded)
- assertEquals(2, detectors1Children.size());
- Set<Class<?>> innerClasses = detectors1Children.subList(0, 1).stream()
+ // BOM + Metadata + ML base detectors + CharSoup meta (html excluded)
+ assertEquals(4, detectors1Children.size());
+ Set<Class<?>> innerClasses = detectors1Children.subList(0, 3).stream()
.map(Object::getClass).collect(Collectors.toSet());
+ assertTrue(innerClasses.contains(BOMDetector.class));
+ assertTrue(innerClasses.contains(MetadataCharsetDetector.class));
assertTrue(innerClasses.contains(MojibusterEncodingDetector.class));
- assertTrue(detectors1Children.get(1) instanceof MetaEncodingDetector);
+ assertTrue(detectors1Children.get(3) instanceof MetaEncodingDetector);
assertTrue(detectors.get(1) instanceof OverrideEncodingDetector);
@@ -178,9 +185,9 @@ public class TikaEncodingDetectorTest extends TikaTest {
((AbstractEncodingDetectorParser) encodingDetectingParser)
.getEncodingDetector();
assertTrue(encodingDetector instanceof CompositeEncodingDetector);
- // ML, Html base detectors + CharSoup MetaEncodingDetector
+ // BOM, Metadata, ML, Html base detectors + CharSoup MetaEncodingDetector
// (ICU4J is excluded but was already not in the default chain)
- assertEquals(3, ((CompositeEncodingDetector) encodingDetector).getDetectors().size());
+ assertEquals(5, ((CompositeEncodingDetector) encodingDetector).getDetectors().size());
for (EncodingDetector child : ((CompositeEncodingDetector) encodingDetector)
.getDetectors()) {
assertNotContained("cu4j", child.getClass().getCanonicalName());
@@ -207,14 +214,17 @@ public class TikaEncodingDetectorTest extends TikaTest {
assertTrue(encodingDetector instanceof CompositeEncodingDetector);
List<EncodingDetector> children =
((CompositeEncodingDetector) encodingDetector).getDetectors();
- assertEquals(3, children.size(), childParser.getClass().toString());
+ // 3 base detectors + 1 MetaEncodingDetector (CharSoup) = 4 total
+ assertEquals(4, children.size(), childParser.getClass().toString());
assertTrue(children.get(0) instanceof MojibusterEncodingDetector,
childParser.getClass().toString());
HtmlEncodingDetector htmlDet = (HtmlEncodingDetector) children.get(1);
- assertEquals(64000, htmlDet.getDefaultConfig().getMarkLimit(),
+ assertEquals(100000, htmlDet.getDefaultConfig().getMarkLimit(),
childParser.getClass().toString());
assertTrue(children.get(2) instanceof StandardHtmlEncodingDetector,
childParser.getClass().toString());
+ assertTrue(children.get(3) instanceof CharSoupEncodingDetector,
+ childParser.getClass().toString());
}
}
@@ -222,7 +232,8 @@ public class TikaEncodingDetectorTest extends TikaTest {
public void testMarkLimitIntegration() throws Exception {
StringBuilder sb = new StringBuilder();
sb.append("<html><head><script>");
- for (int i = 0; i < 4000; i++) { //script length = 20000
+ // script length = ~80000 bytes, beyond the default mark limit of 65536
+ for (int i = 0; i < 16000; i++) {
sb.append("blah ");
}
sb.append("</script>");
@@ -233,19 +244,23 @@ public class TikaEncodingDetectorTest extends TikaTest {
byte[] bytes = sb.toString().getBytes(StandardCharsets.UTF_8);
- // The new pipeline (StandardHtmlEncodingDetector reads past the script block
- // and finds the meta charset) correctly detects UTF-8 even by default.
+ // Default: the meta charset is buried at ~byte 80,000, past the default
+ // mark limit of 65536. The detector falls back to windows-1252 for the
+ // pure-ASCII probe. HTML entities (ø) render correctly regardless;
+ // raw UTF-8 multibyte sequences (e.g. ø in "økologisk") are garbled.
+ // Raise the mark limit via config to fix this (see below).
Parser p = AUTO_DETECT_PARSER;
Metadata metadata = new Metadata();
String xml = getXML(TikaInputStream.get(bytes), p, metadata).xml;
- assertContains("gr\u00F8nd", xml);
- assertContains("\u00f8kologisk", xml);
- assertContains("gr\u00F8nt", xml);
- assertContains("g\u00E5 til", xml);
+ assertContains("gr\u00F8nd", xml); // ø entity — correct regardless
+ assertNotContained("\u00f8kologisk", xml); // raw UTF-8 bytes — garbled by default
+ assertNotContained("gr\u00F8nt", xml);
+ assertNotContained("g\u00E5 til", xml);
- //now test that fix works
+ // With a raised mark limit the detector reaches the meta charset and
+ // correctly decodes UTF-8 content.
p = TikaLoaderHelper.getLoader("TIKA-2485-encoding-detector-mark-limits.json").loadAutoDetectParser();
metadata = new Metadata();
@@ -266,10 +281,12 @@ public class TikaEncodingDetectorTest extends TikaTest {
assertTrue(detector instanceof CompositeEncodingDetector);
List<EncodingDetector> detectors =
((CompositeEncodingDetector) detector).getDetectors();
- // 2 base detectors (ML + StandardHtml), no MetaEncodingDetector
- assertEquals(2, detectors.size());
+ // 4 base detectors (BOM + Metadata + ML + StandardHtml), no MetaEncodingDetector
+ assertEquals(4, detectors.size());
Set<Class<?>> excludedCharSoupClasses = detectors.stream()
.map(Object::getClass).collect(Collectors.toSet());
+ assertTrue(excludedCharSoupClasses.contains(BOMDetector.class));
+ assertTrue(excludedCharSoupClasses.contains(MetadataCharsetDetector.class));
assertTrue(excludedCharSoupClasses.contains(MojibusterEncodingDetector.class));
assertTrue(excludedCharSoupClasses.contains(StandardHtmlEncodingDetector.class));
for (EncodingDetector d : detectors) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/AutoDetectParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/AutoDetectParserTest.java
index 3e05518ebc..8f9b957e90 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/AutoDetectParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/AutoDetectParserTest.java
@@ -55,7 +55,7 @@ public class AutoDetectParserTest extends TikaTest {
// Easy to read constants for the MIME types:
private static final String RAW = "application/octet-stream";
private static final String EXCEL = "application/vnd.ms-excel";
- private static final String HTML = "text/html; charset=UTF-8";
+ private static final String HTML = "text/html; charset=windows-1252";
private static final String PDF = "application/pdf";
private static final String POWERPOINT = "application/vnd.ms-powerpoint";
private static final String KEYNOTE = "application/vnd.apple.keynote";
@@ -63,7 +63,7 @@ public class AutoDetectParserTest extends TikaTest {
private static final String NUMBERS = "application/vnd.apple.numbers";
private static final String CHM = "application/vnd.ms-htmlhelp";
private static final String RTF = "application/rtf";
- private static final String PLAINTEXT = "text/plain; charset=UTF-8";
+ private static final String PLAINTEXT = "text/plain; charset=windows-1252";
private static final String UTF8TEXT = "text/plain; charset=UTF-8";
private static final String WORD = "application/msword";
private static final String XML = "application/xml";
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
index 1411481f1c..7bd45cf813 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
@@ -46,11 +46,11 @@ public class RTFParserTest extends TikaTest {
public void testEmbeddedMonster() throws Exception {
Map<Integer, Pair> expected = new HashMap<>();
- expected.put(3, new Pair("Hw.txt", "text/plain; charset=UTF-8"));
+ expected.put(3, new Pair("Hw.txt", "text/plain; charset=windows-1252"));
expected.put(4, new Pair("file_0.doc", "application/msword"));
expected.put(7, new Pair("file_1.xlsx",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"));
- expected.put(10, new Pair("text.html", "text/html; charset=UTF-8"));
+ expected.put(10, new Pair("text.html", "text/html; charset=windows-1252"));
expected.put(11, new Pair("html-within-zip.zip", "application/zip"));
expected.put(12,
new Pair("test-zip-of-zip_\u666E\u6797\u65AF\u987F.zip", "application/zip"));
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index b7565f11f2..76910c56ff 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -252,7 +252,7 @@ public class PDFParserTest extends TikaTest {
metadatas.get(1).get(Metadata.CONTENT_TYPE));
assertImageContentType("image/tiff",
metadatas.get(2).get(Metadata.CONTENT_TYPE));
- assertEquals("text/plain; charset=UTF-8", metadatas.get(3).get(Metadata.CONTENT_TYPE));
+ assertEquals("text/plain; charset=windows-1252", metadatas.get(3).get(Metadata.CONTENT_TYPE));
assertEquals(TYPE_DOC.toString(), metadatas.get(4).get(Metadata.CONTENT_TYPE));
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/pkg/PackageParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/pkg/PackageParserTest.java
index 785206f8da..bec188b8d8 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/pkg/PackageParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/pkg/PackageParserTest.java
@@ -18,7 +18,6 @@ package org.apache.tika.parser.pkg;
import java.util.List;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
@@ -33,7 +32,6 @@ public class PackageParserTest extends TikaTest {
assertContains("审计压缩", metadataList.get(1).get(TikaCoreProperties.RESOURCE_NAME_KEY));
}
- @Disabled("TIKA-4662: ML model confuses Shift_JIS with Big5 on 9-byte zip entry name probes; needs model improvement")
@Test
public void handleEntryNameWithCharsetShiftJIS() throws Exception {
List<Metadata> metadataList = getRecursiveMetadata("testZipEntryNameCharsetShiftSJIS.zip");
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/TIKA-2485-encoding-detector-mark-limits.json b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/TIKA-2485-encoding-detector-mark-limits.json
index 6da5365b70..8275da4bfc 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/TIKA-2485-encoding-detector-mark-limits.json
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/TIKA-2485-encoding-detector-mark-limits.json
@@ -5,11 +5,16 @@
},
{
"html-encoding-detector": {
- "markLimit": 64000
+ "markLimit": 100000
}
},
{
- "standard-html-encoding-detector": {}
+ "standard-html-encoding-detector": {
+ "markLimit": 100000
+ }
+ },
+ {
+ "charsoup-encoding-detector": {}
}
]
}