You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/03/05 23:13:01 UTC
(tika) 06/09: chardet - WIP
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch chardet-work
in repository https://gitbox.apache.org/repos/asf/tika.git
commit ba5e8ea62c8f04bd4b7967fdeea2786452ff6769
Author: tballison <ti...@elastic.co>
AuthorDate: Wed Mar 4 09:26:01 2026 -0500
chardet - WIP
---
.mvn/maven.config | 1 +
pom.xml | 2 +
.../org/apache/tika/detect/AutoDetectReader.java | 8 +-
.../tika/detect/CompositeEncodingDetector.java | 116 +-
.../tika/detect/DefaultEncodingDetector.java | 37 +-
.../org/apache/tika/detect/EncodingDetector.java | 47 +-
.../tika/detect/EncodingDetectorContext.java | 111 +-
.../tika/detect/OverrideEncodingDetector.java | 6 +-
.../apache/tika/detect/WideUnicodeDetector.java | 230 ++-
.../tika/detect/WideUnicodeDetectorTest.java | 38 +-
tika-langdetect/tika-langdetect-charsoup/pom.xml | 6 +
.../charsoup/CharSoupEncodingDetector.java | 186 --
.../charsoup/CharSoupLanguageDetector.java | 22 +
.../charsoup/CharSoupEncodingDetectorTest.java | 206 ---
.../langdetect/charsoup/TextQualityDiagTest.java | 141 --
.../ml/chardetect/ByteNgramFeatureExtractor.java | 211 ---
.../ml/chardetect/StructuralEncodingRules.java | 422 -----
.../org/apache/tika/ml/chardetect/chardetect.bin | Bin 1835549 -> 0 bytes
.../ml/chardetect/tools/EvalCharsetDetectors.java | 379 ----
.../ml/chardetect/tools/TrainCharsetModel.java | 370 ----
.../src/test/python/build_charset_training.py | 395 +++--
.../tika-parser-html-module/pom.xml | 6 +
.../tika/parser/html/HtmlEncodingDetector.java | 214 ---
.../org/apache/tika/parser/html/JSoupParser.java | 7 +-
.../html/charsetdetector/CharsetAliases.java | 156 --
.../charsetdetector/CharsetDetectionResult.java | 66 -
.../parser/html/charsetdetector/MetaProcessor.java | 76 -
.../parser/html/charsetdetector/PreScanner.java | 297 ----
.../StandardHtmlEncodingDetector.java | 113 --
.../charsets/ReplacementCharset.java | 65 -
.../charsets/XUserDefinedCharset.java | 66 -
.../org.apache.tika.detect.EncodingDetector | 19 -
.../tika/parser/html/HtmlEncodingDetectorTest.java | 6 +-
.../apache/tika/parser/html/HtmlParserTest.java | 10 +-
.../html/StandardHtmlEncodingDetectorTest.java | 6 +-
.../tika-parser-microsoft-module/pom.xml | 6 +
.../tika/parser/microsoft/OutlookExtractor.java | 5 +-
.../tika-parser-miscoffice-module/pom.xml | 6 +
.../java/org/apache/tika/parser/dbf/DBFParser.java | 5 +-
.../java/org/apache/tika/parser/pkg/ZipParser.java | 17 +-
.../tika-parser-text-module/pom.xml | 5 -
.../org/apache/tika/parser/txt/BOMDetector.java | 99 --
.../apache/tika/parser/txt/CharsetDetector.java | 604 -------
.../org/apache/tika/parser/txt/CharsetMatch.java | 291 ----
.../apache/tika/parser/txt/CharsetRecog_2022.java | 159 --
.../apache/tika/parser/txt/CharsetRecog_UTF8.java | 97 --
.../tika/parser/txt/CharsetRecog_Unicode.java | 186 --
.../apache/tika/parser/txt/CharsetRecog_mbcs.java | 536 ------
.../apache/tika/parser/txt/CharsetRecog_sbcs.java | 1807 --------------------
.../apache/tika/parser/txt/CharsetRecognizer.java | 53 -
.../tika/parser/txt/Icu4jEncodingDetector.java | 153 --
.../tika/parser/txt/UniversalEncodingDetector.java | 110 --
.../tika/parser/txt/UniversalEncodingListener.java | 112 --
.../org.apache.tika.detect.EncodingDetector | 21 -
.../apache/tika/parser/txt/BOMDetectorTest.java | 92 -
.../tika/parser/txt/CharsetDetectorTest.java | 149 --
.../tika-parsers-standard-package/pom.xml | 30 +
.../tika/config/TikaEncodingDetectorTest.java | 86 +-
...IKA-2273-exclude-encoding-detector-default.json | 2 +-
59 files changed, 822 insertions(+), 7850 deletions(-)
diff --git a/.mvn/maven.config b/.mvn/maven.config
index f5c8f9f836..f354c80bbf 100644
--- a/.mvn/maven.config
+++ b/.mvn/maven.config
@@ -1,3 +1,4 @@
-Dnisse.compat.osDetector
# fix for nlp module
-DeclipseP2RepoId=invalid
+-Dmaven.repo.local=.local_m2_repo
diff --git a/pom.xml b/pom.xml
index 71e23c5823..9c1a169a1f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,6 +42,8 @@
<module>tika-serialization</module>
<module>tika-plugins-core</module>
<module>tika-detectors</module>
+ <module>tika-ml</module>
+ <module>tika-encoding-detectors</module>
<module>tika-parsers</module>
<module>tika-bundles</module>
<module>tika-xmp</module>
diff --git a/tika-core/src/main/java/org/apache/tika/detect/AutoDetectReader.java b/tika-core/src/main/java/org/apache/tika/detect/AutoDetectReader.java
index 484bc62e60..f306f69548 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/AutoDetectReader.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/AutoDetectReader.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
+import java.util.List;
import org.xml.sax.InputSource;
@@ -96,10 +97,11 @@ public class AutoDetectReader extends BufferedReader {
EncodingDetector detector)
throws IOException, TikaException {
// Ask all given detectors for the character encoding
- Charset charset = detector.detect(tis, metadata, new ParseContext());
- if (charset != null) {
- return charset;
+ List<EncodingResult> results = detector.detect(tis, metadata, new ParseContext());
+ if (!results.isEmpty()) {
+ return results.get(0).getCharset();
}
+ Charset charset = null;
// Try determining the encoding based on hints in document metadata
MediaType type = MediaType.parse(metadata.get(Metadata.CONTENT_TYPE));
diff --git a/tika-core/src/main/java/org/apache/tika/detect/CompositeEncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/CompositeEncodingDetector.java
index 5fbf44dace..1b2db7acdb 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/CompositeEncodingDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/CompositeEncodingDetector.java
@@ -41,7 +41,7 @@ import org.apache.tika.parser.ParseContext;
* {@link EncodingDetectorContext}, then the meta detector runs last
* to arbitrate. Only one meta detector is supported.</p>
*
- * <p>If no meta detector is present, the first non-null result wins
+ * <p>If no meta detector is present, the first non-empty result wins
* (traditional behavior).</p>
*/
public class CompositeEncodingDetector implements EncodingDetector, Serializable {
@@ -74,9 +74,6 @@ public class CompositeEncodingDetector implements EncodingDetector, Serializable
this.metaDetector = partition(this.detectors, baseDetectors);
}
- /**
- * Partition detectors into base detectors and at most one meta detector.
- */
private static MetaEncodingDetector partition(
List<EncodingDetector> all, List<EncodingDetector> base) {
MetaEncodingDetector meta = null;
@@ -85,8 +82,7 @@ public class CompositeEncodingDetector implements EncodingDetector, Serializable
if (meta == null) {
meta = (MetaEncodingDetector) d;
} else {
- LOG.warn("Multiple MetaEncodingDetectors found; " +
- "ignoring {}",
+ LOG.warn("Multiple MetaEncodingDetectors found; ignoring {}",
d.getClass().getName());
}
} else {
@@ -97,8 +93,8 @@ public class CompositeEncodingDetector implements EncodingDetector, Serializable
}
@Override
- public Charset detect(TikaInputStream tis, Metadata metadata,
- ParseContext parseContext) throws IOException {
+ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext) throws IOException {
if (metaDetector != null) {
return detectWithMeta(tis, metadata, parseContext);
}
@@ -106,80 +102,94 @@ public class CompositeEncodingDetector implements EncodingDetector, Serializable
}
/**
- * Traditional first-match-wins behavior.
+ * Traditional first-match-wins behavior: returns the results from the
+ * first detector that produces a non-empty list.
*/
- private Charset detectFirstMatch(TikaInputStream tis, Metadata metadata,
- ParseContext parseContext)
+ private List<EncodingResult> detectFirstMatch(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext)
throws IOException {
for (EncodingDetector detector : getDetectors()) {
- Charset detected = detector.detect(tis, metadata, parseContext);
- if (detected != null) {
- metadata.set(TikaCoreProperties.DETECTED_ENCODING,
- detected.name());
+ List<EncodingResult> results = detector.detect(tis, metadata, parseContext);
+ if (!results.isEmpty()) {
+ Charset detected = results.get(0).getCharset();
+ metadata.set(TikaCoreProperties.DETECTED_ENCODING, detected.name());
if (!detector.getClass().getSimpleName()
.equals("CompositeEncodingDetector")) {
metadata.set(TikaCoreProperties.ENCODING_DETECTOR,
detector.getClass().getSimpleName());
}
- return detected;
+ return results;
}
}
- return null;
+ return Collections.emptyList();
}
/**
* Collect-all mode: run every base detector, populate context,
* then let the meta detector arbitrate.
+ *
+ * <p>The stream is marked once before the base-detector loop. After all
+ * base detectors have run (potentially consuming the stream), it is reset
+ * to that mark so the meta detector can re-read the raw bytes for
+ * language-based arbitration. Base detectors are not reset between
+ * themselves — each handles its own mark/reset internally if needed.</p>
*/
- private Charset detectWithMeta(TikaInputStream tis, Metadata metadata,
- ParseContext parseContext)
+ private List<EncodingResult> detectWithMeta(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext)
throws IOException {
EncodingDetectorContext context = new EncodingDetectorContext();
parseContext.set(EncodingDetectorContext.class, context);
+
+ // Mark so CharSoup can re-read the raw bytes after all base detectors finish.
+ if (tis != null) {
+ tis.mark(Integer.MAX_VALUE);
+ }
+
try {
for (EncodingDetector detector : baseDetectors) {
- Charset detected =
- detector.detect(tis, metadata, parseContext);
- if (detected != null) {
- context.addResult(detected,
- detector.getClass().getSimpleName());
+ List<EncodingResult> detected = detector.detect(tis, metadata, parseContext);
+ if (!detected.isEmpty()) {
+ context.addResult(detected, detector.getClass().getSimpleName());
}
}
- Charset result =
+ // Reset so the meta detector can re-read from the beginning.
+ if (tis != null) {
+ tis.reset();
+ }
+
+ List<EncodingResult> metaResults =
metaDetector.detect(tis, metadata, parseContext);
- // If meta detector returned null (disabled or no candidates),
- // fall back to first base detector's result
- if (result == null && !context.getResults().isEmpty()) {
- EncodingDetectorContext.Result first =
- context.getResults().get(0);
- result = first.getCharset();
- metadata.set(TikaCoreProperties.DETECTED_ENCODING,
- result.name());
- metadata.set(TikaCoreProperties.ENCODING_DETECTOR,
- first.getDetectorName());
- } else if (result != null) {
- metadata.set(TikaCoreProperties.DETECTED_ENCODING,
- result.name());
- String detectorName =
- metaDetector.getClass().getSimpleName();
- for (EncodingDetectorContext.Result r :
- context.getResults()) {
- if (r.getCharset().equals(result)) {
+ List<EncodingResult> finalResults;
+ String detectorName;
+
+ if (metaResults.isEmpty() && !context.getResults().isEmpty()) {
+ // Meta detector abstained — fall back to first base detector's results
+ EncodingDetectorContext.Result first = context.getResults().get(0);
+ finalResults = first.getEncodingResults();
+ detectorName = first.getDetectorName();
+ } else if (!metaResults.isEmpty()) {
+ finalResults = metaResults;
+ Charset winner = metaResults.get(0).getCharset();
+ // Credit the base detector that originally proposed the winning charset
+ detectorName = metaDetector.getClass().getSimpleName();
+ for (EncodingDetectorContext.Result r : context.getResults()) {
+ if (r.getCharset().equals(winner)) {
detectorName = r.getDetectorName();
break;
}
}
- metadata.set(TikaCoreProperties.ENCODING_DETECTOR,
- detectorName);
+ } else {
+ return Collections.emptyList();
}
- // Build and set the detection trace
- metadata.set(TikaCoreProperties.ENCODING_DETECTION_TRACE,
- buildTrace(context));
+ Charset detected = finalResults.get(0).getCharset();
+ metadata.set(TikaCoreProperties.DETECTED_ENCODING, detected.name());
+ metadata.set(TikaCoreProperties.ENCODING_DETECTOR, detectorName);
+ metadata.set(TikaCoreProperties.ENCODING_DETECTION_TRACE, buildTrace(context));
- return result;
+ return finalResults;
} finally {
parseContext.set(EncodingDetectorContext.class, null);
}
@@ -191,12 +201,14 @@ public class CompositeEncodingDetector implements EncodingDetector, Serializable
if (sb.length() > 0) {
sb.append(", ");
}
- sb.append(r.getDetectorName()).append("->")
- .append(r.getCharset().name());
+ sb.append(r.getDetectorName()).append("->").append(r.getCharset().name());
+ if (r.getConfidence() < EncodingResult.CONFIDENCE_DEFINITIVE) {
+ sb.append(String.format(java.util.Locale.ROOT, "(%.2f)", r.getConfidence()));
+ }
}
String info = context.getArbitrationInfo();
if (info != null) {
- sb.append(" (").append(info).append(")");
+ sb.append(" [").append(info).append("]");
}
return sb.toString();
}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/DefaultEncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/DefaultEncodingDetector.java
index 72dd3ba4c0..8f1cb67dce 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/DefaultEncodingDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/DefaultEncodingDetector.java
@@ -17,6 +17,8 @@
package org.apache.tika.detect;
import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
import javax.imageio.spi.ServiceRegistry;
import org.apache.tika.config.ServiceLoader;
@@ -26,13 +28,22 @@ import org.apache.tika.config.ServiceLoader;
* implementations available through the
* {@link ServiceRegistry service provider mechanism}.
*
- * <p>Those loaded via the service provider mechanism are ordered by how
- * they appear in the file, if there is a single service file. If
- * multiple, there is no guarantee of order.</p>
+ * <p>Loaded detectors are sorted in two tiers:
+ * <ol>
+ * <li>Base detectors (non-{@link MetaEncodingDetector}) sorted by full
+ * class name (non-Tika before Tika, then ascending alphabetically).
+ * The package ordering guarantees:
+ * {@code org.apache.tika.detect.WideUnicodeDetector} →
+ * {@code org.apache.tika.ml.*} (Mojibuster) →
+ * {@code org.apache.tika.parser.*} (HTML).</li>
+ * <li>{@link MetaEncodingDetector} instances always run last, after all
+ * base detectors have collected their candidates into
+ * {@link EncodingDetectorContext}.</li>
+ * </ol></p>
*
- * <p>If you need to control the order of the Detectors, you should
- * instead construct your own {@link CompositeEncodingDetector} and pass
- * in the list of Detectors in the required order.</p>
+ * <p>If you need to control the order of the Detectors explicitly, construct
+ * your own {@link CompositeEncodingDetector} and pass in the list in the
+ * required order.</p>
*
* <p>{@link MetaEncodingDetector} handling (collect-all-then-arbitrate)
* is provided by {@link CompositeEncodingDetector}.</p>
@@ -46,13 +57,23 @@ public class DefaultEncodingDetector extends CompositeEncodingDetector {
}
public DefaultEncodingDetector(ServiceLoader loader) {
- super(loader.loadServiceProviders(EncodingDetector.class));
+ super(sorted(loader.loadServiceProviders(EncodingDetector.class)));
}
public DefaultEncodingDetector(ServiceLoader loader,
Collection<Class<? extends EncodingDetector>>
excludeEncodingDetectors) {
- super(loader.loadServiceProviders(EncodingDetector.class),
+ super(sorted(loader.loadServiceProviders(EncodingDetector.class)),
excludeEncodingDetectors);
}
+
+ private static List<EncodingDetector> sorted(List<EncodingDetector> detectors) {
+ // Two-key sort: base detectors first (meta=0) then MetaEncodingDetectors (meta=1),
+ // within each tier sorted by full class name for stability across JARs.
+ detectors.sort(Comparator
+ .<EncodingDetector, Integer>comparing(
+ d -> (d instanceof MetaEncodingDetector) ? 1 : 0)
+ .thenComparing(d -> d.getClass().getName()));
+ return detectors;
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetector.java
index 9d4587907a..ff02a0bcb5 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetector.java
@@ -18,7 +18,7 @@ package org.apache.tika.detect;
import java.io.IOException;
import java.io.Serializable;
-import java.nio.charset.Charset;
+import java.util.List;
import org.apache.tika.config.SelfConfiguring;
import org.apache.tika.io.TikaInputStream;
@@ -26,37 +26,46 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
/**
- * Character encoding detector. Implementations of this interface use
- * various heuristics to detect the character encoding of a text document
- * based on given input metadata or the first few bytes of the document stream.
+ * Character encoding detector. Implementations use various heuristics to
+ * detect the character encoding of a text document based on given input
+ * metadata or the first few bytes of the document stream.
+ *
+ * <p>Detectors return a ranked list of {@link EncodingResult}s in descending
+ * confidence order. An empty list means no opinion. A single result with
+ * confidence {@link EncodingResult#CONFIDENCE_DEFINITIVE} (1.0) indicates a
+ * structural detection that requires no further arbitration. Multiple results
+ * or lower confidence values invite arbitration by a
+ * {@link MetaEncodingDetector}.</p>
*
* @since Apache Tika 0.4
*/
public interface EncodingDetector extends Serializable, SelfConfiguring {
/**
- * Detects the character encoding of the given text document, or
- * <code>null</code> if the encoding of the document can not be detected.
- * <p>
- * If the document input stream is not available, then the first
- * argument may be <code>null</code>. Otherwise the detector may
- * read bytes from the start of the stream to help in encoding detection.
+ * Detects the character encoding of the given text document.
+ *
+ * <p>Returns an empty list if the encoding cannot be determined.
+ * Returns a ranked list of candidates in descending confidence order
+ * otherwise. The first entry is always the best guess.</p>
+ *
+ * <p>If the document input stream is not available, then the first
+ * argument may be {@code null}. Otherwise the detector may read bytes
+ * from the start of the stream to help in encoding detection.
* The given stream is guaranteed to support the
* {@link TikaInputStream#markSupported() mark feature} and the detector
* is expected to {@link TikaInputStream#mark(int) mark} the stream before
* reading any bytes from it, and to {@link TikaInputStream#reset() reset}
* the stream before returning. The stream must not be closed by the
- * detector.
- * <p>
- * The given input metadata is only read, not modified, by the detector.
+ * detector.</p>
*
- * @param tis text document input stream, or <code>null</code>
- * @param metadata input metadata for the document
+ * <p>The given input metadata is only read, not modified, by the detector.</p>
+ *
+ * @param tis text document input stream, or {@code null}
+ * @param metadata input metadata for the document
* @param parseContext the parse context
- * @return detected character encoding, or <code>null</code>
+ * @return ranked list of encoding results, empty if unknown; never {@code null}
* @throws IOException if the document input stream could not be read
*/
- Charset detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext)
- throws IOException;
-
+ List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext) throws IOException;
}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java
index 6ac55f87da..18c8b189a9 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java
@@ -24,11 +24,16 @@ import java.util.List;
import java.util.Set;
/**
- * Context object that collects encoding detection results from base
- * detectors. Stored in {@link org.apache.tika.parser.ParseContext} by
- * {@link DefaultEncodingDetector} so that the {@link MetaEncodingDetector}
- * can see all candidates and arbitrate. Removed after detection to
- * prevent contamination during recursive parsing.
+ * Context object that collects encoding detection results from base detectors.
+ * Stored in {@link org.apache.tika.parser.ParseContext} by
+ * {@link CompositeEncodingDetector} so that a {@link MetaEncodingDetector}
+ * can see all candidates and arbitrate. Removed after detection to prevent
+ * contamination during recursive parsing.
+ *
+ * <p>Each base detector contributes a ranked {@link List} of
+ * {@link EncodingResult}s. The context exposes the top result from each
+ * detector as the primary signal, and provides access to all candidates
+ * for richer arbitration strategies.</p>
*
* @since Apache Tika 3.2
*/
@@ -36,38 +41,67 @@ public class EncodingDetectorContext {
private final List<Result> results = new ArrayList<>();
private String arbitrationInfo;
+ private WideUnicodeSignal wideUnicodeSignal;
/**
- * Record a detection result from a child detector.
+ * Record the ranked results from a child detector.
*
- * @param charset the detected charset (must not be null)
- * @param detectorName the simple class name of the detector
+ * @param encodingResults ranked results, highest confidence first; must not be empty
+ * @param detectorName simple class name of the detector
*/
- public void addResult(Charset charset, String detectorName) {
- results.add(new Result(charset, detectorName));
+ public void addResult(List<EncodingResult> encodingResults, String detectorName) {
+ if (encodingResults != null && !encodingResults.isEmpty()) {
+ results.add(new Result(encodingResults, detectorName));
+ }
}
/**
- * @return unmodifiable list of all results in detection order
+ * @return unmodifiable list of all per-detector results in detection order
*/
public List<Result> getResults() {
return Collections.unmodifiableList(results);
}
/**
- * @return unique charsets in detection order
+ * Returns the unique charsets from ALL results of every detector,
+ * in detection order (top result first within each detector).
+ *
+ * <p>Using all candidates rather than just each detector's top-1 is
+ * important when a single detector returns a ranked list (e.g., Mojibuster
+ * on a short probe returns [windows-1252, windows-1250, Shift-JIS]). If
+ * only the top-1 were used, CharSoup would see a single charset and
+ * return "unanimous" without ever attempting arbitration.</p>
*/
public Set<Charset> getUniqueCharsets() {
Set<Charset> charsets = new LinkedHashSet<>();
for (Result r : results) {
- charsets.add(r.getCharset());
+ for (EncodingResult er : r.getEncodingResults()) {
+ charsets.add(er.getCharset());
+ }
}
return charsets;
}
+ /**
+ * Returns the highest confidence seen for the given charset across all
+ * detector results (not just top results). Useful for arbitrators that
+ * want to propagate the base detector's confidence for the winning charset.
+ */
+ public float getTopConfidenceFor(Charset charset) {
+ float best = 0f;
+ for (Result r : results) {
+ for (EncodingResult er : r.getEncodingResults()) {
+ if (er.getCharset().equals(charset) && er.getConfidence() > best) {
+ best = er.getConfidence();
+ }
+ }
+ }
+ return best;
+ }
+
/**
* Set by the meta detector to describe how it reached its decision.
- * Values: "unanimous", "compatible", "scored", "too-few-bigrams", "disabled".
+ * Values: "unanimous", "scored", "no-stream", "empty-stream", etc.
*/
public void setArbitrationInfo(String info) {
this.arbitrationInfo = info;
@@ -78,19 +112,56 @@ public class EncodingDetectorContext {
}
/**
- * A single detection result pairing a charset with the detector that found it.
+ * Records the outcome of {@link WideUnicodeDetector} for this detection
+ * pass. Stored here rather than directly in the {@code ParseContext} so
+ * that it is scoped to this single detection and does not bleed into
+ * embedded-document parses.
+ */
+ public void setWideUnicodeSignal(WideUnicodeSignal signal) {
+ this.wideUnicodeSignal = signal;
+ }
+
+ /**
+ * Returns the {@link WideUnicodeSignal} written by {@link WideUnicodeDetector},
+ * or {@code null} if {@code WideUnicodeDetector} has not yet run or did not
+ * find wide-Unicode structure.
+ */
+ public WideUnicodeSignal getWideUnicodeSignal() {
+ return wideUnicodeSignal;
+ }
+
+ /**
+ * A single detector's contribution: its ranked list of candidates and its name.
*/
public static class Result {
- private final Charset charset;
+ private final List<EncodingResult> encodingResults;
private final String detectorName;
- public Result(Charset charset, String detectorName) {
- this.charset = charset;
+ public Result(List<EncodingResult> encodingResults, String detectorName) {
+ this.encodingResults = Collections.unmodifiableList(
+ new ArrayList<>(encodingResults));
this.detectorName = detectorName;
}
+ /**
+ * All ranked results from this detector, highest confidence first.
+ */
+ public List<EncodingResult> getEncodingResults() {
+ return encodingResults;
+ }
+
+ /**
+ * The top-ranked charset from this detector.
+ */
public Charset getCharset() {
- return charset;
+ return encodingResults.get(0).getCharset();
+ }
+
+ /**
+ * The confidence of the top-ranked result from this detector.
+ */
+ public float getConfidence() {
+ return encodingResults.get(0).getConfidence();
}
public String getDetectorName() {
@@ -99,7 +170,7 @@ public class EncodingDetectorContext {
@Override
public String toString() {
- return detectorName + "=" + charset.name();
+ return detectorName + "=" + encodingResults.get(0);
}
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/detect/OverrideEncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/OverrideEncodingDetector.java
index 778198e542..3c3ddfa627 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/OverrideEncodingDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/OverrideEncodingDetector.java
@@ -19,6 +19,7 @@ package org.apache.tika.detect;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import org.apache.tika.config.ConfigDeserializer;
import org.apache.tika.config.JsonConfig;
@@ -82,8 +83,9 @@ public class OverrideEncodingDetector implements EncodingDetector {
}
@Override
- public Charset detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException {
- return charset;
+ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext) throws IOException {
+ return List.of(new EncodingResult(charset, EncodingResult.CONFIDENCE_DEFINITIVE));
}
public Charset getCharset() {
diff --git a/tika-core/src/main/java/org/apache/tika/detect/WideUnicodeDetector.java b/tika-core/src/main/java/org/apache/tika/detect/WideUnicodeDetector.java
index 89f904d91a..b1ad9054da 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/WideUnicodeDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/WideUnicodeDetector.java
@@ -20,6 +20,8 @@ import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -46,12 +48,13 @@ import org.apache.tika.parser.ParseContext;
* (Arabic 0x06, Hebrew 0x05, Greek 0x03, Devanagari 0x09, …): the
* glyph-index column has at least 2× as many distinct values as the
* block-prefix column.</li>
- * <li><em>Block-prefix range</em> — CJK (0x4E–0x9F, 82 values) and
- * Hangul (0xAC–0xD7, 44 values) where the variety ratio alone may not
- * be decisive with limited samples: if all values in one column fall
- * below the surrogate boundary (0xD8) and the other column has more
- * distinct values, the constrained column is carrying Unicode
- * block-prefix bytes.</li>
+ * <li><em>Block-prefix range</em> — CJK Unified (0x4E–0x9F), Hangul
+ * (0xAC–0xD7), CJK Compatibility Ideographs (0xF9–0xFA), and Halfwidth
+ * and Fullwidth Forms (0xFF, e.g. Chinese fullwidth punctuation ，！？):
+ * where the variety ratio alone may not be decisive with limited samples.
+ * The constrained column must have no bytes in the surrogate/PUA zone
+ * (0xD8–0xF8); the glyph column must have at least one byte ≥ 0xD8;
+ * and non-null bytes in the constrained column must all be ≥ 0x20.</li>
* </ol>
* </p>
*
@@ -135,13 +138,30 @@ public class WideUnicodeDetector implements EncodingDetector {
private static final double UTF16_CONSTRAINED_MAX_RATIO = 0.40;
/**
- * Upper bound (exclusive) for the UTF-16 block-prefix range check.
- * Set to 0xD8 — the start of the UTF-16 surrogate range — which covers
- * every assigned BMP script: Latin, Greek, Cyrillic, Arabic, Hebrew,
- * Devanagari, CJK (0x4E–0x9F), Yi (0xA0–0xA4), Hangul (0xAC–0xD7).
+ * Boundary of the UTF-16 surrogate range. Bytes ≥ this value in the glyph
+ * (high-diversity) column confirm it is not the block-prefix column: the
+ * block-prefix column of any well-formed BMP document stays below 0xD8 for
+ * the most common scripts, and the glyph column overflows above it for CJK
+ * low bytes and other diverse content.
*/
private static final int BMP_BLOCK_PREFIX_MAX = 0xD8;
+ /**
+ * Lower bound of the re-allowed block-prefix bytes above the surrogate zone.
+ * Bytes in [{@code BMP_BLOCK_PREFIX_MAX}, {@code BLOCK_PREFIX_EXCLUSION_MAX})
+ * — i.e. 0xD8–0xF8 — are excluded from the block-prefix column:
+ * 0xD8–0xDF are UTF-16 surrogates (handled separately by the surrogate
+ * validator) and 0xE0–0xF8 are Private Use Area high bytes unlikely to
+ * appear as legitimate block prefixes in ordinary text.
+ * Bytes from {@code BLOCK_PREFIX_EXCLUSION_MAX} onward are re-allowed:
+ * <ul>
+ * <li>0xF9–0xFA: CJK Compatibility Ideographs (U+F900–U+FAFF)</li>
+ * <li>0xFF: Halfwidth and Fullwidth Forms (U+FF00–U+FFEF) — extremely
+ * common in Chinese text as fullwidth punctuation (，！？：；（）…)</li>
+ * </ul>
+ */
+ private static final int BLOCK_PREFIX_EXCLUSION_MAX = 0xF9;
+
/**
* Known BOM sequences, longest first so that the 4-byte UTF-32 BOMs are
* matched before the 2-byte UTF-16 BOMs that share their prefix.
@@ -155,10 +175,10 @@ public class WideUnicodeDetector implements EncodingDetector {
};
@Override
- public Charset detect(TikaInputStream tis, Metadata metadata,
- ParseContext context) throws IOException {
+ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext) throws IOException {
if (tis == null || !tis.markSupported()) {
- return null;
+ return Collections.emptyList();
}
tis.mark(STREAM_READ_LIMIT);
@@ -171,39 +191,75 @@ public class WideUnicodeDetector implements EncodingDetector {
// 12 = longest BOM (4 bytes) + minimum analysable content (8 bytes)
if (buf.length < 12) {
- return null;
+ return Collections.emptyList();
}
- return detectEncoding(buf);
+ WideUnicodeSignal signal = detectSignal(buf);
+ if (signal == null) {
+ return Collections.emptyList();
+ }
+
+ // Write the signal into the per-detection context so downstream detectors
+ // (e.g. ML models not trained on null-heavy UTF-16/32 data) can defer
+ // without re-examining the bytes. We use EncodingDetectorContext rather
+ // than ParseContext directly so the signal does not outlive this single
+ // detection pass and does not bleed into embedded-document parses.
+ EncodingDetectorContext encodingContext =
+ parseContext.get(EncodingDetectorContext.class);
+ if (encodingContext != null) {
+ encodingContext.setWideUnicodeSignal(signal);
+ }
+
+ if (signal.getKind() == WideUnicodeSignal.Kind.ILLEGAL_SURROGATES) {
+ return Collections.emptyList();
+ }
+ // Structural heuristic, not a BOM read — use high but not absolute confidence
+ // so that language-scoring arbitration can override if the decoded text is garbage.
+ return List.of(new EncodingResult(signal.getCharset(), 0.95f));
}
/**
- * Detect the wide Unicode encoding of a raw byte array.
- * Any leading BOM is stripped before analysis so that the fixed-width
- * group alignment is preserved. Callers do not need to strip the BOM
- * themselves.
- *
- * @param bytes raw content bytes, with or without a leading BOM
- * @return detected charset, or {@code null} if no wide Unicode structure
- * is found
+ * Full detection returning a {@link WideUnicodeSignal}, distinguishing
+ * "valid detection", "illegal surrogates", and "not wide Unicode" (null).
*/
- public static Charset detectEncoding(byte[] bytes) {
+ private static WideUnicodeSignal detectSignal(byte[] bytes) {
if (bytes == null || bytes.length < 8) {
return null;
}
-
bytes = skipBom(bytes);
-
int sampleLen = (Math.min(bytes.length, SAMPLE_LIMIT) / 4) * 4;
if (sampleLen < 8) {
return null;
}
-
Charset utf32 = tryUtf32(bytes, sampleLen);
if (utf32 != null) {
- return utf32;
+ return WideUnicodeSignal.valid(utf32);
}
- return tryUtf16(bytes, sampleLen);
+ return tryUtf16Signal(bytes, sampleLen);
+ }
+
+ /**
+ * Detect the wide Unicode encoding of a raw byte array.
+ * Any leading BOM is stripped before analysis so that the fixed-width
+ * group alignment is preserved. Callers do not need to strip the BOM
+ * themselves.
+ *
+ * <p>Returns {@code null} for both "not wide Unicode" and
+ * "illegal surrogates". Use
+ * {@link #detect(TikaInputStream, Metadata, ParseContext)} with a live
+ * {@link ParseContext} containing an {@link EncodingDetectorContext} if
+ * you need to distinguish those two cases via {@link WideUnicodeSignal}.</p>
+ *
+ * @param bytes raw content bytes, with or without a leading BOM
+ * @return detected charset, or {@code null} if no wide Unicode structure
+ * is found or if surrogates are illegal
+ */
+ public static Charset detectEncoding(byte[] bytes) {
+ WideUnicodeSignal signal = detectSignal(bytes);
+ if (signal == null || signal.isIllegalSurrogates()) {
+ return null;
+ }
+ return signal.getCharset();
}
/**
@@ -288,16 +344,18 @@ public class WideUnicodeDetector implements EncodingDetector {
}
/**
- * UTF-16 detection via three phases, with surrogate-pair sequence validation
- * running in parallel to rule out structurally impossible byte sequences.
+ * UTF-16 detection via three phases, returning a {@link WideUnicodeSignal}
+ * that distinguishes a valid detection, illegal surrogates, and no match.
*
* <p>Surrogate validation: in UTF-16BE the even byte is the high byte of
* each code unit; in UTF-16LE the odd byte is the high byte. A high surrogate
* (0xD8–0xDB) must be immediately followed by a low surrogate (0xDC–0xDF).
* A lone low surrogate, or a high surrogate not followed by a low surrogate,
- * marks that byte order as invalid.</p>
+ * marks that byte order as invalid. When a structural phase would have fired
+ * but validity failed, {@link WideUnicodeSignal#illegalSurrogates()} is
+ * returned so downstream detectors know not to attempt decoding.</p>
*/
- private static Charset tryUtf16(byte[] bytes, int sampleLen) {
+ private static WideUnicodeSignal tryUtf16Signal(byte[] bytes, int sampleLen) {
int pairs = sampleLen / 2;
int nullsAtEven = 0;
int nullsAtOdd = 0;
@@ -361,8 +419,14 @@ public class WideUnicodeDetector implements EncodingDetector {
// Phase 1: null-column (Latin/ASCII)
boolean highEven = nullsAtEven * NULL_THRESHOLD_DENOM > pairs;
boolean highOdd = nullsAtOdd * NULL_THRESHOLD_DENOM > pairs;
- if (highOdd && !highEven && !invalidLe) return StandardCharsets.UTF_16LE;
- if (highEven && !highOdd && !invalidBe) return StandardCharsets.UTF_16BE;
+ if (highOdd && !highEven) {
+ return invalidLe ? WideUnicodeSignal.illegalSurrogates()
+ : WideUnicodeSignal.valid(StandardCharsets.UTF_16LE);
+ }
+ if (highEven && !highOdd) {
+ return invalidBe ? WideUnicodeSignal.illegalSurrogates()
+ : WideUnicodeSignal.valid(StandardCharsets.UTF_16BE);
+ }
// Phase 2: variety-ratio (Arabic, Hebrew, Greek, Devanagari, …)
//
@@ -377,54 +441,60 @@ public class WideUnicodeDetector implements EncodingDetector {
int uniqueOdd = countUnique(countsOdd);
double constrainedMax = pairs * UTF16_CONSTRAINED_MAX_RATIO;
- if (!invalidLe && (double) uniqueEven / uniqueOdd >= UTF16_VARIETY_RATIO
+ if ((double) uniqueEven / uniqueOdd >= UTF16_VARIETY_RATIO
&& uniqueOdd <= constrainedMax
&& mostCommon(countsOdd) != 0) {
- return StandardCharsets.UTF_16LE;
+ return invalidLe ? WideUnicodeSignal.illegalSurrogates()
+ : WideUnicodeSignal.valid(StandardCharsets.UTF_16LE);
}
- if (!invalidBe && (double) uniqueOdd / uniqueEven >= UTF16_VARIETY_RATIO
+ if ((double) uniqueOdd / uniqueEven >= UTF16_VARIETY_RATIO
&& uniqueEven <= constrainedMax
&& mostCommon(countsEven) != 0) {
- return StandardCharsets.UTF_16BE;
+ return invalidBe ? WideUnicodeSignal.illegalSurrogates()
+ : WideUnicodeSignal.valid(StandardCharsets.UTF_16BE);
}
- // Phase 3: block-prefix range (CJK, Hangul — wide block-prefix ranges)
+ // Phase 3: block-prefix range (CJK, Hangul, Fullwidth, CJK Compat)
+ //
+ // Two separate range checks are used — one for each role:
+ //
+ // isBlockPrefixColumn(col) — RELAXED — identifies the block-prefix
+ // (constrained) column. Requires no bytes in the
+ // surrogate/PUA exclusion zone [0xD8, 0xF9). Allows
+ // 0xF9–0xFF so that CJK Compatibility Ideographs and
+ // Fullwidth Forms (0xFF) are accepted. Chinese text
+ // routinely mixes fullwidth punctuation (U+FF0C ，,
+ // U+FF01 ！, …) with CJK Unified Ideographs; the 0xFF
+ // high byte would break the old strict "all < 0xD8"
+ // check even though the stream is perfectly valid UTF-16.
//
- // We require:
- // oddInRange — all odd-position bytes < 0xD8 (below the surrogate
- // boundary). This is the high-byte (block-prefix)
- // column for UTF-16LE.
- // !evenInRange — at least one even-position byte ≥ 0xD8 (the glyph-
- // index column overflows the surrogate boundary, which
- // is expected for CJK low bytes like 0xE5, 0xEF, 0xF4).
- // nonNullAllAbove — no non-null byte below 0x20 in the constrained
+ // !allInRange(col, 0xD8) — STRICT — identifies the glyph-index
+ // (high-diversity) column. At least one byte ≥ 0xD8
+ // must be present; this is the natural overflow from
+ // CJK low bytes (0xD8–0xFF for characters like 以=0xE5,
+ // 们=0xEC, 说=0xF4) and provides orientation so we can
+ // tell which column is which.
+ //
+ // nonNullAllAbove — no non-null byte below 0x20 in the block-prefix
// column. Binary formats inject control tokens here
// (ISOBMFF 0x01/0x02, LZ4 0x04/0x15); CJK and Hangul
- // block-prefix bytes are always ≥ 0x4E/0xAC.
+ // block-prefix bytes are always ≥ 0x4E/0xAC, and
+ // Fullwidth Forms are 0xFF.
//
- // We also require uniqueEven > uniqueOdd for LE (uniqueOdd > uniqueEven
- // for BE). This is not about discriminating from random binary — the
- // scenario where "12 vs 11 unique values, all below 0xD8" could cause
- // a false positive is already prevented by !evenInRange (which requires
- // at least one byte ≥ 0xD8 in the even column, contradicting "all
- // below 0xD8"). The real purpose is orientation: it prevents Latin
- // UTF-16BE data (whose odd column is full of diverse ASCII content
- // bytes and whose even column is dominated by 0x00) from being
- // misidentified as UTF-16LE when a stray byte ≥ 0xD8 lands in the
- // even column (e.g. an injected or corrupted surrogate byte). In that
- // pathological case uniqueEven ≈ 2 << uniqueOdd, so the check fails
- // cleanly. For legitimate CJK and Hangul the glyph-index column
- // always has at least as many distinct values as the block-prefix
- // column, so a gap of 1 is enough.
- boolean oddInRange = allInRange(countsOdd, BMP_BLOCK_PREFIX_MAX);
- boolean evenInRange = allInRange(countsEven, BMP_BLOCK_PREFIX_MAX);
- if (!invalidLe && oddInRange && !evenInRange && uniqueEven > uniqueOdd
- && nonNullAllAbove(countsOdd, 0x20)) {
- return StandardCharsets.UTF_16LE;
+ // uniqueGlyph > uniquePrefix — orientation guard (see Phase 2 note).
+ boolean oddInRange = allInRange(countsOdd, BMP_BLOCK_PREFIX_MAX);
+ boolean evenInRange = allInRange(countsEven, BMP_BLOCK_PREFIX_MAX);
+ boolean oddIsPrefix = isBlockPrefixColumn(countsOdd);
+ boolean evenIsPrefix = isBlockPrefixColumn(countsEven);
+ if (oddIsPrefix && !evenInRange && uniqueEven > uniqueOdd
+ && nonNullAllAbove(countsOdd, 0x20)) {
+ return invalidLe ? WideUnicodeSignal.illegalSurrogates()
+ : WideUnicodeSignal.valid(StandardCharsets.UTF_16LE);
}
- if (!invalidBe && evenInRange && !oddInRange && uniqueOdd > uniqueEven
+ if (evenIsPrefix && !oddInRange && uniqueOdd > uniqueEven
&& nonNullAllAbove(countsEven, 0x20)) {
- return StandardCharsets.UTF_16BE;
+ return invalidBe ? WideUnicodeSignal.illegalSurrogates()
+ : WideUnicodeSignal.valid(StandardCharsets.UTF_16BE);
}
return null;
@@ -443,6 +513,26 @@ public class WideUnicodeDetector implements EncodingDetector {
return true;
}
+ /**
+ * Returns {@code true} if {@code counts} is consistent with a UTF-16
+ * block-prefix column: no bytes in the surrogate/PUA exclusion zone
+ * [{@link #BMP_BLOCK_PREFIX_MAX}, {@link #BLOCK_PREFIX_EXCLUSION_MAX})
+ * (i.e. 0xD8–0xF8).
+ *
+ * <p>This is a relaxed alternative to {@link #allInRange(int[], int)}
+ * for Phase 3's constrained-column check. It accepts high block-prefix
+ * bytes used in CJK-family documents — 0xF9–0xFA (CJK Compatibility
+ * Ideographs) and 0xFF (Halfwidth and Fullwidth Forms) — while still
+ * rejecting surrogates (0xD8–0xDF) and Private Use Area bytes (0xE0–0xF8)
+ * that should not appear as standalone block prefixes.</p>
+ */
+ private static boolean isBlockPrefixColumn(int[] counts) {
+ for (int v = BMP_BLOCK_PREFIX_MAX; v < BLOCK_PREFIX_EXCLUSION_MAX; v++) {
+ if (counts[v] > 0) return false;
+ }
+ return true;
+ }
+
/**
* Returns the byte value (0–255) that appears most frequently in
* {@code counts}. Ties are broken in favour of the lower value.
diff --git a/tika-core/src/test/java/org/apache/tika/detect/WideUnicodeDetectorTest.java b/tika-core/src/test/java/org/apache/tika/detect/WideUnicodeDetectorTest.java
index 66d8c65271..d637ddd3d2 100644
--- a/tika-core/src/test/java/org/apache/tika/detect/WideUnicodeDetectorTest.java
+++ b/tika-core/src/test/java/org/apache/tika/detect/WideUnicodeDetectorTest.java
@@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import java.io.ByteArrayInputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import org.junit.jupiter.api.Test;
@@ -60,7 +61,8 @@ public class WideUnicodeDetectorTest {
private Charset detectViaStream(byte[] bytes) throws Exception {
try (TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes))) {
- return detector.detect(tis, new Metadata(), new ParseContext());
+ List<EncodingResult> results = detector.detect(tis, new Metadata(), new ParseContext());
+ return results.isEmpty() ? null : results.get(0).getCharset();
}
}
@@ -170,6 +172,26 @@ public class WideUnicodeDetectorTest {
WideUnicodeDetector.detectEncoding(encode(mixed, StandardCharsets.UTF_16LE)));
}
+ @Test
+ void utf16LeCjkWithFullwidthPunctuation() {
+ // Fullwidth punctuation (U+FF0C ，, U+FF01 ！) has high byte 0xFF in UTF-16,
+ // above the old BMP_BLOCK_PREFIX_MAX threshold (0xD8). Phase 3 must use the
+ // extended block-prefix range check (isBlockPrefixColumn) to detect this.
+ String text = ("\u4e2d\u6587\uff0c\u6d4b\u8bd5\uff01\u8bf7\u95ee\uff1f").repeat(12);
+ assertEquals(StandardCharsets.UTF_16LE,
+ WideUnicodeDetector.detectEncoding(encode(text, StandardCharsets.UTF_16LE)));
+ }
+
+ @Test
+ void utf16LeCjkCompatIdeographs() {
+ // CJK Compatibility Ideographs U+F900-U+FAFF have high byte 0xF9 or 0xFA,
+ // also above 0xD8. The isBlockPrefixColumn check allows 0xF9+ while
+ // blocking surrogates (0xD8-0xDF) and PUA (0xE0-0xF8).
+ String text = ("\uF900\uF901\uF902\uF903\uF904\uF905\uF906\uF907").repeat(20);
+ assertEquals(StandardCharsets.UTF_16LE,
+ WideUnicodeDetector.detectEncoding(encode(text, StandardCharsets.UTF_16LE)));
+ }
+
// ── UTF-16 BE ────────────────────────────────────────────────────────────
@Test
@@ -221,6 +243,20 @@ public class WideUnicodeDetectorTest {
WideUnicodeDetector.detectEncoding(encode(text, StandardCharsets.UTF_16BE)));
}
+ @Test
+ void utf16BeCjkWithFullwidthPunctuation() {
+ String text = ("\u4e2d\u6587\uff0c\u6d4b\u8bd5\uff01\u8bf7\u95ee\uff1f").repeat(12);
+ assertEquals(StandardCharsets.UTF_16BE,
+ WideUnicodeDetector.detectEncoding(encode(text, StandardCharsets.UTF_16BE)));
+ }
+
+ @Test
+ void utf16BeCjkCompatIdeographs() {
+ String text = ("\uF900\uF901\uF902\uF903\uF904\uF905\uF906\uF907").repeat(20);
+ assertEquals(StandardCharsets.UTF_16BE,
+ WideUnicodeDetector.detectEncoding(encode(text, StandardCharsets.UTF_16BE)));
+ }
+
// ── UTF-32 LE ────────────────────────────────────────────────────────────
@Test
diff --git a/tika-langdetect/tika-langdetect-charsoup/pom.xml b/tika-langdetect/tika-langdetect-charsoup/pom.xml
index bfcee9ac2b..2c4ac553f3 100644
--- a/tika-langdetect/tika-langdetect-charsoup/pom.xml
+++ b/tika-langdetect/tika-langdetect-charsoup/pom.xml
@@ -54,6 +54,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-ml-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java b/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java
deleted file mode 100644
index 75176f69fc..0000000000
--- a/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.langdetect.charsoup;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CodingErrorAction;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.tika.config.TikaComponent;
-import org.apache.tika.detect.EncodingDetectorContext;
-import org.apache.tika.detect.MetaEncodingDetector;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-
-/**
- * A {@link MetaEncodingDetector} that uses the CharSoup language detector
- * to arbitrate when base encoding detectors disagree.
- *
- * <p>When base detectors all agree, the unanimous charset is returned
- * without any language detection. When they disagree, raw bytes are
- * read from the stream, decoded with each candidate charset, and each
- * decoded text is scored by {@link CharSoupLanguageDetector}. The
- * charset that produces the highest-confidence language detection wins.</p>
- *
- * <p>To enable, add this detector to your encoding detector chain in
- * tika-config:</p>
- * <pre>{@code
- * "encoding-detectors": [
- * { "default-encoding-detector": {} },
- * { "charsoup-encoding-detector": {} }
- * ]
- * }</pre>
- *
- * @since Apache Tika 3.2
- */
-@TikaComponent(name = "charsoup-encoding-detector")
-public class CharSoupEncodingDetector implements MetaEncodingDetector {
-
- private static final long serialVersionUID = 1L;
-
- private static final int DEFAULT_READ_LIMIT = 16384;
-
- private int readLimit = DEFAULT_READ_LIMIT;
-
- @Override
- public Charset detect(TikaInputStream tis, Metadata metadata,
- ParseContext parseContext) throws IOException {
- EncodingDetectorContext context =
- parseContext.get(EncodingDetectorContext.class);
- if (context == null || context.getResults().isEmpty()) {
- return null;
- }
-
- Set<Charset> uniqueCharsets = context.getUniqueCharsets();
-
- if (uniqueCharsets.size() <= 1) {
- // Unanimous or single detector — no arbitration needed
- EncodingDetectorContext.Result first = context.getResults().get(0);
- context.setArbitrationInfo("unanimous");
- return first.getCharset();
- }
-
- // Disagreement — arbitrate via language detection scoring
- return arbitrate(tis, context, uniqueCharsets);
- }
-
- private Charset arbitrate(TikaInputStream tis,
- EncodingDetectorContext context,
- Set<Charset> uniqueCharsets) throws IOException {
- EncodingDetectorContext.Result firstResult = context.getResults().get(0);
-
- if (tis == null) {
- context.setArbitrationInfo("no-stream");
- return firstResult.getCharset();
- }
-
- byte[] bytes = readBytes(tis);
- if (bytes == null || bytes.length == 0) {
- context.setArbitrationInfo("empty-stream");
- return firstResult.getCharset();
- }
-
- Map<Charset, String> candidates = new LinkedHashMap<>();
- for (Charset candidate : uniqueCharsets) {
- candidates.put(candidate, stripTags(decode(bytes, candidate)));
- }
-
- CharSoupLanguageDetector langDetector = new CharSoupLanguageDetector();
- Charset bestCharset = langDetector.compareLanguageSignal(candidates);
- if (bestCharset == null) {
- bestCharset = firstResult.getCharset();
- }
-
- context.setArbitrationInfo("scored");
- return bestCharset;
- }
-
- private byte[] readBytes(TikaInputStream tis) throws IOException {
- try {
- tis.mark(readLimit);
- byte[] buf = new byte[readLimit];
- int totalRead = 0;
- int bytesRead;
- while (totalRead < readLimit &&
- (bytesRead = tis.read(buf, totalRead,
- readLimit - totalRead)) != -1) {
- totalRead += bytesRead;
- }
- if (totalRead == 0) {
- return null;
- }
- if (totalRead < readLimit) {
- byte[] trimmed = new byte[totalRead];
- System.arraycopy(buf, 0, trimmed, 0, totalRead);
- return trimmed;
- }
- return buf;
- } finally {
- tis.reset();
- }
- }
-
- /**
- * Decode bytes using the given charset, replacing malformed/unmappable
- * characters rather than throwing.
- */
- static String decode(byte[] bytes, Charset charset) {
- CharsetDecoder decoder = charset.newDecoder()
- .onMalformedInput(CodingErrorAction.REPLACE)
- .onUnmappableCharacter(CodingErrorAction.REPLACE);
- CharBuffer cb = CharBuffer.allocate(bytes.length * 2);
- decoder.decode(ByteBuffer.wrap(bytes), cb, true);
- decoder.flush(cb);
- cb.flip();
- return cb.toString();
- }
-
- /**
- * Simple tag stripping: removes <...> sequences so that
- * HTML/XML tag names and attributes don't pollute language scoring.
- */
- static String stripTags(String text) {
- StringBuilder sb = new StringBuilder(text.length());
- boolean inTag = false;
- for (int i = 0; i < text.length(); i++) {
- char c = text.charAt(i);
- if (c == '<') {
- inTag = true;
- } else if (c == '>') {
- inTag = false;
- } else if (!inTag) {
- sb.append(c);
- }
- }
- return sb.toString();
- }
-
- public int getReadLimit() {
- return readLimit;
- }
-
- public void setReadLimit(int readLimit) {
- this.readLimit = readLimit;
- }
-}
diff --git a/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupLanguageDetector.java b/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupLanguageDetector.java
index abd0f8e0d6..75fccbb806 100644
--- a/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupLanguageDetector.java
+++ b/tika-langdetect/tika-langdetect-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupLanguageDetector.java
@@ -357,6 +357,28 @@ public class CharSoupLanguageDetector extends LanguageDetector {
return bestKey;
}
+ /**
+ * Returns diagnostic language-signal info for the given text: the label
+ * with the highest logit, its raw logit value, and sigmoid(maxLogit).
+ * Package-private for testing.
+ */
+ static float[] maxLogitInfo(String text) {
+ int[] features = EXTRACTOR.extract(text);
+ float[] logits = MODEL.predictLogits(features);
+ int bestIdx = 0;
+ for (int i = 1; i < logits.length; i++) {
+ if (logits[i] > logits[bestIdx]) {
+ bestIdx = i;
+ }
+ }
+ return new float[]{bestIdx, logits[bestIdx], sigmoid(logits[bestIdx])};
+ }
+
+ /** Returns the label for a class index (for use alongside {@link #maxLogitInfo}). */
+ static String labelAt(int idx) {
+ return MODEL.getLabel(idx);
+ }
+
/**
* Ratio of junk characters (U+FFFD replacement + ISO control + C1
* control range U+0080-U+009F) to total characters. High values
diff --git a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetectorTest.java b/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetectorTest.java
deleted file mode 100644
index 5ff028d442..0000000000
--- a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetectorTest.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.langdetect.charsoup;
-
-import static java.nio.charset.StandardCharsets.ISO_8859_1;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.ByteArrayInputStream;
-import java.nio.charset.Charset;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.detect.EncodingDetectorContext;
-import org.apache.tika.detect.MetaEncodingDetector;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-
-public class CharSoupEncodingDetectorTest {
-
- @Test
- public void testIsMetaEncodingDetector() {
- assertTrue(new CharSoupEncodingDetector() instanceof MetaEncodingDetector);
- }
-
- @Test
- public void testUnanimous() throws Exception {
- CharSoupEncodingDetector detector = new CharSoupEncodingDetector();
- EncodingDetectorContext context = new EncodingDetectorContext();
- context.addResult(UTF_8, "DetectorA");
- context.addResult(UTF_8, "DetectorB");
-
- ParseContext parseContext = new ParseContext();
- parseContext.set(EncodingDetectorContext.class, context);
-
- byte[] data = "Hello, world!".getBytes(UTF_8);
- try (TikaInputStream tis = TikaInputStream.get(
- new ByteArrayInputStream(data))) {
- Charset result = detector.detect(tis, new Metadata(), parseContext);
- assertEquals(UTF_8, result);
- assertEquals("unanimous", context.getArbitrationInfo());
- }
- }
-
- @Test
- public void testNoContext() throws Exception {
- CharSoupEncodingDetector detector = new CharSoupEncodingDetector();
- ParseContext parseContext = new ParseContext();
-
- byte[] data = "Test".getBytes(UTF_8);
- try (TikaInputStream tis = TikaInputStream.get(
- new ByteArrayInputStream(data))) {
- Charset result = detector.detect(tis, new Metadata(), parseContext);
- assertNull(result);
- }
- }
-
- @Test
- public void testEmptyResults() throws Exception {
- CharSoupEncodingDetector detector = new CharSoupEncodingDetector();
- EncodingDetectorContext context = new EncodingDetectorContext();
-
- ParseContext parseContext = new ParseContext();
- parseContext.set(EncodingDetectorContext.class, context);
-
- byte[] data = "Test".getBytes(UTF_8);
- try (TikaInputStream tis = TikaInputStream.get(
- new ByteArrayInputStream(data))) {
- Charset result = detector.detect(tis, new Metadata(), parseContext);
- assertNull(result);
- }
- }
-
- @Test
- public void testArabicEncodingArbitration() throws Exception {
- // Arabic text encoded in windows-1256.
- // When decoded as UTF-8 it produces replacement chars / garbage.
- // When decoded as windows-1256 it produces valid Arabic.
- // The language detector should pick windows-1256.
- Charset windows1256 = Charset.forName("windows-1256");
-
- String arabicText =
- "\u0641\u064a \u0642\u0631\u064a\u0629 \u0645\u0646 " +
- "\u0627\u0644\u0642\u0631\u0649 \u0643\u0627\u0646 " +
- "\u0647\u0646\u0627\u0643 \u0631\u062c\u0644 " +
- "\u062d\u0643\u064a\u0645 \u064a\u0639\u0631\u0641 " +
- "\u0643\u0644 \u0634\u064a\u0621 \u0639\u0646 " +
- "\u0627\u0644\u062d\u064a\u0627\u0629 \u0648\u0643\u0627\u0646 " +
- "\u064a\u0639\u0644\u0645 \u0627\u0644\u0646\u0627\u0633 " +
- "\u0643\u064a\u0641 \u064a\u0639\u064a\u0634\u0648\u0646 " +
- "\u0628\u0633\u0644\u0627\u0645 \u0648\u0627\u0646\u0633\u062c\u0627\u0645. " +
- "\u0627\u0644\u0644\u063a\u0629 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 " +
- "\u0647\u064a \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 " +
- "\u0623\u0643\u062b\u0631 \u0627\u0644\u0644\u063a\u0627\u062a " +
- "\u0627\u0646\u062a\u0634\u0627\u0631\u0627 \u0641\u064a " +
- "\u0627\u0644\u0639\u0627\u0644\u0645 \u0648\u064a\u062a\u062d\u062b\u0647\u0627 " +
- "\u0623\u0643\u062b\u0631 \u0645\u0646 \u062b\u0644\u0627\u062b\u0645\u0627\u0626\u0629 " +
- "\u0645\u0644\u064a\u0648\u0646 \u0625\u0646\u0633\u0627\u0646.";
- byte[] arabicBytes = arabicText.getBytes(windows1256);
-
- EncodingDetectorContext context = new EncodingDetectorContext();
- context.addResult(UTF_8, "HtmlEncodingDetector");
- context.addResult(windows1256, "Icu4jEncodingDetector");
-
- ParseContext parseContext = new ParseContext();
- parseContext.set(EncodingDetectorContext.class, context);
-
- CharSoupEncodingDetector detector = new CharSoupEncodingDetector();
- try (TikaInputStream tis = TikaInputStream.get(
- new ByteArrayInputStream(arabicBytes))) {
- Charset result = detector.detect(tis, new Metadata(), parseContext);
- assertEquals(windows1256, result);
- assertEquals("scored", context.getArbitrationInfo());
- }
- }
-
- @Test
- public void testStreamResetAfterDetection() throws Exception {
- EncodingDetectorContext context = new EncodingDetectorContext();
- context.addResult(UTF_8, "DetectorA");
- context.addResult(ISO_8859_1, "DetectorB");
-
- ParseContext parseContext = new ParseContext();
- parseContext.set(EncodingDetectorContext.class, context);
-
- byte[] data = "Hello, world! This is a test of encoding detection.".getBytes(UTF_8);
- CharSoupEncodingDetector detector = new CharSoupEncodingDetector();
- try (TikaInputStream tis = TikaInputStream.get(
- new ByteArrayInputStream(data))) {
- detector.detect(tis, new Metadata(), parseContext);
-
- // Verify stream is back at the start
- byte[] readBack = new byte[data.length];
- int bytesRead = tis.read(readBack);
- assertEquals(data.length, bytesRead);
- assertEquals("Hello, world! This is a test of encoding detection.",
- new String(readBack, UTF_8));
- }
- }
-
- @Test
- public void testStripTags() {
- assertEquals("Hello world",
- CharSoupEncodingDetector.stripTags(
- "<html><body>Hello world</body></html>"));
- assertEquals("no tags here",
- CharSoupEncodingDetector.stripTags("no tags here"));
- assertEquals("",
- CharSoupEncodingDetector.stripTags("<empty/>"));
- }
-
- @Test
- public void testDecode() {
- byte[] utf8Bytes = "caf\u00e9".getBytes(UTF_8);
- assertEquals("caf\u00e9",
- CharSoupEncodingDetector.decode(utf8Bytes, UTF_8));
- }
-
- @Test
- public void testReadLimitGetterSetter() {
- CharSoupEncodingDetector detector = new CharSoupEncodingDetector();
- assertEquals(16384, detector.getReadLimit());
- detector.setReadLimit(4096);
- assertEquals(4096, detector.getReadLimit());
- }
-
- @Test
- public void testJunkRatio() {
- // Clean text — no junk
- assertEquals(0f,
- CharSoupLanguageDetector.junkRatio("Hello, world!"), 0.001f);
-
- // U+FFFD replacement chars
- assertEquals(0.5f,
- CharSoupLanguageDetector.junkRatio("ab\uFFFD\uFFFD"), 0.001f);
-
- // C1 control chars (U+0080-U+009F are isISOControl)
- assertEquals(0.25f,
- CharSoupLanguageDetector.junkRatio("abc\u0080"), 0.001f);
-
- // Mixed: \r\n are control chars too
- assertEquals(2f / 13f,
- CharSoupLanguageDetector.junkRatio("hello world\r\n"), 0.001f);
-
- // Empty/null
- assertEquals(0f, CharSoupLanguageDetector.junkRatio(""), 0.001f);
- assertEquals(0f, CharSoupLanguageDetector.junkRatio(null), 0.001f);
- }
-}
diff --git a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/TextQualityDiagTest.java b/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/TextQualityDiagTest.java
deleted file mode 100644
index 7b56089c26..0000000000
--- a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/TextQualityDiagTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.langdetect.charsoup;
-
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.Locale;
-
-import org.junit.jupiter.api.Test;
-
-/**
- * Diagnostic test to explore text quality scores for encoding arbitration.
- * Not a regression test — just prints scores for analysis.
- */
-public class TextQualityDiagTest {
-
- @Test
- public void dumpScores() {
- // Arabic text in windows-1256
- Charset windows1256 = Charset.forName("windows-1256");
- String arabicText =
- "\u0641\u064a \u0642\u0631\u064a\u0629 \u0645\u0646 " +
- "\u0627\u0644\u0642\u0631\u0649 \u0643\u0627\u0646 " +
- "\u0647\u0646\u0627\u0643 \u0631\u062c\u0644 " +
- "\u062d\u0643\u064a\u0645 \u064a\u0639\u0631\u0641 " +
- "\u0643\u0644 \u0634\u064a\u0621 \u0639\u0646 " +
- "\u0627\u0644\u062d\u064a\u0627\u0629 \u0648\u0643\u0627\u0646 " +
- "\u064a\u0639\u0644\u0645 \u0627\u0644\u0646\u0627\u0633 " +
- "\u0643\u064a\u0641 \u064a\u0639\u064a\u0634\u0648\u0646 " +
- "\u0628\u0633\u0644\u0627\u0645 \u0648\u0627\u0646\u0633\u062c\u0627\u0645.";
- byte[] arabicBytes = arabicText.getBytes(windows1256);
-
- // "hello world\r\n" as windows-1252
- byte[] helloBytes = "hello world\r\n".getBytes(StandardCharsets.US_ASCII);
-
- System.out.println("=== Arabic bytes decoded with different charsets ===");
- for (String csName : new String[]{"windows-1256", "x-MacCyrillic", "UTF-8"}) {
- Charset cs = Charset.forName(csName);
- String decoded = CharSoupEncodingDetector.decode(arabicBytes, cs);
- printScores(csName, decoded);
- }
-
- System.out.println("\n=== 'hello world\\r\\n' decoded with different charsets ===");
- for (String csName : new String[]{"windows-1252", "IBM500"}) {
- Charset cs = Charset.forName(csName);
- String decoded = CharSoupEncodingDetector.decode(helloBytes, cs);
- printScores(csName, decoded);
- }
-
- // Also try some real-world short text
- System.out.println("\n=== Short real text ===");
- printScores("English sentence", "The quick brown fox jumps over the lazy dog.");
- printScores("French sentence", "Le renard brun rapide saute par-dessus le chien paresseux.");
- printScores("German sentence", "Der schnelle braune Fuchs springt \u00fcber den faulen Hund.");
- }
-
- private void printScores(String label, String text) {
- int totalChars = text.length();
- int letterCount = 0;
- int replacementCount = 0;
- int controlCount = 0;
- int spaceCount = 0;
- int digitCount = 0;
- int punctCount = 0;
- int otherCount = 0;
-
- for (int i = 0; i < text.length(); ) {
- int cp = text.codePointAt(i);
- i += Character.charCount(cp);
-
- if (cp == 0xFFFD) {
- replacementCount++;
- } else if (Character.isISOControl(cp) || (cp >= 0x80 && cp <= 0x9F)) {
- controlCount++;
- } else if (Character.isLetter(cp)) {
- letterCount++;
- } else if (Character.isWhitespace(cp)) {
- spaceCount++;
- } else if (Character.isDigit(cp)) {
- digitCount++;
- } else if (isPunctuation(cp)) {
- punctCount++;
- } else {
- otherCount++;
- }
- }
-
- float letterRatio = totalChars > 0 ? (float) letterCount / totalChars : 0;
- float junkRatio = totalChars > 0 ?
- (float) (replacementCount + controlCount) / totalChars : 0;
- float nonLetterNonSpaceRatio = totalChars > 0 ?
- (float) (totalChars - letterCount - spaceCount) / totalChars : 0;
-
- System.out.printf(Locale.ROOT,
- " %-20s len=%3d letters=%.2f junk(repl+ctrl)=%.2f " +
- "nonLetterNonSpace=%.2f [L=%d S=%d P=%d D=%d R=%d C=%d O=%d]%n",
- label, totalChars, letterRatio, junkRatio, nonLetterNonSpaceRatio,
- letterCount, spaceCount, punctCount, digitCount,
- replacementCount, controlCount, otherCount);
-
- // Show first 60 chars with hex for non-printable
- StringBuilder preview = new StringBuilder();
- for (int i = 0; i < Math.min(text.length(), 60); ) {
- int cp = text.codePointAt(i);
- i += Character.charCount(cp);
- if (cp >= 0x20 && cp < 0x7F) {
- preview.appendCodePoint(cp);
- } else if (Character.isLetter(cp)) {
- preview.appendCodePoint(cp);
- } else {
- preview.append(String.format(Locale.ROOT, "\\u%04X", cp));
- }
- }
- System.out.printf(Locale.ROOT, " %-20s text: %s%n", "", preview);
- }
-
- private boolean isPunctuation(int cp) {
- int type = Character.getType(cp);
- return type == Character.CONNECTOR_PUNCTUATION ||
- type == Character.DASH_PUNCTUATION ||
- type == Character.END_PUNCTUATION ||
- type == Character.FINAL_QUOTE_PUNCTUATION ||
- type == Character.INITIAL_QUOTE_PUNCTUATION ||
- type == Character.OTHER_PUNCTUATION ||
- type == Character.START_PUNCTUATION;
- }
-}
diff --git a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/ByteNgramFeatureExtractor.java b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/ByteNgramFeatureExtractor.java
deleted file mode 100644
index e72286411d..0000000000
--- a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/ByteNgramFeatureExtractor.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.chardetect;
-
-import org.apache.tika.ml.FeatureExtractor;
-
-/**
- * Feature extractor for raw bytes for charset detection, using FNV-1a hashing
- * into a fixed-width bucket array.
- *
- * <h3>Features emitted</h3>
- * <ul>
- * <li><strong>Unigrams</strong>: every byte {@code b} where
- * {@code (b & 0xFF) >= 0x80}. These directly encode the high-byte
- * frequency distribution that distinguishes single-byte encodings
- * (KOI8-R vs Windows-1251 vs ISO-8859-2, etc.).</li>
- * <li><strong>Bigrams</strong>: consecutive pairs {@code (b[i], b[i+1])}
- * where {@code (b[i] & 0xFF) >= 0x80}. Anchoring on a high first byte
- * captures multi-byte character structure (Big5, Shift-JIS, GBK,
- * EUC-* lead/trail pairs) while automatically excluding ASCII-ASCII
- * pairs produced by HTML tag markup — those bytes are all below 0x80
- * and carry no charset signal.</li>
- * </ul>
- *
- * <h3>Why the high-byte filter matters</h3>
- * <p>Training data is clean text (no HTML tags). Inference data is often raw
- * HTML (many ASCII tag bytes). Without the filter, the model would see a
- * different byte distribution at inference time than at training time. By
- * ignoring bytes below 0x80 entirely, HTML tags are invisible to both the
- * training and inference feature computation — no stripping needed.</p>
- *
- * <h3>No salting needed</h3>
- * <p>Unigrams hash values {@code 0x0080–0x00FF}; bigrams anchored on a high
- * first byte produce values {@code 0x8000–0xFFFF}. These ranges do not
- * overlap, so unigrams and bigrams naturally occupy different regions of the
- * hash space without an explicit salt.</p>
- */
-public class ByteNgramFeatureExtractor implements FeatureExtractor<byte[]> {
-
- private static final int FNV_PRIME = 0x01000193;
- private static final int FNV_OFFSET = 0x811c9dc5;
-
- private final int numBuckets;
-
- /**
- * @param numBuckets number of hash buckets (feature-vector dimension).
- * 2048 is a good default: large enough to limit collisions
- * across the tens of thousands of active multi-byte bigrams,
- * small enough that the model stays compact.
- */
- public ByteNgramFeatureExtractor(int numBuckets) {
- if (numBuckets <= 0) {
- throw new IllegalArgumentException("numBuckets must be positive: " + numBuckets);
- }
- this.numBuckets = numBuckets;
- }
-
- @Override
- public int[] extract(byte[] input) {
- int[] counts = new int[numBuckets];
- if (input == null || input.length == 0) {
- return counts;
- }
- extractInto(input, 0, input.length, counts);
- return counts;
- }
-
- /**
- * Extract from a sub-range of a byte array.
- */
- public int[] extract(byte[] input, int offset, int length) {
- int[] counts = new int[numBuckets];
- if (input == null || length == 0) {
- return counts;
- }
- extractInto(input, offset, offset + length, counts);
- return counts;
- }
-
- /**
- * Sparse extraction into caller-owned, reusable buffers.
- *
- * <p>This is O(probe length), not O(numBuckets), making it safe for large
- * bucket counts (e.g. 65536) in tight training loops.
- *
- * <p>After the call, {@code dense[touched[0..n-1]]} hold the non-zero
- * counts. The caller <em>must</em> clear those entries after use:
- * <pre>{@code
- * for (int i = 0; i < n; i++) dense[touched[i]] = 0;
- * }</pre>
- * This keeps the dense buffer zeroed for the next call without a full
- * {@code Arrays.fill}.
- *
- * @param input raw bytes to extract features from
- * @param dense caller-allocated scratch buffer of length {@code numBuckets}
- * (must be all-zeros on entry; caller clears it after use)
- * @param touched caller-allocated buffer; receives the indices of non-zero
- * buckets (length {@code numBuckets} is a safe upper bound,
- * but 2 * probe.length + 2 is the true worst case)
- * @return number of active entries written into {@code touched}
- */
- public int extractSparseInto(byte[] input, int[] dense, int[] touched) {
- if (input == null || input.length == 0) {
- return 0;
- }
- int n = 0;
- for (int i = 0; i < input.length; i++) {
- int bi = input[i] & 0xFF;
- if (bi < 0x80) {
- continue;
- }
- // Unigram
- int h = (FNV_OFFSET ^ bi) * FNV_PRIME;
- int b = (h & 0x7fffffff) % numBuckets;
- if (dense[b] == 0) {
- touched[n++] = b;
- }
- dense[b]++;
-
- // Bigram
- if (i + 1 < input.length) {
- int bi1 = input[i + 1] & 0xFF;
- h = (FNV_OFFSET ^ bi) * FNV_PRIME;
- h = (h ^ bi1) * FNV_PRIME;
- b = (h & 0x7fffffff) % numBuckets;
- if (dense[b] == 0) {
- touched[n++] = b;
- }
- dense[b]++;
- }
- }
- return n;
- }
-
- private void extractInto(byte[] b, int from, int to, int[] counts) {
- for (int i = from; i < to; i++) {
- int bi = b[i] & 0xFF;
- if (bi < 0x80) {
- continue; // ASCII — no charset signal, skip
- }
-
- // Unigram: hash the single high byte
- int h = (FNV_OFFSET ^ bi) * FNV_PRIME;
- counts[bucket(h)]++;
-
- // Bigram: anchor on this high byte, pair with whatever follows
- if (i + 1 < to) {
- int bi1 = b[i + 1] & 0xFF;
- h = (FNV_OFFSET ^ bi) * FNV_PRIME;
- h = (h ^ bi1) * FNV_PRIME;
- counts[bucket(h)]++;
- }
- }
- }
-
- private int bucket(int hash) {
- return (hash & 0x7fffffff) % numBuckets;
- }
-
- @Override
- public int getNumBuckets() {
- return numBuckets;
- }
-
- /**
- * Returns the fraction of bytes in {@code input} that are below 0x80 and
- * therefore contribute <em>no features</em> to this extractor.
- *
- * <p>This is the byte-level analogue of the word-level OOV ("out of
- * vocabulary") rate used in language-detection evaluation: a high ratio
- * means the sample is essentially pure ASCII and the model has nothing to
- * distinguish it from any other encoding.</p>
- *
- * <p>Thresholds used by the training-data pipeline to filter out low-signal
- * chunks ({@code build_charset_training.py}):
- * <ul>
- * <li>CJK multibyte encodings: OOV > 0.80 (i.e. high-byte ratio < 0.20)</li>
- * <li>SBCS / other legacy encodings: OOV > 0.98 (high-byte ratio < 0.02)</li>
- * <li>ASCII / ISO-2022 / UTF-16 / UTF-32: exempt (by design)</li>
- * </ul>
- *
- * @return value in [0.0, 1.0]; 1.0 means all bytes are ASCII (fully OOV),
- * 0.0 means all bytes are high bytes.
- */
- public static double oovRate(byte[] input) {
- if (input == null || input.length == 0) {
- return 1.0;
- }
- int ascii = 0;
- for (byte b : input) {
- if ((b & 0xFF) < 0x80) {
- ascii++;
- }
- }
- return (double) ascii / input.length;
- }
-}
diff --git a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java b/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
deleted file mode 100644
index d10e8161c6..0000000000
--- a/tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
+++ /dev/null
@@ -1,422 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.chardetect;
-
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-
-/**
- * Fast, rule-based encoding checks that run before the statistical model.
- *
- * <h3>Pipeline</h3>
- * <ol>
- * <li>{@link #checkAscii}: no bytes >= 0x80 → UTF-8 (ASCII is a subset)</li>
- * <li>{@link #detectIso2022}: ISO-2022 escape sequences present → ISO-2022-JP,
- * ISO-2022-KR, or ISO-2022-CN depending on the designation sequence</li>
- * <li>{@link #checkUtf8}: validate UTF-8 multi-byte grammar; returns a
- * {@link Utf8Result} indicating whether the bytes are definitively UTF-8,
- * definitively not UTF-8, or ambiguous (pass to model).</li>
- * </ol>
- *
- * <p>UTF-16/32 detection is handled upstream by
- * {@link org.apache.tika.utils.ByteEncodingHint} and is not repeated here.</p>
- *
- * <p>IBM424 (EBCDIC Hebrew) is detected via {@link #checkIbm424}: the Hebrew
- * letters in this code page occupy bytes 0x41–0x6A, which fall entirely below
- * the 0x80 threshold used by the statistical model's feature extractor. The
- * EBCDIC space (0x40) vs ASCII space (0x20) frequency ratio provides a cheap
- * first-pass EBCDIC gate before the Hebrew letter frequencies are checked.</p>
- *
- * <p>All methods are stateless and safe to call from multiple threads.</p>
- */
-public final class StructuralEncodingRules {
-
- private StructuralEncodingRules() {}
-
- /** ISO-2022 ESC byte. */
- private static final int ESC = 0x1B;
-
- /**
- * Ratio of valid high bytes required to call UTF-8 "definitive".
- * If the sample has more than {@value #MIN_HIGH_BYTE_RATIO_FOR_UTF8} of its
- * bytes in multi-byte sequences and they all parse correctly, we trust UTF-8.
- */
- private static final double MIN_HIGH_BYTE_RATIO_FOR_UTF8 = 0.01; // at least 1%
-
- // -----------------------------------------------------------------------
- // Public API
- // -----------------------------------------------------------------------
-
- /**
- * Returns {@code true} if {@code bytes} contains no bytes with value
- * >= 0x80 (i.e. pure 7-bit ASCII, which is a strict subset of UTF-8).
- */
- public static boolean checkAscii(byte[] bytes) {
- return checkAscii(bytes, 0, bytes.length);
- }
-
- public static boolean checkAscii(byte[] bytes, int offset, int length) {
- int end = offset + length;
- for (int i = offset; i < end; i++) {
- if ((bytes[i] & 0xFF) >= 0x80) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Detects ISO-2022-JP, ISO-2022-KR, and ISO-2022-CN by scanning for their
- * characteristic ESC designation sequences.
- *
- * <p>All three share the {@code ESC $} ({@code 0x1B 0x24}) prefix, so we
- * must read further to distinguish them:</p>
- * <pre>
- * ISO-2022-JP: ESC $ B (JIS X 0208-1983)
- * ESC $ @ (JIS X 0208-1978)
- * ESC $ ( D (JIS X 0212 supplementary)
- * ISO-2022-KR: ESC $ ) C
- * ISO-2022-CN: ESC $ ) A (GB2312)
- * ESC $ ) G (CNS 11643 plane 1)
- * ESC $ * H (CNS 11643 plane 2)
- * </pre>
- *
- * <p>If {@code ESC $} is found but no recognised third byte follows (or the
- * buffer is too short), ISO-2022-JP is returned as the most common default.</p>
- *
- * @return the detected ISO-2022 charset, or {@code null} if no ISO-2022
- * escape sequence is found
- */
- public static Charset detectIso2022(byte[] bytes) {
- return detectIso2022(bytes, 0, bytes.length);
- }
-
- public static Charset detectIso2022(byte[] bytes, int offset, int length) {
- int end = offset + length;
- for (int i = offset; i < end - 1; i++) {
- if ((bytes[i] & 0xFF) != ESC) {
- continue;
- }
- int b1 = bytes[i + 1] & 0xFF;
-
- if (b1 == 0x24) { // ESC $
- // Need at least one more byte to classify
- if (i + 2 >= end) {
- return Charset.forName("ISO-2022-JP"); // ESC $ alone → JP default
- }
- int b2 = bytes[i + 2] & 0xFF;
- switch (b2) {
- case 0x42: // ESC $ B — JIS X 0208-1983
- case 0x40: // ESC $ @ — JIS X 0208-1978
- return Charset.forName("ISO-2022-JP");
- case 0x28: // ESC $ ( — JIS X 0212 (JP supplementary)
- return Charset.forName("ISO-2022-JP");
- case 0x29: // ESC $ ) — KR or CN depending on b3
- if (i + 3 >= end) {
- return Charset.forName("ISO-2022-JP"); // can't tell, JP most common
- }
- int b3paren = bytes[i + 3] & 0xFF;
- if (b3paren == 0x43) return Charset.forName("ISO-2022-KR"); // ESC $ ) C
- if (b3paren == 0x41 || b3paren == 0x47) return Charset.forName("ISO-2022-CN");
- return Charset.forName("ISO-2022-JP");
- case 0x2A: // ESC $ * — CNS 11643 plane 2
- return Charset.forName("ISO-2022-CN");
- default:
- return Charset.forName("ISO-2022-JP"); // unknown designation → JP default
- }
- }
-
- if (b1 == 0x28) { // ESC ( — single-byte designation (e.g. ASCII restore)
- // These appear in all ISO-2022 variants and don't help distinguish
- continue;
- }
- }
- return null; // no ISO-2022 escape found
- }
-
- /**
- * Returns {@code true} if HZ-GB-2312 switching sequences are present.
- *
- * <p>HZ is a 7-bit encoding: it uses {@code ~\{} ({@code 0x7E 0x7B}) to
- * enter two-byte GB2312 mode and {@code ~\}} ({@code 0x7E 0x7D}) to return
- * to ASCII mode. Like ISO-2022, all bytes are below 0x80, so the model
- * would see no features and must be bypassed with this structural check.</p>
- */
- public static boolean checkHz(byte[] bytes) {
- return checkHz(bytes, 0, bytes.length);
- }
-
- public static boolean checkHz(byte[] bytes, int offset, int length) {
- int end = offset + length - 1;
- for (int i = offset; i < end; i++) {
- if ((bytes[i] & 0xFF) == 0x7E) {
- int next = bytes[i + 1] & 0xFF;
- if (next == 0x7B || next == 0x7D) { // ~{ or ~}
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Detects IBM424 (EBCDIC Hebrew) by examining the sub-0x80 byte landscape.
- *
- * <h3>Why this is needed</h3>
- * <p>In EBCDIC, the space character is {@code 0x40} (not {@code 0x20} as in
- * ASCII). In IBM424 specifically, the 22 Hebrew base letters plus their five
- * final forms occupy three byte clusters entirely below {@code 0x80}:</p>
- * <pre>
- * 0x41–0x49 alef … tet (9 letters)
- * 0x51–0x59 yod … samekh (9 letters)
- * 0x62–0x6A ayin … tav (9 letters + final-pe, tsadi, etc.)
- * </pre>
- * <p>The statistical model ignores all bytes below {@code 0x80}, so these
- * letters are invisible to it. This structural rule detects them directly.</p>
- *
- * <h3>Algorithm</h3>
- * <ol>
- * <li><b>EBCDIC gate:</b> byte {@code 0x40} (EBCDIC space) must appear
- * significantly more often than {@code 0x20} (ASCII space). In normal
- * Latin text {@code 0x40} is the rare {@code @} character; in any EBCDIC
- * text it is the word separator and appears at ~10–20% of bytes.</li>
- * <li><b>Hebrew letter gate:</b> the combined frequency of bytes in the
- * three Hebrew clusters above must exceed {@value #IBM424_HEBREW_THRESHOLD}
- * of the sample length. Genuine Hebrew text has ~65% of its
- * printable characters in these ranges. ASCII text with the same byte
- * values (upper-case A–I, Q–Y, lower-case b–j) stays well below this
- * threshold in practice.</li>
- * </ol>
- *
- * @return {@code true} if the byte stream is almost certainly IBM424
- */
- public static boolean checkIbm424(byte[] bytes) {
- return checkIbm424(bytes, 0, bytes.length);
- }
-
- public static boolean checkIbm424(byte[] bytes, int offset, int length) {
- if (length < 8) {
- return false;
- }
- int sample = Math.min(length, 4096);
- int end = offset + sample;
-
- int ebcdicSpace = 0; // 0x40 — EBCDIC word separator
- int asciiSpace = 0; // 0x20 — ASCII word separator
- int hebrewBytes = 0; // 0x41-0x49, 0x51-0x59, 0x62-0x6A
-
- int prev = -1;
- for (int i = offset; i < end; i++) {
- int b = bytes[i] & 0xFF;
- if (b == 0x40) {
- // In Shift_JIS, 0x40 appears only as a trail byte after a lead byte
- // (0x81–0x9F or 0xE0–0xFC). Discount it as EBCDIC space in that case.
- boolean isShiftJisTrail = (prev >= 0x81 && prev <= 0x9F)
- || (prev >= 0xE0 && prev <= 0xFC);
- if (!isShiftJisTrail) {
- ebcdicSpace++;
- }
- } else if (b == 0x20) {
- asciiSpace++;
- } else if ((b >= 0x41 && b <= 0x49)
- || (b >= 0x51 && b <= 0x59)
- || (b >= 0x62 && b <= 0x6A)) {
- hebrewBytes++;
- }
- prev = b;
- }
-
- // Gate 1: 0x40 must dominate over 0x20 (EBCDIC vs ASCII whitespace).
- // We require 0x40 to be at least 3× as frequent as 0x20, and appear
- // at least 3% of the sample (rules out near-empty / binary content).
- boolean ebcdicLikely = ebcdicSpace >= sample * 0.03
- && ebcdicSpace > asciiSpace * 3;
-
- // Gate 2: Hebrew letter density must exceed the threshold.
- boolean hebrewDense = hebrewBytes > sample * IBM424_HEBREW_THRESHOLD;
-
- return ebcdicLikely && hebrewDense;
- }
-
- /**
- * Minimum fraction of bytes in IBM424 Hebrew letter positions (0x41–0x49,
- * 0x51–0x59, 0x62–0x6A) required to confirm IBM424. Set conservatively to
- * avoid false positives on ASCII text where those same byte values are
- * upper-/lower-case Latin letters.
- */
- private static final double IBM424_HEBREW_THRESHOLD = 0.12;
-
- /**
- * Returns {@code true} if the probe contains any byte in the C1 control
- * range {@code 0x80–0x9F}.
- *
- * <p>In every ISO-8859-X encoding those byte values are C1 control
- * characters that never appear in real text. In every Windows-12XX
- * encoding they are printable characters (smart quotes, Euro sign,
- * em-dash, …). Their presence is therefore definitive proof that the
- * content is <em>not</em> a valid ISO-8859-X encoding and should be
- * attributed to the corresponding Windows-12XX variant instead.</p>
- */
- public static boolean hasC1Bytes(byte[] bytes) {
- return hasC1Bytes(bytes, 0, bytes.length);
- }
-
- public static boolean hasC1Bytes(byte[] bytes, int offset, int length) {
- int end = offset + length;
- for (int i = offset; i < end; i++) {
- int v = bytes[i] & 0xFF;
- if (v >= 0x80 && v <= 0x9F) {
- return true;
- }
- }
- return false;
- }
-
- /** @deprecated Use {@link #detectIso2022} which distinguishes JP/KR/CN. */
- @Deprecated
- public static boolean checkIso2022Jp(byte[] bytes) {
- return detectIso2022(bytes) != null;
- }
-
- /**
- * Validates the UTF-8 byte grammar of the sample and returns one of three
- * outcomes:
- * <ul>
- * <li>{@link Utf8Result#DEFINITIVE_UTF8}: all multi-byte sequences are
- * valid <em>and</em> the sample contains enough high bytes to be
- * informative. Use UTF-8.</li>
- * <li>{@link Utf8Result#NOT_UTF8}: at least one invalid byte sequence was
- * found. Remove UTF-8 from the candidate set.</li>
- * <li>{@link Utf8Result#AMBIGUOUS}: the sample is structurally valid UTF-8
- * but contains very few high bytes (almost pure ASCII), so validity is
- * uninformative. Pass to the model.</li>
- * </ul>
- */
- public static Utf8Result checkUtf8(byte[] bytes) {
- return checkUtf8(bytes, 0, bytes.length);
- }
-
- public static Utf8Result checkUtf8(byte[] bytes, int offset, int length) {
- int highByteCount = 0;
- int i = offset;
- int end = offset + length;
-
- while (i < end) {
- int b = bytes[i] & 0xFF;
-
- if (b < 0x80) {
- i++;
- continue;
- }
-
- highByteCount++;
-
- // Determine expected continuation count from the lead byte
- int seqLen;
- if (b >= 0xF8) {
- // 5-/6-byte sequences are not valid Unicode
- return Utf8Result.NOT_UTF8;
- } else if (b >= 0xF0) {
- seqLen = 4;
- } else if (b >= 0xE0) {
- seqLen = 3;
- } else if (b >= 0xC0) {
- seqLen = 2;
- } else {
- // 0x80–0xBF is a continuation byte without a lead → invalid
- return Utf8Result.NOT_UTF8;
- }
-
- // Overlong 2-byte sequence (C0 or C1 lead)
- if (seqLen == 2 && b <= 0xC1) {
- return Utf8Result.NOT_UTF8;
- }
-
- // Check that the right number of continuation bytes follow
- for (int k = 1; k < seqLen; k++) {
- if (i + k >= end) {
- // Truncated sequence at end of sample — treat as ambiguous
- // (the sample just ran out, not necessarily bad data)
- break;
- }
- int cb = bytes[i + k] & 0xFF;
- if (cb < 0x80 || cb > 0xBF) {
- return Utf8Result.NOT_UTF8;
- }
- }
-
- // Validate scalar value ranges for 3- and 4-byte sequences
- if (seqLen == 3) {
- int cp = ((b & 0x0F) << 12)
- | ((i + 1 < end ? bytes[i + 1] & 0xFF : 0) & 0x3F) << 6
- | ((i + 2 < end ? bytes[i + 2] & 0xFF : 0) & 0x3F);
- // Overlong encoding (< U+0800) or surrogate pair range
- if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF)) {
- return Utf8Result.NOT_UTF8;
- }
- } else if (seqLen == 4) {
- int cp = ((b & 0x07) << 18)
- | ((i + 1 < end ? bytes[i + 1] & 0xFF : 0) & 0x3F) << 12
- | ((i + 2 < end ? bytes[i + 2] & 0xFF : 0) & 0x3F) << 6
- | ((i + 3 < end ? bytes[i + 3] & 0xFF : 0) & 0x3F);
- // Overlong or above U+10FFFF
- if (cp < 0x10000 || cp > 0x10FFFF) {
- return Utf8Result.NOT_UTF8;
- }
- }
-
- i += seqLen;
- }
-
- // Grammar is valid. Was there enough evidence?
- double highRatio = length > 0 ? (double) highByteCount / length : 0.0;
- if (highRatio >= MIN_HIGH_BYTE_RATIO_FOR_UTF8) {
- return Utf8Result.DEFINITIVE_UTF8;
- }
- return Utf8Result.AMBIGUOUS;
- }
-
- // -----------------------------------------------------------------------
- // Result type
- // -----------------------------------------------------------------------
-
- /**
- * Outcome of the UTF-8 structural check.
- */
- public enum Utf8Result {
- /** Sample is structurally valid UTF-8 with enough high bytes to be sure. */
- DEFINITIVE_UTF8,
- /** Sample contains at least one invalid UTF-8 sequence. */
- NOT_UTF8,
- /**
- * Sample is structurally valid but nearly all-ASCII. Cannot confirm or
- * deny; pass to the statistical model.
- */
- AMBIGUOUS;
-
- public boolean isDefinitive() {
- return this != AMBIGUOUS;
- }
-
- public Charset toCharset() {
- if (this == DEFINITIVE_UTF8) {
- return StandardCharsets.UTF_8;
- }
- throw new IllegalStateException("Only DEFINITIVE_UTF8 has a Charset: " + this);
- }
- }
-}
diff --git a/tika-ml/tika-ml-chardetect/src/main/resources/org/apache/tika/ml/chardetect/chardetect.bin b/tika-ml/tika-ml-chardetect/src/main/resources/org/apache/tika/ml/chardetect/chardetect.bin
deleted file mode 100644
index 36e4fd3682..0000000000
Binary files a/tika-ml/tika-ml-chardetect/src/main/resources/org/apache/tika/ml/chardetect/chardetect.bin and /dev/null differ
diff --git a/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/tools/EvalCharsetDetectors.java b/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/tools/EvalCharsetDetectors.java
deleted file mode 100644
index f529c8fc53..0000000000
--- a/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/tools/EvalCharsetDetectors.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.chardetect.tools;
-
-import java.io.DataInputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.tika.detect.EncodingDetector;
-import org.apache.tika.detect.EncodingResult;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.ml.chardetect.CharsetConfusables;
-import org.apache.tika.ml.chardetect.MlEncodingDetector;
-import org.apache.tika.ml.chardetect.MlEncodingDetector.Rule;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.txt.Icu4jEncodingDetector;
-import org.apache.tika.parser.txt.UniversalEncodingDetector;
-
-/**
- * Compares {@link MlEncodingDetector} against ICU4J and juniversalchardet.
- *
- * <p>Supports:
- * <ul>
- * <li>{@code --lengths 20,50,100,200,full} — per-probe-length accuracy sweep</li>
- * <li>{@code --confusion} — top-confusion report for the ML-All detector</li>
- * </ul>
- *
- * <p>Usage:
- * <pre>
- * java EvalCharsetDetectors \
- * [--model /path/to/chardetect.bin] \
- * --data /path/to/test-dir \
- * [--lengths 20,50,100,200,full] \
- * [--confusion]
- * </pre>
- */
-public class EvalCharsetDetectors {
-
- private static final String NULL_LABEL = "(null)";
- private static final int FULL_LENGTH = Integer.MAX_VALUE;
-
- private static final double OOV_THRESHOLD_CJK = 0.80;
- private static final double OOV_THRESHOLD_SBCS = 0.98;
- private static final Set<String> CJK_CHARSETS = Set.of(
- "Big5", "Big5-HKSCS", "EUC-JP", "EUC-KR", "EUC-TW",
- "GB18030", "GB2312", "GBK", "Shift_JIS"
- );
- private static final Set<String> OOV_EXEMPT = Set.of(
- "US-ASCII", "UTF-16-LE", "UTF-16-BE", "UTF-32-LE", "UTF-32-BE",
- "ISO-2022-JP", "ISO-2022-KR", "ISO-2022-CN", "HZ"
- );
-
- private static final String[] COL_NAMES = {"Stat", "+ISO", "+CJK", "All", "ICU4J", "juniv"};
- private static final int NUM_DETECTORS = COL_NAMES.length;
- // Index of the "All" detector — used for confusion matrix
- private static final int IDX_ALL = 3;
-
- public static void main(String[] args) throws Exception {
- Path modelPath = null;
- Path dataDir = null;
- int[] probeLengths = {FULL_LENGTH};
- boolean showConfusion = false;
-
- for (int i = 0; i < args.length; i++) {
- switch (args[i]) {
- case "--model":
- modelPath = Paths.get(args[++i]);
- break;
- case "--data":
- dataDir = Paths.get(args[++i]);
- break;
- case "--lengths":
- probeLengths = parseLengths(args[++i]);
- break;
- case "--confusion":
- showConfusion = true;
- break;
- default:
- System.err.println("Unknown argument: " + args[i]);
- System.exit(1);
- }
- }
- if (dataDir == null) {
- System.err.println(
- "Usage: EvalCharsetDetectors [--model <path>] --data <dir>"
- + " [--lengths 20,50,100,full] [--confusion]");
- System.exit(1);
- }
-
- MlEncodingDetector base = modelPath != null
- ? new MlEncodingDetector(modelPath)
- : new MlEncodingDetector();
-
- EncodingDetector[] detectors = {
- base.withRules(EnumSet.noneOf(Rule.class)),
- base.withRules(EnumSet.of(Rule.STRUCTURAL_GATES, Rule.ISO_TO_WINDOWS)),
- base.withRules(EnumSet.of(Rule.STRUCTURAL_GATES, Rule.CJK_GRAMMAR)),
- base,
- new Icu4jEncodingDetector(),
- new UniversalEncodingDetector()
- };
-
- List<Path> testFiles = Files.list(dataDir)
- .filter(p -> p.getFileName().toString().endsWith(".bin.gz"))
- .sorted()
- .collect(Collectors.toList());
-
- if (testFiles.isEmpty()) {
- System.err.println("No .bin.gz files found in: " + dataDir);
- System.exit(1);
- }
-
- // Load all samples once; truncation happens per probe-length sweep
- List<String> charsets = new ArrayList<>();
- List<List<byte[]>> allSamplesPerCharset = new ArrayList<>();
- for (Path f : testFiles) {
- String cs = f.getFileName().toString().replaceAll("\\.bin\\.gz$", "");
- List<byte[]> samples = loadSamples(f);
- if (!samples.isEmpty()) {
- charsets.add(cs);
- allSamplesPerCharset.add(samples);
- }
- }
-
- // One pass per probe length
- for (int probeLen : probeLengths) {
- String lenLabel = probeLen == FULL_LENGTH ? "full" : probeLen + "B";
- System.out.println("\n=== Probe length: " + lenLabel + " ===");
- printHeader();
-
- long totalN = 0;
- long[][] totals = new long[NUM_DETECTORS][2];
- // confusion[trueIdx][predLabel] = count (only for IDX_ALL)
- List<Map<String, Integer>> confusion = new ArrayList<>();
- for (int ci = 0; ci < charsets.size(); ci++) {
- confusion.add(new HashMap<>());
- }
-
- for (int ci = 0; ci < charsets.size(); ci++) {
- String charset = charsets.get(ci);
- List<byte[]> samples = truncate(allSamplesPerCharset.get(ci), probeLen);
- int n = samples.size();
- if (n == 0) {
- continue;
- }
-
- int[][] counts = new int[NUM_DETECTORS][2];
- for (byte[] sample : samples) {
- for (int d = 0; d < NUM_DETECTORS; d++) {
- String pred = predict(detectors[d], sample);
- if (isStrict(charset, pred)) {
- counts[d][0]++;
- }
- if (isSoft(charset, pred)) {
- counts[d][1]++;
- }
- if (d == IDX_ALL && !isStrict(charset, pred)) {
- confusion.get(ci).merge(pred, 1, Integer::sum);
- }
- }
- }
-
- printRow(charset, n, counts);
- totalN += n;
- for (int d = 0; d < NUM_DETECTORS; d++) {
- totals[d][0] += counts[d][0];
- totals[d][1] += counts[d][1];
- }
- }
-
- printFooter(totalN, totals);
-
- if (showConfusion) {
- printConfusion(charsets, allSamplesPerCharset, confusion, probeLen, lenLabel);
- }
- }
- }
-
- // -----------------------------------------------------------------------
- // Confusion matrix
- // -----------------------------------------------------------------------
-
- private static void printConfusion(List<String> charsets,
- List<List<byte[]>> allSamplesPerCharset,
- List<Map<String, Integer>> confusion,
- int probeLen, String lenLabel) {
- System.out.println("\n--- Confusion (ML-All, " + lenLabel
- + ", top errors per charset) ---");
-
- boolean anyError = false;
- for (int ci = 0; ci < charsets.size(); ci++) {
- Map<String, Integer> errors = confusion.get(ci);
- if (errors.isEmpty()) {
- continue;
- }
- anyError = true;
- int total = truncate(allSamplesPerCharset.get(ci), probeLen).size();
- int totalErrors = errors.values().stream().mapToInt(Integer::intValue).sum();
- double errPct = 100.0 * totalErrors / total;
-
- // Sort errors by count descending
- List<Map.Entry<String, Integer>> sorted = new ArrayList<>(errors.entrySet());
- sorted.sort((a, b) -> b.getValue() - a.getValue());
-
- StringBuilder sb = new StringBuilder();
- sb.append(String.format(Locale.ROOT, " %-22s %5.1f%% wrong → ", charsets.get(ci), errPct));
- int shown = 0;
- for (Map.Entry<String, Integer> e : sorted) {
- if (shown > 0) {
- sb.append(", ");
- }
- sb.append(String.format(Locale.ROOT, "%s:%.1f%%",
- e.getKey(), 100.0 * e.getValue() / total));
- if (++shown >= 5) {
- break;
- }
- }
- System.out.println(sb);
- }
- if (!anyError) {
- System.out.println(" (no errors)");
- }
- }
-
- // -----------------------------------------------------------------------
- // Table formatting
- // -----------------------------------------------------------------------
-
- private static void printHeader() {
- StringBuilder sb1 = new StringBuilder();
- sb1.append(String.format(Locale.ROOT, "%-22s %5s ", "", "N"));
- sb1.append("| --- ML ablation --------------------------------- ");
- sb1.append("| --- Baselines ----------------------- |");
- System.out.println(sb1);
-
- StringBuilder sb2 = new StringBuilder();
- sb2.append(String.format(Locale.ROOT, "%-22s %5s ", "Charset", ""));
- for (String name : COL_NAMES) {
- sb2.append(String.format(Locale.ROOT, "| %-4s R%% S%% ", name));
- }
- sb2.append("|");
- System.out.println(sb2);
- System.out.println("-".repeat(sb2.length()));
- }
-
- private static void printRow(String charset, int n, int[][] counts) {
- StringBuilder sb = new StringBuilder();
- sb.append(String.format(Locale.ROOT, "%-22s %5d ", charset, n));
- for (int d = 0; d < NUM_DETECTORS; d++) {
- sb.append(String.format(Locale.ROOT, "| %5.1f %5.1f ",
- pct(counts[d][0], n), pct(counts[d][1], n)));
- }
- sb.append("|");
- System.out.println(sb);
- }
-
- private static void printFooter(long totalN, long[][] totals) {
- System.out.println("-".repeat(120));
- StringBuilder sb = new StringBuilder();
- sb.append(String.format(Locale.ROOT, "%-22s %5d ", "OVERALL", totalN));
- for (int d = 0; d < NUM_DETECTORS; d++) {
- sb.append(String.format(Locale.ROOT, "| %5.1f %5.1f ",
- pct(totals[d][0], totalN), pct(totals[d][1], totalN)));
- }
- sb.append("|");
- System.out.println(sb);
- System.out.println(" Stat=model only | +ISO=+C1-correction | +CJK=+grammar | "
- + "All=production | R%=strict | S%=soft");
- }
-
- // -----------------------------------------------------------------------
- // Helpers
- // -----------------------------------------------------------------------
-
- private static int[] parseLengths(String spec) {
- String[] parts = spec.split(",");
- int[] result = new int[parts.length];
- for (int i = 0; i < parts.length; i++) {
- result[i] = "full".equalsIgnoreCase(parts[i].trim())
- ? FULL_LENGTH : Integer.parseInt(parts[i].trim());
- }
- return result;
- }
-
- /** Returns samples truncated to at most {@code maxLen} bytes. */
- private static List<byte[]> truncate(List<byte[]> samples, int maxLen) {
- if (maxLen == FULL_LENGTH) {
- return samples;
- }
- List<byte[]> out = new ArrayList<>(samples.size());
- for (byte[] s : samples) {
- out.add(s.length <= maxLen ? s : Arrays.copyOf(s, maxLen));
- }
- return out;
- }
-
- private static String predict(EncodingDetector detector, byte[] sample) {
- try (TikaInputStream tis = TikaInputStream.get(sample)) {
- List<EncodingResult> results =
- detector.detect(tis, new Metadata(), new ParseContext());
- return results.isEmpty() ? NULL_LABEL : results.get(0).getCharset().name();
- } catch (Exception e) {
- return NULL_LABEL;
- }
- }
-
- private static boolean isStrict(String actual, String predicted) {
- return !NULL_LABEL.equals(predicted)
- && normalize(actual).equals(normalize(predicted));
- }
-
- private static boolean isSoft(String actual, String predicted) {
- if (NULL_LABEL.equals(predicted)) {
- return false;
- }
- if (normalize(actual).equals(normalize(predicted))) {
- return true;
- }
- return CharsetConfusables.isLenientMatch(actual, predicted)
- || CharsetConfusables.isLenientMatch(predicted, actual);
- }
-
- private static String normalize(String name) {
- return name.toLowerCase(Locale.ROOT).replace("-", "").replace("_", "");
- }
-
- private static double pct(long correct, long total) {
- return total == 0 ? 0.0 : 100.0 * correct / total;
- }
-
- private static List<byte[]> loadSamples(Path file) throws IOException {
- List<byte[]> out = new ArrayList<>();
- try (InputStream fis = new FileInputStream(file.toFile());
- GZIPInputStream gis = new GZIPInputStream(fis);
- DataInputStream dis = new DataInputStream(gis)) {
- while (true) {
- int len;
- try {
- len = dis.readUnsignedShort();
- } catch (java.io.EOFException e) {
- break;
- }
- byte[] chunk = new byte[len];
- dis.readFully(chunk);
- out.add(chunk);
- }
- }
- return out;
- }
-}
diff --git a/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/tools/TrainCharsetModel.java b/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/tools/TrainCharsetModel.java
deleted file mode 100644
index bd9fe9546c..0000000000
--- a/tika-ml/tika-ml-chardetect/src/test/java/org/apache/tika/ml/chardetect/tools/TrainCharsetModel.java
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.ml.chardetect.tools;
-
-import java.io.DataInputStream;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.tika.ml.LinearModel;
-import org.apache.tika.ml.chardetect.ByteNgramFeatureExtractor;
-import org.apache.tika.ml.chardetect.CharsetConfusables;
-
-/**
- * Trains a {@link LinearModel} for charset detection from the binary training
- * data produced by {@code build_charset_training.py}.
- * <p>
- * <strong>Usage:</strong>
- * <pre>
- * java TrainCharsetModel \
- * --data /path/to/chardet-training \
- * --output chardetect.bin \
- * [--buckets 65536] \
- * [--epochs 3] \
- * [--lr 0.05] \
- * [--max-samples-per-class 500000]
- * </pre>
- * <p>
- * Training file format (per charset, one {@code <charset>.bin.gz} file):
- * <pre>
- * [uint16 length][bytes of that length]
- * ... repeated ...
- * </pre>
- * Each record is a raw byte chunk that should be classified as the charset
- * named by the filename.
- */
-public class TrainCharsetModel {
-
- private static final int DEFAULT_NUM_BUCKETS = 2048;
- private static final int DEFAULT_EPOCHS = 3;
- private static final float DEFAULT_LR = 0.05f;
- private static final int DEFAULT_MAX_SAMPLES = 500_000;
-
- public static void main(String[] args) throws IOException {
- Path dataDir = null;
- Path outputPath = Paths.get("chardetect.bin");
- int numBuckets = DEFAULT_NUM_BUCKETS;
- int epochs = DEFAULT_EPOCHS;
- float lr = DEFAULT_LR;
- int maxSamplesPerClass = DEFAULT_MAX_SAMPLES;
-
- for (int i = 0; i < args.length; i++) {
- switch (args[i]) {
- case "--data":
- dataDir = Paths.get(args[++i]);
- break;
- case "--output":
- outputPath = Paths.get(args[++i]);
- break;
- case "--buckets":
- numBuckets = Integer.parseInt(args[++i]);
- break;
- case "--epochs":
- epochs = Integer.parseInt(args[++i]);
- break;
- case "--lr":
- lr = Float.parseFloat(args[++i]);
- break;
- case "--max-samples-per-class":
- maxSamplesPerClass = Integer.parseInt(args[++i]);
- break;
- default:
- System.err.println("Unknown argument: " + args[i]);
- System.exit(1);
- }
- }
-
- if (dataDir == null) {
- System.err.println("Usage: TrainCharsetModel --data <dir> [options]");
- System.exit(1);
- }
-
- // Discover charset files
- List<Path> charsetFiles = Files.list(dataDir)
- .filter(p -> p.getFileName().toString().endsWith(".bin.gz"))
- .sorted()
- .collect(Collectors.toList());
-
- if (charsetFiles.isEmpty()) {
- System.err.println("No .bin.gz files found in: " + dataDir);
- System.exit(1);
- }
-
- // Build class label list (charset names stripped of .bin.gz)
- String[] labels = charsetFiles.stream()
- .map(p -> p.getFileName().toString()
- .replaceAll("\\.bin\\.gz$", ""))
- .toArray(String[]::new);
- int numClasses = labels.length;
-
- System.out.println("Classes (" + numClasses + "): " + Arrays.toString(labels));
- System.out.println("Buckets: " + numBuckets + ", epochs: " + epochs +
- ", lr: " + lr + ", max-samples/class: " + maxSamplesPerClass);
-
- ByteNgramFeatureExtractor extractor =
- new ByteNgramFeatureExtractor(numBuckets);
-
- // Build class index map
- Map<String, Integer> labelIndex = new HashMap<>();
- for (int i = 0; i < numClasses; i++) {
- labelIndex.put(labels[i], i);
- }
-
- // Load samples (up to maxSamplesPerClass per class)
- System.out.println("Loading training data ...");
- List<byte[]>[] samplesPerClass = new List[numClasses];
- long totalSamples = 0;
- for (int ci = 0; ci < numClasses; ci++) {
- samplesPerClass[ci] = loadSamples(charsetFiles.get(ci), maxSamplesPerClass);
- totalSamples += samplesPerClass[ci].size();
- System.out.printf(java.util.Locale.ROOT, " %-30s %,d samples%n", labels[ci], samplesPerClass[ci].size());
- }
- System.out.printf(java.util.Locale.ROOT, "Total training samples: %,d%n", totalSamples);
-
- // SGD training: multinomial logistic regression with L2 regularisation
- // Weight matrix: [numClasses][numBuckets]
- float[][] weights = new float[numClasses][numBuckets];
- float[] biases = new float[numClasses];
- float lambda = 1e-5f; // L2 regularisation coefficient
-
- // Build a shuffled training index: list of (classIndex, sampleIndex) pairs
- List<int[]> index = new ArrayList<>((int) Math.min(totalSamples, Integer.MAX_VALUE));
- for (int ci = 0; ci < numClasses; ci++) {
- for (int si = 0; si < samplesPerClass[ci].size(); si++) {
- index.add(new int[]{ci, si});
- }
- }
-
- // Build int[][] group indices for probability collapsing and per-charset eval
- int[][] groupIndices = CharsetConfusables.buildGroupIndices(labels);
-
- // Reusable sparse-extraction buffers (avoids per-sample allocation)
- int[] denseScratch = new int[numBuckets];
- int[] touched = new int[numBuckets]; // worst-case size
-
- for (int epoch = 0; epoch < epochs; epoch++) {
- Collections.shuffle(index);
- long strictCorrect = 0;
- float lossSum = 0f;
- int count = 0;
-
- for (int[] pair : index) {
- int trueClass = pair[0];
- byte[] sample = samplesPerClass[trueClass].get(pair[1]);
-
- // Sparse extraction: O(probeLength), not O(numBuckets)
- int nActive = extractor.extractSparseInto(sample, denseScratch, touched);
-
- // Forward pass: only iterate active buckets
- float[] logits = new float[numClasses];
- for (int c = 0; c < numClasses; c++) {
- float dot = biases[c];
- for (int t = 0; t < nActive; t++) {
- dot += weights[c][touched[t]] * denseScratch[touched[t]];
- }
- logits[c] = dot;
- }
- float[] probs = LinearModel.softmax(logits.clone());
-
- // Cross-entropy loss on the true class
- lossSum -= (float) Math.log(Math.max(probs[trueClass], 1e-12f));
-
- if (argmax(probs) == trueClass) {
- strictCorrect++;
- }
-
- // Backward pass: gradient = probs - one_hot(trueClass)
- float[] grad = probs.clone();
- grad[trueClass] -= 1f;
-
- // Sparse SGD update with lazy L2: only active buckets are touched.
- // Inactive weights start at 0 and are never pushed away without a
- // gradient, so skipping their L2 decay is correct.
- for (int c = 0; c < numClasses; c++) {
- float g = grad[c];
- biases[c] -= lr * g;
- for (int t = 0; t < nActive; t++) {
- int b = touched[t];
- weights[c][b] -= lr * (g * denseScratch[b] + lambda * weights[c][b]);
- }
- }
- count++;
-
- // Clear only the active entries (O(nActive), not O(numBuckets))
- for (int t = 0; t < nActive; t++) {
- denseScratch[touched[t]] = 0;
- }
- }
-
- System.out.printf(java.util.Locale.ROOT,
- "Epoch %d/%d loss=%.4f strict-acc=%.2f%%%n",
- epoch + 1, epochs, lossSum / count,
- 100.0 * strictCorrect / count);
- }
-
- // Quantize and save
- System.out.println("Quantizing ...");
- String[] qLabels = labels;
- float[] qScales = new float[numClasses];
- float[] qBiases = biases;
- byte[][] qWeights = new byte[numClasses][numBuckets];
-
- for (int c = 0; c < numClasses; c++) {
- float maxAbs = 1e-6f;
- for (float w : weights[c]) {
- float abs = Math.abs(w);
- if (abs > maxAbs) {
- maxAbs = abs;
- }
- }
- qScales[c] = maxAbs / 127f;
- for (int b = 0; b < numBuckets; b++) {
- int q = Math.round(weights[c][b] / qScales[c]);
- qWeights[c][b] = (byte) Math.max(-127, Math.min(127, q));
- }
- }
-
- LinearModel model = new LinearModel(numBuckets, numClasses,
- qLabels, qScales, qBiases, qWeights);
-
- try (OutputStream os = new FileOutputStream(outputPath.toFile())) {
- model.save(os);
- }
- System.out.println("Model saved to: " + outputPath);
-
- // Per-charset evaluation on the training data (in-sample, sanity check)
- System.out.println("\nPer-charset evaluation (quantized model, training data):");
- evaluatePerCharset(model, extractor, samplesPerClass, labels, groupIndices);
-
- System.out.println("Done.");
- }
-
- private static List<byte[]> loadSamples(Path file, int maxSamples) throws IOException {
- List<byte[]> samples = new ArrayList<>();
- try (InputStream fis = new FileInputStream(file.toFile());
- GZIPInputStream gis = new GZIPInputStream(fis);
- DataInputStream dis = new DataInputStream(gis)) {
- while (samples.size() < maxSamples) {
- int len;
- try {
- len = dis.readUnsignedShort();
- } catch (java.io.EOFException e) {
- break;
- }
- byte[] chunk = new byte[len];
- dis.readFully(chunk);
- samples.add(chunk);
- }
- }
- return samples;
- }
-
- /**
- * Evaluate the quantized model on the training samples and print a table
- * showing per-charset strict accuracy and lenient accuracy (confusable group
- * match counts as correct).
- */
- private static void evaluatePerCharset(
- LinearModel model,
- ByteNgramFeatureExtractor extractor,
- List<byte[]>[] samplesPerClass,
- String[] labels,
- int[][] groupIndices) {
-
- int numClasses = labels.length;
- int[] strictCorrect = new int[numClasses];
- int[] lenientCorrect = new int[numClasses];
- int[] totals = new int[numClasses];
-
- for (int trueClass = 0; trueClass < numClasses; trueClass++) {
- for (byte[] sample : samplesPerClass[trueClass]) {
- int[] features = extractor.extract(sample);
- float[] probs = CharsetConfusables.collapseGroups(
- model.predict(features), groupIndices);
- int predicted = argmax(probs);
- totals[trueClass]++;
- if (predicted == trueClass) {
- strictCorrect[trueClass]++;
- lenientCorrect[trueClass]++;
- } else if (CharsetConfusables.isLenientMatch(
- labels[trueClass], labels[predicted])) {
- lenientCorrect[trueClass]++;
- }
- }
- }
-
- // Print table
- int maxLabelLen = 0;
- for (String l : labels) {
- maxLabelLen = Math.max(maxLabelLen, l.length());
- }
- String fmt = " %-" + maxLabelLen + "s %7d %7.2f%% %7.2f%% %s%n";
- System.out.printf(java.util.Locale.ROOT,
- " %-" + maxLabelLen + "s %7s %8s %8s%n",
- "Charset", "N", "Strict", "Soft");
- System.out.println(" " + "-".repeat(maxLabelLen + 32));
-
- long totalStrict = 0;
- long totalLenient = 0;
- long totalN = 0;
- for (int c = 0; c < numClasses; c++) {
- int n = totals[c];
- if (n == 0) {
- continue;
- }
- double strict = 100.0 * strictCorrect[c] / n;
- double lenient = 100.0 * lenientCorrect[c] / n;
- // Flag rows where lenient > strict (confusable errors are happening)
- String flag = (lenientCorrect[c] > strictCorrect[c]) ? "*" : "";
- System.out.printf(java.util.Locale.ROOT, fmt,
- labels[c], n, strict, lenient, flag);
- totalStrict += strictCorrect[c];
- totalLenient += lenientCorrect[c];
- totalN += n;
- }
- System.out.println(" " + "-".repeat(maxLabelLen + 32));
- System.out.printf(java.util.Locale.ROOT, fmt,
- "OVERALL", totalN,
- 100.0 * totalStrict / totalN,
- 100.0 * totalLenient / totalN, "");
- System.out.println(" (* = confusable-group errors present; lenient > strict)");
- }
-
- private static int argmax(float[] arr) {
- int best = 0;
- for (int i = 1; i < arr.length; i++) {
- if (arr[i] > arr[best]) {
- best = i;
- }
- }
- return best;
- }
-}
diff --git a/tika-ml/tika-ml-chardetect/src/test/python/build_charset_training.py b/tika-ml/tika-ml-chardetect/src/test/python/build_charset_training.py
index 1435954a70..ad9c5caf62 100644
--- a/tika-ml/tika-ml-chardetect/src/test/python/build_charset_training.py
+++ b/tika-ml/tika-ml-chardetect/src/test/python/build_charset_training.py
@@ -18,12 +18,14 @@
Build charset-detection training, devtest, and test data from:
1. MADLAD-400 per-language sentence files (--madlad-dir, default ~/datasets/madlad/data)
- 2. Flores-200 Traditional Chinese files (--flores-dir, default ~/datasets/flores-200/flores200_dataset)
- Used only for Big5, Big5-HKSCS, EUC-TW (no Traditional Chinese in MADLAD).
+ 2. Cantonese Wikipedia plain-text sentences (--zh-yue-file,
+ default ~/datasets/zh_yuewiki/sentences_zh_yue.txt)
+ Used for Big5 and Big5-HKSCS (Traditional Chinese).
Pipeline per language
---------------------
1. Load up to --max-source-per-lang sentences from sentences_madlad.txt
+ (or sentences_zh_yue.txt for the "yue" virtual language).
2. Apply fastText contamination filter (optional but recommended):
remove sentences that fastText confidently (>= 0.80) assigns to one of
the top-10 internet languages but the directory label says otherwise.
@@ -44,6 +46,25 @@ Charset design decisions
ISO-8859-3 (Maltese — ħ, ġ, ċ, ż not representable in any Windows charset)
Distinct Cyrillic variants retained:
KOI8-R, KOI8-U, IBM855, IBM866, x-mac-cyrillic
+ Superset-only policy (subset encodings dropped from training):
+ Big5-HKSCS only (superset of Big5 — files labelled Big5 decode safely as HKSCS)
+ GB18030 only (superset of GBK and GB2312 — byte patterns are identical for
+ the common character subset; the model cannot distinguish them)
+ Shift_JIS trained via cp932 codec (Windows-31J superset — Java's Shift_JIS
+ resolves to CP932 internally; training on strict JIS X 0208
+ would miss NEC/IBM extension characters in real Windows files)
+ windows-874 replaces TIS-620 (superset; Thai chars 0xA1–0xFB are identical,
+ Charset.forName("TIS-620") not guaranteed on all JVMs)
+ Handled structurally (not by the ML model):
+ UTF-16-LE/BE, UTF-32-LE/BE — WideUnicodeDetector intercepts these before the
+ model runs; the feature extractor's high-byte filter makes
+ the model nearly blind to them anyway (dominant 0x00 bytes)
+ US-ASCII, ISO-2022-JP/KR/CN — structural gates in MojibusterEncodingDetector
+ HZ-GB-2312 — dropped: Java has no HZ Charset and returning a wrong decode
+ charset (GB18030) would silently corrupt content
+ Not supported (no Python codec):
+ EUC-TW — JDK-only extension, vanishingly rare in practice (superseded by
+ Big5/UTF-8). Falls through to ICU4J/Universal as fallback.
Encoding quality gates
----------------------
@@ -54,17 +75,16 @@ Encoding quality gates
High-byte ratio: encoded chunk must have enough bytes >= 0x80 to be
discriminative (thresholds vary by encoding family).
-Flores Traditional Chinese (Big5, Big5-HKSCS, EUC-TW)
--------------------------------------------------------
- MADLAD-400 does not include Cantonese (yue) or Min Nan (nan) in Han script.
- Flores-200 provides ~2000 sentences each of yue_Hant and zho_Hant.
- ALL Flores sentences go to train. Devtest and test are in-sample copies
- of train for these charsets.
+Traditional Chinese (Big5, Big5-HKSCS)
+------------------------------------------------
+ Sourced from the Cantonese Wikipedia dump (zh_yuewiki), extracted to
+ ~/datasets/zh_yuewiki/sentences_zh_yue.txt (~940K sentences).
+ The "yue" virtual language in LANG_CHARSETS maps to Big5 and Big5-HKSCS.
+ All three splits (train/devtest/test) are proper held-out partitions —
+ no in-sample evaluation.
- *** WARNING: Big5 / Big5-HKSCS / EUC-TW devtest and test accuracy numbers
- are IN-SAMPLE (trained and evaluated on the same Flores sentences).
- They are NOT held-out and should NOT be compared to other charsets.
- The manifest flags these with "eval_is_in_sample": true. ***
+ EUC-TW is omitted: Python has no euc_tw codec (JDK-only extension) and
+ the encoding is vanishingly rare in practice, superseded by Big5/UTF-8.
Output structure
----------------
@@ -75,8 +95,8 @@ Output structure
manifest.json
Caps (configurable):
- --train-cap 50000 encoded samples per charset
- --devtest-cap 5000 encoded samples per charset
+ --train-cap 20000 encoded samples per charset
+ --devtest-cap 2000 encoded samples per charset
--test-cap 5000 encoded samples per charset
Usage
@@ -85,7 +105,7 @@ Usage
python build_charset_training.py \\
--madlad-dir ~/datasets/madlad/data \\
- --flores-dir ~/datasets/flores-200/flores200_dataset \\
+ --zh-yue-file ~/datasets/zh_yuewiki/sentences_zh_yue.txt \\
--output-dir ~/datasets/madlad/charset-detect2 \\
--fasttext-model ~/datasets/madlad/lid.176.bin
"""
@@ -101,6 +121,17 @@ import unicodedata
from collections import defaultdict
from pathlib import Path
+# ---------------------------------------------------------------------------
+# Optional ebcdic dependency — registers cp420 (IBM420) and other EBCDIC
+# codecs not available in the standard CPython codec library.
+# Install with: pip install ebcdic
+# ---------------------------------------------------------------------------
+try:
+ import ebcdic as _ebcdic # noqa: F401 — import for side-effect (codec registration)
+ EBCDIC_AVAILABLE = True
+except ImportError:
+ EBCDIC_AVAILABLE = False
+
# ---------------------------------------------------------------------------
# Optional fastText dependency
# ---------------------------------------------------------------------------
@@ -118,24 +149,36 @@ except ImportError:
CHARSET_CODEC: dict[str, str] = {
"UTF-8": "utf-8",
+ # UTF-16/32 variants without BOM — detected by WideUnicodeDetector before
+ # the ML model runs. Included as eval-only (STRUCTURAL_ONLY_CHARSETS) so
+ # EvalCharsetDetectors can measure full-pipeline coverage.
"UTF-16-LE": "utf-16-le",
"UTF-16-BE": "utf-16-be",
"UTF-32-LE": "utf-32-le",
"UTF-32-BE": "utf-32-be",
"US-ASCII": "ascii",
- "Shift_JIS": "shift-jis",
+ # CP932 (Windows-31J) is trained under the IANA name Shift_JIS because
+ # Java's Charset.forName("Shift_JIS") resolves to CP932 internally.
+ # Training on Python's strict "shift-jis" codec would miss the NEC/IBM
+ # extension characters (0x87–0x9F, 0xE0–0xEA lead-byte ranges) that real
+ # Windows Shift-JIS files use. The superset covers everything the subset does.
+ "Shift_JIS": "cp932",
"EUC-JP": "euc-jp",
"ISO-2022-JP": "iso-2022-jp",
"EUC-KR": "euc-kr",
"ISO-2022-KR": "iso-2022-kr",
"ISO-2022-CN": "iso2022_cn",
+ # GB18030 is the superset of GBK and GB2312; training only on GB18030
+ # avoids teaching the model to distinguish encodings that share identical
+ # byte patterns for the common character subset. Files labelled GB2312 or
+ # GBK in the wild are safely decoded as GB18030.
"GB18030": "gb18030",
- "GB2312": "gb2312",
- "GBK": "gbk",
- "Big5": "big5",
+ # Big5-HKSCS is the superset of plain Big5; same rationale as GB18030.
"Big5-HKSCS": "big5hkscs",
- "EUC-TW": "euc_tw",
- "HZ": "hz",
+ # EUC-TW omitted: Python has no euc_tw codec (JDK-only extension).
+ # EUC-TW is vanishingly rare in practice, superseded by Big5/UTF-8.
+ # Files encountered in the wild will fall through to ICU4J/Universal as
+ # fallback detectors.
# ISO-8859-3 kept: no Windows equivalent (Maltese)
"ISO-8859-3": "iso-8859-3",
# Windows single-byte charsets (ISO-8859-X equivalents dropped)
@@ -154,8 +197,10 @@ CHARSET_CODEC: dict[str, str] = {
"IBM855": "cp855",
"IBM866": "cp866",
"x-mac-cyrillic": "mac_cyrillic",
- # Thai
- "TIS-620": "cp874",
+ # Thai — windows-874 (CP874) is the superset of TIS-620; Thai characters
+ # occupy 0xA1–0xFB in both. Charset.forName("TIS-620") is not guaranteed
+ # on all JVMs; windows-874 is. Training on cp874 covers all real Thai files.
+ "windows-874": "cp874",
# EBCDIC
"IBM500": "cp500",
"IBM424-ltr": "cp424",
@@ -228,36 +273,50 @@ LANG_CHARSETS: dict[str, list[str]] = {
"vie": ["windows-1258"],
# Japanese
"jpn": ["Shift_JIS", "EUC-JP", "ISO-2022-JP"],
- # Chinese simplified
- "zho": ["GB18030", "GBK", "GB2312", "HZ", "ISO-2022-CN"],
+ # Chinese simplified — also used for UTF-16/32 eval (CJK text has few null
+ # bytes in UTF-16 and reveals WideUnicodeDetector's null-pattern detection)
+ "zho": ["GB18030", "ISO-2022-CN",
+ "UTF-16-LE", "UTF-16-BE", "UTF-32-LE", "UTF-32-BE"],
# Korean
"kor": ["EUC-KR", "ISO-2022-KR"],
# Thai
- "tha": ["TIS-620"],
+ "tha": ["windows-874"],
+ # Traditional Chinese — sourced from Cantonese Wikipedia (zh_yuewiki),
+ # not MADLAD (which has no Cantonese/Traditional Chinese in Han script).
+ # "yue" is a virtual language key handled by load_plaintext_sentences().
+ # Big5-HKSCS only (superset of plain Big5).
+ "yue": ["Big5-HKSCS"],
}
-# UTF-* added for all MADLAD languages below
-UNICODE_CHARSETS = ["UTF-8", "UTF-16-LE", "UTF-16-BE", "UTF-32-LE", "UTF-32-BE"]
+# UTF-8 is added for all MADLAD languages.
+# UTF-16-LE/BE and UTF-32-LE/BE are intentionally EXCLUDED:
+# - WideUnicodeDetector in tika-core intercepts them via structural checks
+# (null-column, variety-ratio, block-prefix range) before the ML model runs.
+# - The feature extractor's high-byte filter (skip bytes < 0x80) makes the
+# model nearly blind to UTF-16/32 anyway — Latin UTF-16 is dominated by
+# 0x00 bytes, which are all below 0x80 and produce no features.
+# - Including them as trained classes wastes model capacity for inputs that
+# will never reach the model in production.
+UNICODE_CHARSETS = ["UTF-8"]
# These charsets produce zero high bytes (all content < 0x80), so the ML
# feature extractor sees no signal. They are detected by structural gates
# in MlEncodingDetector before the model is ever called:
-# - US-ASCII → checkAscii() → returns UTF-8
-# - HZ → checkHz() → returns HZ
-# - ISO-2022* → detectIso2022() → returns ISO-2022-JP/KR/CN
+# - US-ASCII → checkAscii() → returns UTF-8
+# - ISO-2022* → detectIso2022() → returns ISO-2022-JP/KR/CN
#
# We still generate devtest/test files so EvalCharsetDetectors can measure
# full-pipeline accuracy (with structural gates enabled), but we skip train
# files so these classes don't pollute the ML model with zero-feature samples.
STRUCTURAL_ONLY_CHARSETS = frozenset({
- "US-ASCII", "HZ", "ISO-2022-JP", "ISO-2022-KR", "ISO-2022-CN",
+ "US-ASCII", "ISO-2022-JP", "ISO-2022-KR", "ISO-2022-CN",
+ "UTF-16-LE", "UTF-16-BE", "UTF-32-LE", "UTF-32-BE",
})
-# Charsets sourced from Flores Traditional Chinese.
-# MADLAD-400 has no Cantonese/Min Nan in Han script.
-# *** devtest and test for these charsets are IN-SAMPLE copies of train. ***
-FLORES_CHARSETS = ["Big5", "Big5-HKSCS", "EUC-TW"]
-FLORES_LANGS = ["yue_Hant", "zho_Hant"] # file basenames (without extension)
+# Virtual language key used in LANG_CHARSETS for Traditional Chinese.
+# Sentences are loaded from --zh-yue-file (Cantonese Wikipedia) rather than
+# a MADLAD per-language directory.
+YUE_LANG = "yue"
# ---------------------------------------------------------------------------
# fastText contamination filter
@@ -327,11 +386,11 @@ def load_fasttext_model(model_path: Path):
_HIGH_BYTE_EXEMPT = frozenset({
"ascii", "utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be",
- "iso-2022-jp", "iso-2022-kr", "iso2022_cn", "hz",
+ "iso-2022-jp", "iso-2022-kr", "iso2022_cn",
})
_HIGH_BYTE_CJK = frozenset({
- "big5", "big5hkscs", "euc-jp", "euc-kr", "euc_tw",
- "gb18030", "gbk", "gb2312", "shift-jis",
+ "big5hkscs", "euc-jp", "euc-kr",
+ "gb18030", "cp932",
})
MIN_HIGH_BYTE_CJK = 0.20
MIN_HIGH_BYTE_UTF8 = 0.05
@@ -344,6 +403,51 @@ RTL_CHARSETS = {"IBM424-rtl", "IBM420-rtl"}
# the drop-count comparison.
NFD_CHARSETS = {"windows-1258"}
+# IBM420 (Arabic EBCDIC) encodes only the 28 basic Arabic letters plus a
+# small set of punctuation, digits, and presentation forms. Modern Arabic
+# Unicode text (especially from web corpora like MADLAD) contains:
+# • Harakat / tashkeel (U+064B–U+065F, U+0610–U+061A, etc.) — not in IBM420
+# • إ (U+0625, alef with hamza below) and ٱ (U+0671, alef wasla) — not in IBM420
+# Mainframe Arabic text was written without combining diacritics; stripping
+# them and normalising the two missing alef forms produces valid training data.
+# The drop-count is measured against the pre-processed text so MAX_DROPPED_CHARS
+# still catches genuine failures (emoji, unsupported Latin-extended chars, etc.).
+STRIP_ARABIC_DIACRITICS_CHARSETS = {"IBM420-ltr", "IBM420-rtl"}
+
+# Arabic combining mark codepoint ranges to strip for IBM420:
+# U+0610–U+061A Arabic extended combining (salat, etc.)
+# U+064B–U+065F Harakat (fatha, damma, kasra, shadda, sukun, tanwin, etc.)
+# U+06D6–U+06E4 Quranic annotation signs (combining)
+# U+06E7–U+06E8 Combining above / below
+# U+06EA–U+06ED Combining letters
+_ARABIC_DIACRITIC_RANGES = (
+ (0x0610, 0x061A),
+ (0x064B, 0x065F),
+ (0x06D6, 0x06E4),
+ (0x06E7, 0x06E8),
+ (0x06EA, 0x06ED),
+)
+
+# IBM420 only encodes the 28 basic Arabic letters plus a small set of
+# punctuation/digits. Several common Unicode Arabic letters are absent:
+# U+0625 إ ARABIC LETTER ALEF WITH HAMZA BELOW → ا (U+0627)
+# U+0671 ٱ ARABIC LETTER ALEF WASLA → ا (U+0627)
+# These are normalised to their plain alef base before encoding, matching
+# the simplified Arabic orthography used on mainframes.
+_IBM420_ALEF_MAP: dict[str, str] = {
+ "\u0625": "\u0627", # إ → ا
+ "\u0671": "\u0627", # ٱ → ا
+}
+
+
+def _prepare_for_ibm420(text: str) -> str:
+ """Strip Arabic diacritics and normalise alef variants for IBM420."""
+ text = "".join(
+ c for c in text
+ if not any(lo <= ord(c) <= hi for lo, hi in _ARABIC_DIACRITIC_RANGES)
+ )
+ return "".join(_IBM420_ALEF_MAP.get(c, c) for c in text)
+
# Maximum number of characters that may be silently dropped during encoding
# (unencodable chars removed by errors='ignore') plus corrupt sequences
# (U+FFFD produced on decode) before a sentence is rejected.
@@ -374,6 +478,15 @@ def encode_chunk(text: str, charset: str, codec: str,
if charset in RTL_CHARSETS:
text = text[::-1]
+ # IBM420 (Arabic EBCDIC): strip harakat and normalise alef variants before
+ # encoding. Mainframe Arabic text was never written with combining
+ # diacritics, and certain hamza forms (إ U+0625, ٱ U+0671) absent from the
+ # IBM420 codepage are normalised to plain alef (ا U+0627). The drop-count
+ # is measured against the pre-processed text so MAX_DROPPED_CHARS still
+ # catches genuine encoding failures (emoji, Latin-extended, etc.).
+ if charset in STRIP_ARABIC_DIACRITICS_CHARSETS:
+ text = _prepare_for_ibm420(text)
+
# CP1258 needs NFD decomposition; compare after recomposing back to NFC
norm_text = unicodedata.normalize("NFD", text) if charset in NFD_CHARSETS else text
@@ -417,6 +530,20 @@ def encode_chunk(text: str, charset: str, codec: str,
# Sentence loading
# ---------------------------------------------------------------------------
+def _clean_madlad_text(text: str) -> str:
+ """Normalise a raw MADLAD sentence.
+
+ MADLAD stores multi-paragraph source documents as a single sentence with
+ embedded literal \\n / \\r / \\t escape sequences (two-character sequences,
+ not actual control characters). Replace them with a single space so they
+ don't become unencodable backslash characters in EBCDIC charsets (e.g.
+ IBM420, which has no backslash in its mapping) or spurious punctuation in
+ single-byte charsets that do encode backslash.
+ """
+ text = text.replace("\\n", " ").replace("\\r", "").replace("\\t", " ")
+ return " ".join(text.split()) # collapse runs of whitespace
+
+
def load_madlad_sentences(lang_dir: Path, max_sentences: int) -> list[str]:
"""Load up to max_sentences from sentences_madlad.txt (lineNum\\ttext format)."""
sentences = []
@@ -431,6 +558,7 @@ def load_madlad_sentences(lang_dir: Path, max_sentences: int) -> list[str]:
continue
tab = line.find("\t")
text = line[tab + 1:] if tab >= 0 else line
+ text = _clean_madlad_text(text)
if text:
sentences.append(text)
if len(sentences) >= max_sentences:
@@ -440,21 +568,22 @@ def load_madlad_sentences(lang_dir: Path, max_sentences: int) -> list[str]:
return sentences
-def load_flores_sentences(flores_dir: Path) -> list[str]:
- """Load all Traditional Chinese sentences from Flores dev + devtest files."""
+def load_plaintext_sentences(path: Path, max_sentences: int) -> list[str]:
+ """Load up to max_sentences from a plain text file (one sentence per line)."""
sentences = []
- for split in ("dev", "devtest"):
- for lang in FLORES_LANGS:
- ext = "dev" if split == "dev" else "devtest"
- p = flores_dir / split / f"{lang}.{ext}"
- if not p.exists():
- print(f" WARNING: Flores file not found: {p}", file=sys.stderr)
- continue
- with open(p, encoding="utf-8", errors="replace") as f:
- for line in f:
- line = line.strip()
- if line:
- sentences.append(line)
+ if not path.exists():
+ print(f" WARNING: file not found: {path}", file=sys.stderr)
+ return sentences
+ try:
+ with open(path, encoding="utf-8", errors="replace") as f:
+ for line in f:
+ text = line.strip()
+ if text:
+ sentences.append(text)
+ if len(sentences) >= max_sentences:
+ break
+ except OSError as e:
+ print(f" WARNING: could not read {path}: {e}", file=sys.stderr)
return sentences
# ---------------------------------------------------------------------------
@@ -516,7 +645,9 @@ def probe_codec(charset: str, codec: str) -> bool:
"hello".encode(codec)
return True
except LookupError:
- print(f" WARNING: codec '{codec}' for {charset} not available — skipping",
+ hint = (" (install the 'ebcdic' package: pip install ebcdic)"
+ if codec.startswith("cp4") and not EBCDIC_AVAILABLE else "")
+ print(f" WARNING: codec '{codec}' for {charset} not available — skipping{hint}",
file=sys.stderr)
return False
@@ -525,7 +656,7 @@ def probe_codec(charset: str, codec: str) -> bool:
# ---------------------------------------------------------------------------
DEFAULT_MADLAD_DIR = Path.home() / "datasets" / "madlad" / "data"
-DEFAULT_FLORES_DIR = Path.home() / "datasets" / "flores-200" / "flores200_dataset"
+DEFAULT_ZH_YUE_FILE = Path.home() / "datasets" / "zh_yuewiki" / "sentences_zh_yue.txt"
DEFAULT_OUTPUT_DIR = Path.home() / "datasets" / "madlad" / "charset-detect2"
DEFAULT_FASTTEXT = Path.home() / "datasets" / "madlad" / "lid.176.bin"
DEFAULT_MAX_SOURCE = 200_000
@@ -543,7 +674,9 @@ def main():
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--madlad-dir", type=Path, default=DEFAULT_MADLAD_DIR)
- parser.add_argument("--flores-dir", type=Path, default=DEFAULT_FLORES_DIR)
+ parser.add_argument("--zh-yue-file", type=Path, default=DEFAULT_ZH_YUE_FILE,
+ help="Plain-text Cantonese Wikipedia sentences (one per line); "
+ "used for Big5 and Big5-HKSCS")
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
parser.add_argument("--fasttext-model", type=Path, default=DEFAULT_FASTTEXT,
help="Path to lid.176.bin; omit to skip contamination filter")
@@ -572,18 +705,6 @@ def main():
else:
print("fastText filter: disabled (model not found)")
- # -----------------------------------------------------------------------
- # Step 1: Load Flores Traditional Chinese (tiny — ~4K sentences — load once)
- # -----------------------------------------------------------------------
- print("=== Loading Flores Traditional Chinese ===")
- flores_sentences = load_flores_sentences(args.flores_dir)
- rng_flores = random.Random(args.seed)
- rng_flores.shuffle(flores_sentences)
- print(f" Loaded {len(flores_sentences):,} sentences from Flores "
- f"({', '.join(FLORES_LANGS)})")
- print(" *** ALL Flores sentences go to TRAIN; devtest/test are "
- "IN-SAMPLE copies ***")
-
caps = {"train": args.train_cap, "devtest": args.devtest_cap,
"test": args.test_cap}
@@ -593,7 +714,6 @@ def main():
)
all_charsets = {cs for cs in target_charsets
if cs in CHARSET_CODEC and probe_codec(cs, CHARSET_CODEC[cs])}
- flores_charsets = {cs for cs in FLORES_CHARSETS if flores_sentences} & all_charsets
# -----------------------------------------------------------------------
# Step 2 + 3: For each charset, load only the languages it needs, encode,
@@ -610,43 +730,40 @@ def main():
for charset in sorted(all_charsets):
codec = CHARSET_CODEC[charset]
- in_sample = charset in flores_charsets
- flag = " *** IN-SAMPLE ***" if in_sample else ""
- print(f" {charset}{flag}", flush=True)
+ print(f" {charset}", flush=True)
split_counts: dict[str, int] = {}
- if in_sample:
- # Flores charsets: same sentences go to all splits (in-sample eval)
- for split in split_names:
- out_file = args.output_dir / split / f"{charset}.bin.gz"
- n = build_split_file(
- list(flores_sentences), charset, codec, out_file,
- caps[split], args.min_chunk, args.max_chunk,
- random.Random(args.seed + hash(charset + split + "enc")),
- )
- split_counts[split] = n
- print(f" {split:7s}: {n:>6,} samples")
- else:
- # MADLAD charsets: determine contributing languages
- is_unicode = charset in UNICODE_CHARSETS
- contributing = sorted(
- lang for lang, csets in LANG_CHARSETS.items()
- if is_unicode or charset in csets
- )
-
- # Load sentences for each contributing language.
- # Per-language load cap: we need roughly train_cap*6/n_langs sentences
- # total to survive encoding rejections; the 80/10/10 split means
- # train gets 80% of loaded data. Use max_source_per_lang as ceiling.
- n_langs = max(1, len(contributing))
- load_cap = min(
- args.max_source_per_lang,
- max(5_000, (args.train_cap * 10) // n_langs),
- )
-
- lang_pools: dict[str, dict[str, list[str]]] = {}
- for lang in contributing:
+ # Determine contributing languages for this charset
+ is_unicode = charset in UNICODE_CHARSETS
+ contributing = sorted(
+ lang for lang, csets in LANG_CHARSETS.items()
+ if is_unicode or charset in csets
+ )
+
+ # Load sentences for each contributing language.
+ # Per-language load cap: we need roughly train_cap*6/n_langs sentences
+ # total to survive encoding rejections; the 80/10/10 split means
+ # train gets 80% of loaded data. Use max_source_per_lang as ceiling.
+ n_langs = max(1, len(contributing))
+ load_cap = min(
+ args.max_source_per_lang,
+ max(5_000, (args.train_cap * 10) // n_langs),
+ )
+
+ lang_pools: dict[str, dict[str, list[str]]] = {}
+ for lang in contributing:
+ if lang == YUE_LANG:
+ # Traditional Chinese: load from Cantonese Wikipedia dump
+ raw = load_plaintext_sentences(args.zh_yue_file, load_cap)
+ if not raw:
+ continue
+ lang_pools[lang] = split_pool(raw, args.seed)
+ pools = lang_pools[lang]
+ print(f" load {lang} (zh_yuewiki): {len(raw):>6,} → split "
+ f"{len(pools['train']):>5,}/{len(pools['devtest']):>4,}"
+ f"/{len(pools['test']):>4,}", flush=True)
+ else:
lang_dir = args.madlad_dir / lang
if not lang_dir.is_dir():
continue
@@ -665,43 +782,42 @@ def main():
+ (f" (-{removed} filtered)" if removed else ""),
flush=True)
- structural_only = charset in STRUCTURAL_ONLY_CHARSETS
- if structural_only:
- print(f" (structural-only: skipping train, generating devtest/test only)")
+ structural_only = charset in STRUCTURAL_ONLY_CHARSETS
+ if structural_only:
+ print(f" (structural-only: skipping train, generating devtest/test only)")
- for split in split_names:
- # Skip train split for structural-only charsets: the ML model
- # sees zero features for these encodings; structural gates in
- # MlEncodingDetector handle them before the model is called.
- if structural_only and split == "train":
- split_counts[split] = 0
- continue
+ for split in split_names:
+ # Skip train split for structural-only charsets: the ML model
+ # sees zero features for these encodings; structural gates in
+ # MlEncodingDetector handle them before the model is called.
+ if structural_only and split == "train":
+ split_counts[split] = 0
+ continue
+
+ cap = caps[split]
+ per_lang = max(200, (cap * 4) // max(1, len(lang_pools)))
+ sents: list[str] = []
+ for lang, pools in lang_pools.items():
+ sents.extend(pools[split][:per_lang])
+ rng_split = random.Random(args.seed + hash(charset + split))
+ rng_split.shuffle(sents)
+
+ out_file = args.output_dir / split / f"{charset}.bin.gz"
+ n = build_split_file(
+ sents, charset, codec, out_file,
+ cap, args.min_chunk, args.max_chunk,
+ random.Random(args.seed + hash(charset + split + "enc")),
+ )
+ split_counts[split] = n
+ print(f" {split:7s}: {n:>6,} samples")
- cap = caps[split]
- per_lang = max(200, (cap * 4) // max(1, len(lang_pools)))
- sents: list[str] = []
- for lang, pools in lang_pools.items():
- sents.extend(pools[split][:per_lang])
- rng_split = random.Random(args.seed + hash(charset + split))
- rng_split.shuffle(sents)
-
- out_file = args.output_dir / split / f"{charset}.bin.gz"
- n = build_split_file(
- sents, charset, codec, out_file,
- cap, args.min_chunk, args.max_chunk,
- random.Random(args.seed + hash(charset + split + "enc")),
- )
- split_counts[split] = n
- print(f" {split:7s}: {n:>6,} samples")
-
- # Release the sentence pools for this charset before moving on
- del lang_pools
+ # Release the sentence pools for this charset before moving on
+ del lang_pools
manifest[charset] = {
- "codec": codec,
- "eval_is_in_sample": in_sample,
- "structural_only": charset in STRUCTURAL_ONLY_CHARSETS,
- "samples": split_counts,
+ "codec": codec,
+ "structural_only": charset in STRUCTURAL_ONLY_CHARSETS,
+ "samples": split_counts,
}
# -----------------------------------------------------------------------
@@ -718,11 +834,6 @@ def main():
print(f" train: {total['train']:>8,} total samples")
print(f" devtest: {total['devtest']:>8,} total samples")
print(f" test: {total['test']:>8,} total samples")
- in_sample_charsets = [cs for cs, v in manifest.items()
- if v["eval_is_in_sample"]]
- if in_sample_charsets:
- print(f"\n *** IN-SAMPLE eval charsets (Big5 family from Flores): "
- f"{', '.join(sorted(in_sample_charsets))} ***")
print(f" Manifest: {manifest_path}")
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml
index 84e030b293..23260d78eb 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml
@@ -30,6 +30,12 @@
<name>Apache Tika html parser module</name>
<dependencies>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-encoding-detector-html</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
deleted file mode 100644
index f766023949..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.html;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Locale;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.tika.config.ConfigDeserializer;
-import org.apache.tika.config.JsonConfig;
-import org.apache.tika.config.TikaComponent;
-import org.apache.tika.detect.EncodingDetector;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.utils.CharsetUtils;
-
-/**
- * Character encoding detector for determining the character encoding of a
- * HTML document based on the potential charset parameter found in a
- * Content-Type http-equiv meta tag somewhere near the beginning. Especially
- * useful for determining the type among multiple closely related encodings
- * (ISO-8859-*) for which other types of encoding detection are unreliable.
- *
- * @since Apache Tika 1.2
- */
-@TikaComponent(spi = false)
-public class HtmlEncodingDetector implements EncodingDetector {
-
- // TIKA-357 - use bigger buffer for meta tag sniffing (was 4K)
- private static final int DEFAULT_MARK_LIMIT = 8192;
-
- /**
- * Configuration class for JSON deserialization.
- */
- public static class Config implements Serializable {
- private int markLimit = DEFAULT_MARK_LIMIT;
-
- public int getMarkLimit() {
- return markLimit;
- }
-
- public void setMarkLimit(int markLimit) {
- this.markLimit = markLimit;
- }
- }
- private static final Pattern HTTP_META_PATTERN =
- Pattern.compile("(?is)<\\s*meta(?:/|\\s+)([^<>]+)");
- //this should match both the older:
- //<meta http-equiv="content-type" content="text/html; charset=xyz"/>
- //and
- //html5 <meta charset="xyz">
- //See http://webdesign.about.com/od/metatags/qt/meta-charset.htm
- //for the noisiness that one might encounter in charset attrs.
- //Chose to go with strict ([-_:\\.a-z0-9]+) to match encodings
- //following http://docs.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html
- //For a more general "not" matcher, try:
- //("(?is)charset\\s*=\\s*['\\\"]?\\s*([^<>\\s'\\\";]+)")
- private static final Pattern FLEXIBLE_CHARSET_ATTR_PATTERN =
- Pattern.compile(("(?is)\\bcharset\\s*=\\s*(?:['\\\"]\\s*)?([-_:\\.a-z0-9]+)"));
- private static final Charset ASCII = Charset.forName("US-ASCII");
- /**
- * HTML can include non-iana supported charsets that Java
- * recognizes, e.g. "unicode". This can lead to incorrect detection/mojibake.
- * Ignore charsets in html meta-headers that are not supported by IANA.
- * See: TIKA-2592
- */
- private static Set<String> CHARSETS_UNSUPPORTED_BY_IANA;
-
- static {
- Set<String> unsupported = new HashSet<>();
- try (BufferedReader reader = new BufferedReader(new InputStreamReader(
- HtmlEncodingDetector.class
- .getResourceAsStream("StandardCharsets_unsupported_by_IANA.txt"),
- StandardCharsets.UTF_8))) {
- String line = reader.readLine();
- while (line != null) {
- if (line.startsWith("#")) {
- line = reader.readLine();
- continue;
- }
- line = line.trim();
- if (line.length() > 0) {
- unsupported.add(line.toLowerCase(Locale.US));
- }
- line = reader.readLine();
- }
- } catch (IOException e) {
- throw new IllegalArgumentException(
- "couldn't find StandardCharsets_unsupported_by_IANA.txt on the class path");
- }
- CHARSETS_UNSUPPORTED_BY_IANA = Collections.unmodifiableSet(unsupported);
- }
-
- private Config defaultConfig = new Config();
-
- /**
- * Default constructor for SPI loading.
- */
- public HtmlEncodingDetector() {
- }
-
- /**
- * Constructor with explicit Config object.
- *
- * @param config the configuration
- */
- public HtmlEncodingDetector(Config config) {
- this.defaultConfig = config;
- }
-
- /**
- * Constructor for JSON configuration.
- * Requires Jackson on the classpath.
- *
- * @param jsonConfig JSON configuration
- */
- public HtmlEncodingDetector(JsonConfig jsonConfig) {
- this(ConfigDeserializer.buildConfig(jsonConfig, Config.class));
- }
-
- public Charset detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException {
- if (tis == null) {
- return null;
- }
-
- // Read enough of the text stream to capture possible meta tags
- int markLimit = defaultConfig.getMarkLimit();
- tis.mark(markLimit);
- byte[] buffer = new byte[markLimit];
- int n = 0;
- int m = tis.read(buffer);
- while (m != -1 && n < buffer.length) {
- n += m;
- m = tis.read(buffer, n, buffer.length - n);
- }
- tis.reset();
-
- // Interpret the head as ASCII and try to spot a meta tag with
- // a possible character encoding hint
-
- String head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n)).toString();
- //strip out comments
- String headNoComments = head.replaceAll("<!--.*?(-->|$)", " ");
- //try to find the encoding in head without comments
- Charset charset = findCharset(headNoComments);
- //if nothing is found, back off to find any encoding
- if (charset == null) {
- return findCharset(head);
- }
- return charset;
-
- }
-
- //returns null if no charset was found
- private Charset findCharset(String s) {
-
- Matcher equiv = HTTP_META_PATTERN.matcher(s);
- Matcher charsetMatcher = FLEXIBLE_CHARSET_ATTR_PATTERN.matcher("");
- //iterate through meta tags
- while (equiv.find()) {
- String attrs = equiv.group(1);
- charsetMatcher.reset(attrs);
- //iterate through charset= and return the first match
- //that is valid
- while (charsetMatcher.find()) {
- String candCharset = charsetMatcher.group(1);
- if (CHARSETS_UNSUPPORTED_BY_IANA.contains(candCharset.toLowerCase(Locale.US))) {
- continue;
- }
- if ("x-user-defined".equalsIgnoreCase(candCharset)) {
- candCharset = "windows-1252";
- }
-
- if (CharsetUtils.isSupported(candCharset)) {
- try {
- return CharsetUtils.forName(candCharset);
- } catch (IllegalArgumentException e) {
- //ignore
- }
- }
- }
- }
- return null;
- }
-
- public Config getDefaultConfig() {
- return defaultConfig;
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java
index ac93a8f731..4e45a8723e 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Set;
import javax.xml.XMLConstants;
@@ -48,6 +49,7 @@ import org.apache.tika.config.ConfigDeserializer;
import org.apache.tika.config.JsonConfig;
import org.apache.tika.config.TikaComponent;
import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.EncodingResult;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -158,8 +160,9 @@ public class JSoupParser extends AbstractEncodingDetectorParser {
ParseContext context) throws IOException, SAXException, TikaException {
EncodingDetector encodingDetector = getEncodingDetector(context);
- Charset charset = encodingDetector.detect(tis, metadata, context);
- charset = charset == null ? DEFAULT_CHARSET : charset;
+ List<EncodingResult> encResults = encodingDetector.detect(tis, metadata, context);
+ Charset charset = encResults.isEmpty() ? DEFAULT_CHARSET
+ : encResults.get(0).getCharset();
String previous = metadata.get(Metadata.CONTENT_TYPE);
MediaType contentType = null;
if (previous == null || previous.startsWith("text/html")) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/CharsetAliases.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/CharsetAliases.java
deleted file mode 100644
index fa055bda23..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/CharsetAliases.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.html.charsetdetector;
-
-
-import java.nio.charset.Charset;
-import java.nio.charset.IllegalCharsetNameException;
-import java.nio.charset.StandardCharsets;
-import java.nio.charset.UnsupportedCharsetException;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-import org.apache.tika.parser.html.charsetdetector.charsets.ReplacementCharset;
-import org.apache.tika.parser.html.charsetdetector.charsets.XUserDefinedCharset;
-
-/**
- * Singleton class that associates standard charset names to java charset implementations
- * https://encoding.spec.whatwg.org/#ref-for-iso-8859-8-i
- */
-final class CharsetAliases {
-
- private static final Map<String, Charset> charsetsByLabel = new HashMap<>();
-
- private CharsetAliases() {
- }
-
- /**
- * @param label a charset name
- * @return the corresponding java charset, if there is one. Otherwise, null
- */
- static Charset getCharsetByLabel(String label) {
- if (label == null) {
- return null;
- }
- synchronized (charsetsByLabel) {
- // Lazy initialization
- if (charsetsByLabel.isEmpty()) {
- addAll();
- }
- }
- label = label.trim().toLowerCase(Locale.US);
- return charsetsByLabel.get(label);
- }
-
- private static void addAll() {
- addCharset(charset("Big5"), "big5", "big5-hkscs", "cn-big5", "csbig5", "x-x-big5");
- addCharset(charset("EUC-JP"), "cseucpkdfmtjapanese", "euc-jp", "x-euc-jp");
- addCharset(charset("EUC-KR"), "cseuckr", "csksc56011987", "euc-kr", "iso-ir-149", "korean",
- "ks_c_5601-1987", "ks_c_5601-1989", "ksc5601", "ksc_5601", "windows-949");
- addCharset(charset("GBK"), "chinese", "csgb2312", "csiso58gb231280", "gb2312", "gb_2312",
- "gb_2312-80", "gbk", "iso-ir-58", "x-gbk");
- addCharset(charset("IBM866"), "866", "cp866", "csibm866", "ibm866");
- addCharset(charset("ISO-2022-JP"), "csiso2022jp", "iso-2022-jp");
- addCharset(charset("ISO-8859-10", "ISO-8859-4"), "csisolatin6", "iso-8859-10", "iso-ir-157",
- "iso8859-10", "iso885910", "l6", "latin6");
- addCharset(charset("ISO-8859-13"), "iso-8859-13", "iso8859-13", "iso885913");
- addCharset(charset("ISO-8859-14", "ISO-8859-1"), "iso-8859-14", "iso8859-14", "iso885914");
- addCharset(charset("ISO-8859-15"), "csisolatin9", "iso-8859-15", "iso8859-15", "iso885915",
- "iso_8859-15", "l9");
- addCharset(charset("ISO-8859-16", "ISO-8859-1"), "iso-8859-16");
- addCharset(charset("ISO-8859-2"), "csisolatin2", "iso-8859-2", "iso-ir-101", "iso8859-2",
- "iso88592", "iso_8859-2", "iso_8859-2:1987", "l2", "latin2");
- addCharset(charset("ISO-8859-3"), "csisolatin3", "iso-8859-3", "iso-ir-109", "iso8859-3",
- "iso88593", "iso_8859-3", "iso_8859-3:1988", "l3", "latin3");
- addCharset(charset("ISO-8859-4"), "csisolatin4", "iso-8859-4", "iso-ir-110", "iso8859-4",
- "iso88594", "iso_8859-4", "iso_8859-4:1988", "l4", "latin4");
- addCharset(charset("ISO-8859-5"), "csisolatincyrillic", "cyrillic", "iso-8859-5",
- "iso-ir-144", "iso8859-5", "iso88595", "iso_8859-5", "iso_8859-5:1988");
- addCharset(charset("ISO-8859-6"), "arabic", "asmo-708", "csiso88596e", "csiso88596i",
- "csisolatinarabic", "ecma-114", "iso-8859-6", "iso-8859-6-e", "iso-8859-6-i",
- "iso-ir-127", "iso8859-6", "iso88596", "iso_8859-6", "iso_8859-6:1987");
- addCharset(charset("ISO-8859-7"), "csisolatingreek", "ecma-118", "elot_928", "greek",
- "greek8", "iso-8859-7", "iso-ir-126", "iso8859-7", "iso88597", "iso_8859-7",
- "iso_8859-7:1987", "sun_eu_greek");
- // ISO-8859-8 actually should have an influence on the layout direction
- // (text should be decoded in the visual order). However, this is not implemented in tika.
- addCharset(charset("ISO-8859-8"), "csiso88598e", "csisolatinhebrew", "hebrew", "iso-8859-8",
- "iso-8859-8-e", "iso-ir-138", "iso8859-8", "iso88598", "iso_8859-8",
- "iso_8859-8:1988", "visual");
- addCharset(charset("ISO-8859-8-I", "ISO-8859-8"), "csiso88598i", "iso-8859-8-i", "logical");
- addCharset(charset("KOI8-R"), "cskoi8r", "koi", "koi8", "koi8-r", "koi8_r");
- addCharset(charset("KOI8-U"), "koi8-ru", "koi8-u");
- addCharset(charset("Shift_JIS"), "csshiftjis", "ms932", "ms_kanji", "shift-jis",
- "shift_jis", "sjis", "windows-31j", "x-sjis");
- addCharset(charset("UTF-16BE"), "utf-16be");
- addCharset(charset("UTF-16LE"), "utf-16", "utf-16le");
- addCharset(charset("UTF-8"), "unicode-1-1-utf-8", "utf-8", "utf8");
- addCharset(charset("gb18030"), "gb18030");
- addCharset(charset("windows-1250"), "cp1250", "windows-1250", "x-cp1250");
- addCharset(charset("windows-1251"), "cp1251", "windows-1251", "x-cp1251");
- addCharset(charset("windows-1252"), "ansi_x3.4-1968", "ascii", "cp1252", "cp819",
- "csisolatin1", "ibm819", "iso-8859-1", "iso-ir-100", "iso8859-1", "iso88591",
- "iso_8859-1", "iso_8859-1:1987", "l1", "latin1", "us-ascii", "windows-1252",
- "x-cp1252");
- addCharset(charset("windows-1253"), "cp1253", "windows-1253", "x-cp1253");
- addCharset(charset("windows-1254"), "cp1254", "csisolatin5", "iso-8859-9", "iso-ir-148",
- "iso8859-9", "iso88599", "iso_8859-9", "iso_8859-9:1989", "l5", "latin5",
- "windows-1254", "x-cp1254");
- addCharset(charset("windows-1255"), "cp1255", "windows-1255", "x-cp1255");
- addCharset(charset("windows-1256"), "cp1256", "windows-1256", "x-cp1256");
- addCharset(charset("windows-1257"), "cp1257", "windows-1257", "x-cp1257");
- addCharset(charset("windows-1258"), "cp1258", "windows-1258", "x-cp1258");
- addCharset(charset("windows-874"), "dos-874", "iso-8859-11", "iso8859-11", "iso885911",
- "tis-620", "windows-874");
- addCharset(charset("x-MacCyrillic"), "x-mac-cyrillic", "x-mac-ukrainian");
- addCharset(charset("x-MacRoman"), "csmacintosh", "mac", "macintosh", "x-mac-roman");
- // The "replacement" charset is a dummy charset. It is present to mitigate wrong-charset
- // attacks
- addCharset(new ReplacementCharset(), "csiso2022kr", "hz-gb-2312", "iso-2022-cn",
- "iso-2022-cn-ext", "iso-2022-kr", "replacement");
- // The x-user-defined charset is not present in java
- addCharset(new XUserDefinedCharset(), "x-user-defined");
- }
-
- /**
- * @param names jvm charset names
- * @return the first of the given charsets that exists in the current JVM,
- * or ISO_8859_1 if none exists
- */
- private static Charset charset(String... names) {
- for (String name : names) {
- try {
- return Charset.forName(name);
- } catch (IllegalCharsetNameException | UnsupportedCharsetException e) {
- /* pass */
- }
- }
- // The only single-byte charset extended charset that must be present on every Java platform
- return StandardCharsets.ISO_8859_1;
- }
-
- /**
- * @param charset name of the charset in the JVM
- * @param names standard W3C charset names
- */
- private static void addCharset(Charset charset, String... names) {
- for (String name : names) {
- charsetsByLabel.put(name, charset);
- }
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/CharsetDetectionResult.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/CharsetDetectionResult.java
deleted file mode 100644
index 2bb2246c5f..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/CharsetDetectionResult.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.html.charsetdetector;
-
-import static java.nio.charset.StandardCharsets.UTF_16BE;
-import static java.nio.charset.StandardCharsets.UTF_16LE;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.nio.charset.Charset;
-
-
-/**
- * A detection may either not find a charset, find an invalid charset, or find a valid charset
- */
-class CharsetDetectionResult {
- private boolean found = false;
- private Charset charset = null;
-
- private CharsetDetectionResult() { /* default result: not found */}
-
- static CharsetDetectionResult notFound() {
- return new CharsetDetectionResult();
- }
-
- public boolean isFound() {
- return found;
- }
-
- public void find(String charsetName) {
- this.found = true;
- charsetName = charsetName.trim();
- if ("x-user-defined".equals(charsetName)) {
- charsetName = "windows-1252";
- }
- this.charset = CharsetAliases.getCharsetByLabel(charsetName);
- // The specification states: If charset is a UTF-16 encoding, then set charset to UTF-8.
- if (UTF_16LE.equals(charset) || UTF_16BE.equals(charset)) {
- charset = UTF_8;
- }
- }
-
- public Charset getCharset() {
- // the result may be null even if found is true, in the case there is a charset specified,
- // but it is invalid
- return charset;
- }
-
- public void setCharset(Charset charset) {
- this.found = true;
- this.charset = charset;
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/MetaProcessor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/MetaProcessor.java
deleted file mode 100644
index c0a3f3737b..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/MetaProcessor.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.html.charsetdetector;
-
-import static org.apache.tika.parser.html.charsetdetector.PreScanner.getEncodingFromMeta;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-
-/**
- * A class to process the attributes of an HTML meta tag in order to extract a character set.
- * The user should repeatedly call {@link #processAttribute} on each attributes of the tag,
- * then update its current detection result with
- * {@link #updateDetectedCharset(CharsetDetectionResult)}
- * <p>
- * The algorithm implemented is meant to match the one described by the W3C here:
- * https://html.spec.whatwg.org/multipage/parsing.html#prescan-a-byte-stream-to-determine-its-encoding
- */
-class MetaProcessor {
- private Set<String> attributeNames = new HashSet<>();
- private boolean gotPragma = false;
- private Boolean needPragma = null; // needPragma can be null, true, or false
- private CharsetDetectionResult detectionResult = CharsetDetectionResult.notFound();
-
- void updateDetectedCharset(CharsetDetectionResult currentDetectionResult) {
- if (detectionResult.isFound() && needPragma != null && !(needPragma && !gotPragma)) {
- currentDetectionResult.setCharset(detectionResult.getCharset());
- }
- }
-
- void processAttribute(Map.Entry<String, String> attribute) {
- // Ignore duplicate attributes
- if (attributeNames.contains(attribute.getKey())) {
- return;
- }
-
- attributeNames.add(attribute.getKey());
-
- // Handle charset-related attributes
- switch (attribute.getKey()) {
- case "http-equiv":
- if (attribute.getValue().equals("content-type")) {
- gotPragma = true;
- }
- break;
- case "content":
- String charsetName = getEncodingFromMeta(attribute.getValue());
- if (!detectionResult.isFound() && charsetName != null) {
- detectionResult.find(charsetName);
- needPragma = true;
- }
- break;
- case "charset":
- detectionResult.find(attribute.getValue());
- needPragma = false;
- break;
- default: // Ignore non-charset related attributes
- }
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/PreScanner.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/PreScanner.java
deleted file mode 100644
index c1b69ab2b5..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/PreScanner.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.html.charsetdetector;
-
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.AbstractMap;
-import java.util.BitSet;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * A scanner meant to detect charset meta tags in a byte stream
- * See: https://html.spec.whatwg.org/multipage/parsing.html#prescan-a-byte-stream-to-determine-its-encoding
- */
-class PreScanner {
-
- private static final Pattern CHARSET_PATTERN =
- Pattern.compile("charset\\s*=\\s*([\"']?)([^\"'\\s;]+)\\1");
- private static final byte[] COMMENT_START = {(byte) '<', (byte) '!', (byte) '-', (byte) '-'};
- private static final byte[] COMMENT_END = {(byte) '-', (byte) '-', (byte) '>'};
- private static final byte[] META_TAG_START =
- {(byte) '<', (byte) 'm', (byte) 'e', (byte) 't', (byte) 'a'};
- private static final byte SLASH = (byte) '/';
- private static final byte EQUAL = (byte) '=';
- private static final byte TAG_START = (byte) '<';
- private static final byte TAG_END = (byte) '>';
- private static final BitSet QUOTE = bitSet('"', '\'');
-
- private static final BitSet WHITESPACE = bitSet(0x09, 0x0A, 0x0C, 0x0D, 0x0D, 0x20);
- private static final BitSet SPACE_OR_TAG_END = bitSet(WHITESPACE, TAG_END);
- private static final BitSet SPACE_OR_SLASH = bitSet(WHITESPACE, SLASH);
- private static final BitSet SPECIAL_TAGS = bitSet('!', '/', '?');
-
- private static final byte[] UTF8_BOM = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF};
- private static final byte[] UTF16_BE_BOM = {(byte) 0xFE, (byte) 0xFF};
- private static final byte[] UTF16_LE_BOM = {(byte) 0xFF, (byte) 0xFE};
- private static final byte LOWER_A = (byte) 'a';
- private static final byte LOWER_Z = (byte) 'z';
- private static final byte UPPER_A = (byte) 'A';
- private static final byte UPPER_Z = (byte) 'Z';
- private BufferedInputStream stream;
- private CharsetDetectionResult detectedCharset = CharsetDetectionResult.notFound();
-
- PreScanner(InputStream inputStream) {
- this.stream = new BufferedInputStream(inputStream);
- }
-
- private static BitSet bitSet(int... bs) {
- BitSet bitSet = new BitSet(0xFF);
- for (int b : bs) bitSet.set(b);
- return bitSet;
- }
-
- private static BitSet bitSet(BitSet base, int... bs) {
- BitSet bitSet = (BitSet) base.clone();
- for (int b : bs) bitSet.set(b);
- return bitSet;
- }
-
- static String getEncodingFromMeta(String attributeValue) {
- Matcher matcher = CHARSET_PATTERN.matcher(attributeValue);
- if (!matcher.find()) {
- return null;
- }
- return matcher.group(2);
- }
-
- private static boolean contains(BitSet bitSet, byte b) {
- return bitSet.get(b & 0xFF);
- }
-
- Charset scan() {
- while (processAtLeastOneByte()) {
- if (detectedCharset.isFound()) {
- return detectedCharset.getCharset();
- }
- }
- return null;
- }
-
- Charset detectBOM() {
- try {
- if (expect(UTF8_BOM)) {
- return StandardCharsets.UTF_8;
- } else if (expect(UTF16_BE_BOM)) {
- return StandardCharsets.UTF_16BE;
- } else if (expect(UTF16_LE_BOM)) {
- return StandardCharsets.UTF_16LE;
- }
- } catch (IOException e) { /* stream could not be read, also return null */ }
- return null;
- }
-
- private boolean processAtLeastOneByte() {
- try {
- return processComment() || processMeta() || processTag() || processSpecialTag() ||
- processAny();
- } catch (IOException e) {
- return false;
- }
- }
-
- private boolean processAny() throws IOException {
- int read = stream.read();
- return read != -1;
- }
-
- private boolean processTag() throws IOException {
- stream.mark(3);
- if (read() == TAG_START) {
- int read = stream.read();
- if (read == SLASH) {
- read = stream.read();
- }
- if ((LOWER_A <= read && read <= LOWER_Z) || (UPPER_A <= read && read <= UPPER_Z)) {
- do {
- stream.mark(1);
- } while (!contains(SPACE_OR_TAG_END, read()));
- stream.reset();
- while (getAttribute() != null) {/* ignore the attribute*/}
- return true;
- }
- }
- stream.reset();
- return false;
- }
-
- private boolean processSpecialTag() throws IOException {
- stream.mark(2);
- if (read() == TAG_START && contains(SPECIAL_TAGS, read())) {
- skipUntil(TAG_END);
- return true;
- }
- stream.reset();
- return false;
- }
-
- private boolean processMeta() throws IOException {
- stream.mark(6); // len("<meta ") == 6
- if (readCaseInsensitive(META_TAG_START) && contains(SPACE_OR_SLASH, read())) {
- MetaProcessor metaProcessor = new MetaProcessor();
- for (Map.Entry<String, String> attribute = getAttribute(); attribute != null;
- attribute = getAttribute()) {
- metaProcessor.processAttribute(attribute);
- }
- metaProcessor.updateDetectedCharset(detectedCharset);
- return true;
- }
- stream.reset();
- return false;
- }
-
- /**
- * Read an attribute from the stream
- *
- * @return the attribute as a Map.Entry, where the key is the attribute's name and
- * the value is the attribute's value. If there is no attribute, return null
- */
- private Map.Entry<String, String> getAttribute() throws IOException {
- String name = getAttributeName();
- if (name == null) {
- return null;
- }
-
- if (!expect(EQUAL)) {
- return new AbstractMap.SimpleEntry<>(name, "");
- }
- skipAll(WHITESPACE);
-
- String value = getAttributeValue();
- return new AbstractMap.SimpleEntry<>(name, value);
- }
-
- private String getAttributeName() throws IOException {
- skipAll(SPACE_OR_SLASH);
- if (expect(TAG_END)) {
- return null;
- }
- StringBuilder name = new StringBuilder();
- while (!(peek() == EQUAL && name.length() > 0) && !(peek() == TAG_END || peek() == SLASH) &&
- !skipAll(WHITESPACE)) {
- name.append((char) getLowerCaseChar());
- }
- return name.toString();
- }
-
- private String getAttributeValue() throws IOException {
- StringBuilder value = new StringBuilder();
- stream.mark(1);
- byte quote = read();
- if (contains(QUOTE, quote)) {
- for (byte b = getLowerCaseChar(); b != quote; b = getLowerCaseChar()) {
- value.append((char) b);
- }
- } else {
- stream.reset();
- for (byte b = getLowerCaseChar(); !contains(SPACE_OR_TAG_END, b);
- b = getLowerCaseChar()) {
- value.append((char) b);
- stream.mark(1);
- }
- stream.reset(); // unread the space or tag end
- }
- return value.toString();
- }
-
- private boolean skipAll(BitSet bitSet) throws IOException {
- boolean skipped = false;
- stream.mark(1);
- for (byte read = read(); contains(bitSet, read); read = read()) {
- skipped = true;
- stream.mark(1);
- }
- stream.reset();
- return skipped;
- }
-
- private byte getLowerCaseChar() throws IOException {
- byte nextPoint = read();
- if (nextPoint >= 'A' && nextPoint <= 'Z') {
- nextPoint += 0x20; // lowercase
- }
- return nextPoint;
- }
-
- private boolean processComment() throws IOException {
- if (!expect(COMMENT_START)) {
- return false;
- }
- if (!expect(TAG_END)) {
- skipUntil(COMMENT_END);
- }
- return true;
- }
-
- private boolean expect(byte... expected) throws IOException {
- stream.mark(expected.length);
- for (byte b : expected) {
- byte read = read();
- if (read != b) {
- stream.reset();
- return false;
- }
- }
- return true;
- }
-
- private void skipUntil(byte... expected) throws IOException {
- while (!expect(expected)) {
- if (stream.read() == -1) {
- return;
- }
- }
- }
-
- private boolean readCaseInsensitive(byte... bs) throws IOException {
- for (byte b : bs)
- if (getLowerCaseChar() != b) {
- return false;
- }
- return true;
- }
-
- private byte read() throws IOException {
- int r = stream.read();
- if (r == -1) {
- throw new IOException();
- }
- return (byte) r;
- }
-
- private byte peek() throws IOException {
- stream.mark(1);
- byte b = read();
- stream.reset();
- return b;
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/StandardHtmlEncodingDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/StandardHtmlEncodingDetector.java
deleted file mode 100644
index d29cd2a279..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/StandardHtmlEncodingDetector.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.html.charsetdetector;
-
-import static org.apache.tika.parser.html.charsetdetector.CharsetAliases.getCharsetByLabel;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-
-import org.apache.commons.io.input.BoundedInputStream;
-
-import org.apache.tika.config.TikaComponent;
-import org.apache.tika.detect.EncodingDetector;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.ParseContext;
-
-/**
- * An encoding detector that tries to respect the spirit of the HTML spec
- * part 12.2.3 "The input byte stream", or at least the part that is compatible with
- * the implementation of tika.
- * <p>
- * https://html.spec.whatwg.org/multipage/parsing.html#the-input-byte-stream
- * <p>
- * If a resource was fetched over HTTP, then HTTP headers should be added to tika metadata
- * when using {@link #detect}, especially {@link Metadata#CONTENT_TYPE}, as it may contain
- * charset information.
- * <p>
- * This encoding detector may return null if no encoding is detected.
- * It is meant to be used inside a {@link org.apache.tika.detect.CompositeEncodingDetector}.
- * For instance:
- * <pre> {@code
- * EncodingDetector detector = new CompositeEncodingDetector(
- * Arrays.asList(
- * new StandardHtmlEncodingDetector(),
- * new Icu4jEncodingDetector()));
- * }</pre>
- * <p>
- */
-@TikaComponent(spi = false)
-public final class StandardHtmlEncodingDetector implements EncodingDetector {
- private static final int META_TAG_BUFFER_SIZE = 8192;
-
- private int markLimit = META_TAG_BUFFER_SIZE;
-
- /**
- * Extracts a charset from a Content-Type HTTP header.
- *
- * @param metadata parser metadata
- * @return a charset if there is one specified, or null
- */
- private static Charset charsetFromContentType(Metadata metadata) {
- String contentType = metadata.get(Metadata.CONTENT_TYPE);
- MediaType mediatype = MediaType.parse(contentType);
- if (mediatype == null) {
- return null;
- }
- String charsetLabel = mediatype.getParameters().get("charset");
- return getCharsetByLabel(charsetLabel);
- }
-
- @Override
- public Charset detect(TikaInputStream tis, Metadata metadata, ParseContext context) throws IOException {
- int limit = getMarkLimit();
- tis.mark(limit);
- // Never read more than the first META_TAG_BUFFER_SIZE bytes
- InputStream limitedStream = BoundedInputStream.builder().setInputStream(tis).setMaxCount(limit).get();
- PreScanner preScanner = new PreScanner(limitedStream);
-
- // The order of priority for detection is:
- // 1. Byte Order Mark
- Charset detectedCharset = preScanner.detectBOM();
- // 2. Transport-level information (Content-Type HTTP header)
- if (detectedCharset == null) {
- detectedCharset = charsetFromContentType(metadata);
- }
- // 3. HTML <meta> tag
- if (detectedCharset == null) {
- detectedCharset = preScanner.scan();
- }
-
- tis.reset();
- return detectedCharset;
- }
-
- public int getMarkLimit() {
- return markLimit;
- }
-
- /**
- * How far into the stream to read for charset detection.
- * Default is 8192.
- */
- public void setMarkLimit(int markLimit) {
- this.markLimit = markLimit;
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/charsets/ReplacementCharset.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/charsets/ReplacementCharset.java
deleted file mode 100644
index 32b96cf4c5..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/charsets/ReplacementCharset.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.html.charsetdetector.charsets;
-
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CoderResult;
-
-/**
- * An implementation of the standard "replacement" charset defined by the W3C.
- * See: https://encoding.spec.whatwg.org/#replacement
- */
-public class ReplacementCharset extends Charset {
-
- public ReplacementCharset() {
- super("replacement", null);
- }
-
- @Override
- public boolean contains(Charset cs) {
- return cs.equals(this);
- }
-
- public CharsetDecoder newDecoder() {
- return new CharsetDecoder(this, Float.MIN_VALUE, 1) {
- private boolean replacementErrorReturned = false;
-
- @Override
- protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
- if (in.hasRemaining() && !replacementErrorReturned) {
- replacementErrorReturned = true;
- return CoderResult.malformedForLength(in.remaining());
- }
- in.position(in.limit());
- return CoderResult.UNDERFLOW;
- }
-
- @Override
- protected void implReset() {
- replacementErrorReturned = false;
- }
- };
- }
-
- public CharsetEncoder newEncoder() {
- throw new UnsupportedOperationException("This charset does not support encoding");
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/charsets/XUserDefinedCharset.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/charsets/XUserDefinedCharset.java
deleted file mode 100644
index b550cd19a8..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/charsetdetector/charsets/XUserDefinedCharset.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.html.charsetdetector.charsets;
-
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CoderResult;
-import java.nio.charset.StandardCharsets;
-
-public class XUserDefinedCharset extends Charset {
-
- public XUserDefinedCharset() {
- super("x-user-defined", null);
- }
-
- @Override
- public boolean contains(Charset cs) {
- return cs.equals(StandardCharsets.US_ASCII);
- }
-
- public CharsetDecoder newDecoder() {
- return new CharsetDecoder(this, 1, 1) {
- @Override
- protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
- while (true) {
- if (!in.hasRemaining()) {
- return CoderResult.UNDERFLOW;
- }
- if (!out.hasRemaining()) {
- return CoderResult.OVERFLOW;
- }
- byte b = in.get();
- out.append((char) ((b >= 0) ? b : 0xF700 + (b & 0xFF)));
- }
- }
- };
- }
-
- public CharsetEncoder newEncoder() {
- throw new NotImplementedException("Encoding to x-user-defined is not implemented");
- }
-
- public static class NotImplementedException extends RuntimeException {
- public NotImplementedException(String msg) {
- super(msg);
- }
- }
-
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector
deleted file mode 100644
index 9025ca781f..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# TODO: This is a manual service file because EncodingDetectors are not re-sorted at runtime
-# (unlike Parsers/Detectors which call ServiceLoaderUtils.sortLoadedClasses()).
-# Need to implement a runtime priority mechanism for proper ordering across modules.
-org.apache.tika.parser.html.HtmlEncodingDetector
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
index ee30f75d30..6de0376f73 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
@@ -22,10 +22,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import org.apache.tika.detect.EncodingResult;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
@@ -133,7 +135,9 @@ public class HtmlEncodingDetectorTest {
private Charset detectCharset(String test) throws IOException {
Metadata metadata = new Metadata();
try (TikaInputStream tis = TikaInputStream.get(test.getBytes(StandardCharsets.UTF_8))) {
- return new HtmlEncodingDetector().detect(tis, metadata, new ParseContext());
+ List<EncodingResult> results =
+ new HtmlEncodingDetector().detect(tis, metadata, new ParseContext());
+ return results.isEmpty() ? null : results.get(0).getCharset();
}
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
index af5fd2aed3..96d5f56178 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
@@ -28,7 +28,6 @@ import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
-import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -65,6 +64,7 @@ import org.apache.tika.config.ServiceLoader;
import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.detect.AutoDetectReader;
import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.EncodingResult;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Geographic;
@@ -1252,12 +1252,8 @@ public class HtmlParserTest extends TikaTest {
public String getEncoding(EncodingDetector detector, Path p) throws IOException {
try (TikaInputStream tis = TikaInputStream.get(p)) {
- Charset charset = detector.detect(tis, new Metadata(), new ParseContext());
- if (charset == null) {
- return "NULL";
- } else {
- return charset.toString();
- }
+ List<EncodingResult> results = detector.detect(tis, new Metadata(), new ParseContext());
+ return results.isEmpty() ? "NULL" : results.get(0).getCharset().toString();
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/StandardHtmlEncodingDetectorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/StandardHtmlEncodingDetectorTest.java
index 834cbbe3ef..8106c99252 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/StandardHtmlEncodingDetectorTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/StandardHtmlEncodingDetectorTest.java
@@ -25,10 +25,12 @@ import java.io.InputStream;
import java.io.SequenceInputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.apache.tika.detect.EncodingResult;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
@@ -358,7 +360,9 @@ public class StandardHtmlEncodingDetectorTest {
private Charset detectCharset(InputStream inStream) throws IOException {
TikaInputStream tis = (inStream instanceof TikaInputStream) ?
(TikaInputStream) inStream : TikaInputStream.get(inStream);
- return new StandardHtmlEncodingDetector().detect(tis, metadata, new ParseContext());
+ List<EncodingResult> results =
+ new StandardHtmlEncodingDetector().detect(tis, metadata, new ParseContext());
+ return results.isEmpty() ? null : results.get(0).getCharset();
}
private InputStream throwAfter(String html) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
index 160e9f1b5f..63cc9605cd 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
@@ -46,6 +46,12 @@
<artifactId>tika-parser-text-module</artifactId>
<version>${project.version}</version>
</dependency>
+ <!-- CharsetDetector/CharsetMatch moved from tika-parser-text-module here -->
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-encoding-detector-icu4j</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-parser-xml-module</artifactId>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
index 7871c0f047..552c52889c 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
@@ -62,6 +62,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
+import org.apache.tika.detect.EncodingResult;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.TikaInputStream;
@@ -814,7 +815,9 @@ public class OutlookExtractor extends AbstractPOIFSExtractor {
if (html != null && html.length() > 0) {
Charset charset = null;
try (TikaInputStream tis = TikaInputStream.get(html.getBytes(UTF_8))) {
- charset = detector.detect(tis, EMPTY_METADATA, context);
+ List<EncodingResult> encResults =
+ detector.detect(tis, EMPTY_METADATA, context);
+ charset = encResults.isEmpty() ? null : encResults.get(0).getCharset();
} catch (IOException e) {
//swallow
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/pom.xml
index 781cba608a..bf0cc796bd 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/pom.xml
@@ -40,6 +40,12 @@
<artifactId>tika-parser-text-module</artifactId>
<version>${project.version}</version>
</dependency>
+ <!-- Icu4jEncodingDetector moved from tika-parser-text-module here -->
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-encoding-detector-icu4j</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-parser-xml-module</artifactId>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
index 153574f432..d6e9d8d863 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
@@ -31,6 +31,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.config.TikaComponent;
import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.EncodingResult;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -138,7 +139,9 @@ public class DBFParser implements Parser {
if (bytes.length > 20) {
EncodingDetector detector = new Icu4jEncodingDetector();
try (TikaInputStream tis = TikaInputStream.get(bytes)) {
- charset = detector.detect(TikaInputStream.get(bytes), new Metadata(), parseContext);
+ List<EncodingResult> results =
+ detector.detect(TikaInputStream.get(bytes), new Metadata(), parseContext);
+ charset = results.isEmpty() ? null : results.get(0).getCharset();
}
}
return charset;
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
index 661b69035d..0f33218e76 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
@@ -45,6 +45,7 @@ import org.apache.tika.config.ConfigDeserializer;
import org.apache.tika.config.JsonConfig;
import org.apache.tika.config.TikaComponent;
import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.EncodingResult;
import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
@@ -102,8 +103,6 @@ public class ZipParser extends AbstractArchiveParser {
private static final Set<MediaType> SUPPORTED_TYPES = MediaType.set(ZIP, JAR);
- private static final int MIN_BYTES_FOR_DETECTING_CHARSET = 100;
-
/**
* Maximum number of entries to record in integrity check metadata fields.
* Prevents excessive metadata in ZIPs with many discrepancies.
@@ -532,17 +531,11 @@ public class ZipParser extends AbstractArchiveParser {
// If charset detection is enabled, try to detect and decode
if (config.isDetectCharsetsInEntryNames()) {
byte[] entryName = entry.getRawName();
- byte[] extendedEntryName = entryName;
- if (0 < entryName.length && entryName.length < MIN_BYTES_FOR_DETECTING_CHARSET) {
- int len = entryName.length * (MIN_BYTES_FOR_DETECTING_CHARSET / entryName.length);
- extendedEntryName = new byte[len];
- for (int i = 0; i < len; i++) {
- extendedEntryName[i] = entryName[i % entryName.length];
- }
- }
- try (TikaInputStream detectStream = TikaInputStream.get(extendedEntryName)) {
- Charset candidate = getEncodingDetector().detect(detectStream, parentMetadata, context);
+ try (TikaInputStream detectStream = TikaInputStream.get(entryName)) {
+ List<EncodingResult> encResults =
+ getEncodingDetector().detect(detectStream, parentMetadata, context);
+ Charset candidate = encResults.isEmpty() ? null : encResults.get(0).getCharset();
if (candidate != null) {
return new String(entry.getRawName(), candidate);
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
index 4527981f6d..1e6958270d 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/pom.xml
@@ -30,10 +30,6 @@
<name>Apache Tika text parser module</name>
<dependencies>
- <dependency>
- <groupId>com.github.albfernandez</groupId>
- <artifactId>juniversalchardet</artifactId>
- </dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
@@ -74,7 +70,6 @@
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<inputExcludes>
- <inputExclude>src/main/java/org/apache/tika/parser/txt/Charset*.java</inputExclude>
<inputExclude>src/test/resources/test-documents/**</inputExclude>
</inputExcludes>
</configuration>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/BOMDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/BOMDetector.java
deleted file mode 100644
index a952724dbd..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/BOMDetector.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.txt;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.charset.UnsupportedCharsetException;
-
-import org.apache.commons.io.ByteOrderMark;
-import org.apache.commons.io.IOUtils;
-
-import org.apache.tika.config.TikaComponent;
-import org.apache.tika.detect.EncodingDetector;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-
-/*
- This is an optional detector that is not loaded automatically by SPI. You need to configure it.
- */
-@TikaComponent(spi = false)
-public class BOMDetector implements EncodingDetector {
-
- private static final ByteOrderMark[] BOMS =
- //order matters -- have to try the 32 before the 16
- new ByteOrderMark[] {
- ByteOrderMark.UTF_8,
- ByteOrderMark.UTF_32BE,
- ByteOrderMark.UTF_32LE,
- ByteOrderMark.UTF_16BE,
- ByteOrderMark.UTF_16LE
- };
- private static final Charset[] CHARSETS = new Charset[BOMS.length];
-
- private static final int MIN_BYTES = 2;
- private static final int MAX_BYTES = 4;
-
- static {
- for (int i = 0; i < BOMS.length; i++) {
- try {
- CHARSETS[i] = Charset.forName(BOMS[i].getCharsetName());
- } catch (UnsupportedCharsetException e) {
- //log it
- }
- }
- }
- @Override
- public Charset detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException {
- tis.mark(MAX_BYTES);
- byte[] bytes = new byte[MAX_BYTES];
- try {
- int numRead = IOUtils.read(tis, bytes);
- if (numRead < MIN_BYTES) {
- return null;
- } else if (numRead < MAX_BYTES) {
- //s
- byte[] tmpBytes = new byte[numRead];
- System.arraycopy(bytes, 0, tmpBytes, 0, numRead);
- bytes = tmpBytes;
- }
- } finally {
- tis.reset();
- }
- for (int i = 0; i < BOMS.length; i++) {
- ByteOrderMark bom = BOMS[i];
- if (startsWith(bom, bytes)) {
- return CHARSETS[i];
- }
- }
- return null;
- }
-
- private boolean startsWith(ByteOrderMark bom, byte[] bytes) {
- byte[] bomBytes = bom.getBytes();
- if (bytes.length < bomBytes.length) {
- return false;
- }
- for (int i = 0; i < bomBytes.length; i++) {
- if (bomBytes[i] != bytes[i]) {
- return false;
- }
- }
- return true;
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java
deleted file mode 100644
index 591ccdfb0f..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java
+++ /dev/null
@@ -1,604 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/**
- * ******************************************************************************
- * Copyright (C) 2005-2016, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- * ******************************************************************************
- */
-package org.apache.tika.parser.txt;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Reader;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-
-
-/**
- * <code>CharsetDetector</code> provides a facility for detecting the
- * charset or encoding of character data in an unknown format.
- * The input data can either be from an input stream or an array of bytes.
- * The result of the detection operation is a list of possibly matching
- * charsets, or, for simple use, you can just ask for a Java Reader that
- * will will work over the input data.
- * <p>
- * Character set detection is at best an imprecise operation. The detection
- * process will attempt to identify the charset that best matches the characteristics
- * of the byte data, but the process is partly statistical in nature, and
- * the results can not be guaranteed to always be correct.
- * <p>
- * For best accuracy in charset detection, the input data should be primarily
- * in a single language, and a minimum of a few hundred bytes worth of plain text
- * in the language are needed. The detection process will attempt to
- * ignore html or xml style markup that could otherwise obscure the content.
- * <p>
- *
- * @stable ICU 3.4
- */
-public class CharsetDetector {
-
-// Question: Should we have getters corresponding to the setters for input text
-// and declared encoding?
-
-// A thought: If we were to create our own type of Java Reader, we could defer
-// figuring out an actual charset for data that starts out with too much English
-// only ASCII until the user actually read through to something that didn't look
-// like 7 bit English. If nothing else ever appeared, we would never need to
-// actually choose the "real" charset. All assuming that the application just
-// wants the data, and doesn't care about a char set name.
-
- static final int DEFAULT_MARK_LIMIT = 12000; //This is a Tika modification; ICU's is 8000
- private static final int MAX_CONFIDENCE = 100;
- /*
- * List of recognizers for all charsets known to the implementation.
- */
- private static final List<CSRecognizerInfo> ALL_CS_RECOGNIZERS;
-
- static {
- List<CSRecognizerInfo> list = new ArrayList<>();
-
- list.add(new CSRecognizerInfo(new CharsetRecog_UTF8(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_Unicode.CharsetRecog_UTF_16_BE(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_Unicode.CharsetRecog_UTF_16_LE(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_Unicode.CharsetRecog_UTF_32_BE(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_Unicode.CharsetRecog_UTF_32_LE(), true));
-
- list.add(new CSRecognizerInfo(new CharsetRecog_mbcs.CharsetRecog_sjis(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_2022.CharsetRecog_2022JP(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_2022.CharsetRecog_2022CN(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_2022.CharsetRecog_2022KR(), true));
- list.add(
- new CSRecognizerInfo(new CharsetRecog_mbcs.CharsetRecog_euc.CharsetRecog_gb_18030(),
- true));
- list.add(new CSRecognizerInfo(new CharsetRecog_mbcs.CharsetRecog_euc.CharsetRecog_euc_jp(),
- true));
- list.add(new CSRecognizerInfo(new CharsetRecog_mbcs.CharsetRecog_euc.CharsetRecog_euc_kr(),
- true));
- list.add(new CSRecognizerInfo(new CharsetRecog_mbcs.CharsetRecog_big5(), true));
-
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_8859_1(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_8859_2(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_8859_5_ru(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_8859_6_ar(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_8859_7_el(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_8859_8_I_he(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_8859_8_he(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_windows_1251(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_windows_1256(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_KOI8_R(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_8859_9_tr(), true));
-
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_EBCDIC_500_de(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_EBCDIC_500_en(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_EBCDIC_500_es(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_EBCDIC_500_fr(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_EBCDIC_500_it(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_EBCDIC_500_nl(), true));
-
- // IBM 420/424 recognizers are disabled by default
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_IBM424_he_rtl(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_IBM424_he_ltr(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_IBM420_ar_rtl(), true));
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_IBM420_ar_ltr(), true));
-
- list.add(new CSRecognizerInfo(new CharsetRecog_sbcs.CharsetRecog_IBM866_ru(), true));
- ALL_CS_RECOGNIZERS = Collections.unmodifiableList(list);
- }
-
- /*
- * The following items are accessed by individual CharsetRecongizers during
- * the recognition process
- *
- */
- final byte[] fInputBytes; // The text to be checked. Markup will have been
- // removed if appropriate.
- private final int kBufSize;
- int fInputLen; // Length of the byte data in fInputBytes.
- short[] fByteStats = // byte frequency statistics for the input text.
- new short[256]; // Value is percent, not absolute.
- boolean fC1Bytes = // True if any bytes in the range 0x80 - 0x9F are in the input;
- false;
- String fDeclaredEncoding;
- byte[] fRawInput; // Original, untouched input bytes.
- // If user gave us a byte array, this is it.
- // If user gave us a stream, it's read to a
- // buffer here.
- int fRawLength; // Length of data in fRawInput array.
- InputStream fInputStream; // User's input stream, or null if the user
- //
- // Stuff private to CharsetDetector
- //
- private boolean fStripTags = // If true, setText() will strip tags from input text.
- false;
- private boolean[] fEnabledRecognizers; // If not null, active set of charset recognizers had
-
- /**
- * Constructor
- *
- * @stable ICU 3.4
- */
- public CharsetDetector() {
- this(DEFAULT_MARK_LIMIT);
- }
-
- public CharsetDetector(int markLimit) {
- kBufSize = markLimit;
- fInputBytes = new byte[kBufSize];
- }
-
- /**
- * Get the names of all charsets supported by <code>CharsetDetector</code> class.
- * <p>
- * <b>Note:</b> Multiple different charset encodings in a same family may use
- * a single shared name in this implementation. For example, this method returns
- * an array including "ISO-8859-1" (ISO Latin 1), but not including "windows-1252"
- * (Windows Latin 1). However, actual detection result could be "windows-1252"
- * when the input data matches Latin 1 code points with any points only available
- * in "windows-1252".
- *
- * @return an array of the names of all charsets supported by
- * <code>CharsetDetector</code> class.
- * @stable ICU 3.4
- */
- public static String[] getAllDetectableCharsets() {
- String[] allCharsetNames = new String[ALL_CS_RECOGNIZERS.size()];
- for (int i = 0; i < allCharsetNames.length; i++) {
- allCharsetNames[i] = ALL_CS_RECOGNIZERS.get(i).recognizer.getName();
- }
- return allCharsetNames;
- }
-
- /**
- * Set the declared encoding for charset detection.
- * The declared encoding of an input text is an encoding obtained
- * from an http header or xml declaration or similar source that
- * can be provided as additional information to the charset detector.
- * A match between a declared encoding and a possible detected encoding
- * will raise the quality of that detected encoding by a small delta,
- * and will also appear as a "reason" for the match.
- * <p>
- * A declared encoding that is incompatible with the input data being
- * analyzed will not be added to the list of possible encodings.
- *
- * @param encoding The declared encoding
- * @return This CharsetDetector
- * @stable ICU 3.4
- */
- public CharsetDetector setDeclaredEncoding(String encoding) {
- setCanonicalDeclaredEncoding(encoding);
- return this;
- }
- // Value is rounded up, so zero really means zero occurences.
-
- /**
- * Set the input text (byte) data whose charset is to be detected.
- *
- * @param in the input text of unknown encoding
- * @return This CharsetDetector
- * @stable ICU 3.4
- */
- public CharsetDetector setText(byte[] in) {
- return setText(in, in.length);
- }
-
- private CharsetDetector setText(byte[] in, int length) {
- fRawInput = in;
- fRawLength = length;
-
- MungeInput();
- return this;
- }
-
- /**
- * Set the input text (byte) data whose charset is to be detected.
- * <p>
- * The input stream that supplies the character data must have markSupported()
- * == true; the charset detection process will read a small amount of data,
- * then return the stream to its original position via
- * the InputStream.reset() operation. The exact amount that will
- * be read depends on the characteristics of the data itself.
- *
- * @param in the input text of unknown encoding
- * @return This CharsetDetector
- * @throws java.io.IOException if there is an error while reading.
- * @stable ICU 3.4
- */
-
- public CharsetDetector setText(InputStream in) throws IOException {
- fInputStream = in;
- fInputStream.mark(kBufSize);
- byte[] inputBytes = new byte[kBufSize]; // Always make a new buffer because the
- // previous one may have come from the caller,
- // in which case we can't touch it.
- long bytesRead = -1;
- try {
- bytesRead = IOUtils.read(fInputStream, inputBytes);
- if (bytesRead >= Integer.MAX_VALUE) {
- throw new IOException("Can't have read > Integer.MAX_VALUE bytes");
- }
- } finally {
- fInputStream.reset();
- }
-
- if (bytesRead < 1) {
- return setText(new byte[0]);
- } else if (kBufSize > bytesRead) {
- return setText(inputBytes, (int) bytesRead);
- } else {
- return setText(inputBytes);
- }
- }
-
- /**
- * Return the charset that best matches the supplied input data.
- * <p>
- * Note though, that because the detection
- * only looks at the start of the input data,
- * there is a possibility that the returned charset will fail to handle
- * the full set of input data.
- *
- * @return a CharsetMatch object representing the best matching charset, or
- * <code>null</code> if there are no matches.
- * @stable ICU 3.4
- */
- public CharsetMatch detect() {
-// TODO: A better implementation would be to copy the detect loop from
-// detectAll(), and cut it short as soon as a match with a high confidence
-// is found. This is something to be done later, after things are otherwise
-// working.
- CharsetMatch[] matches = detectAll();
-
- if (matches == null || matches.length == 0) {
- return null;
- }
-
- return matches[0];
- }
-
- /**
- * Return an array of all charsets that appear to be plausible
- * matches with the input data. The array is ordered with the
- * best quality match first.
- *
- * @return An array of CharsetMatch objects representing possibly matching charsets.
- * @stable ICU 3.4
- */
- public CharsetMatch[] detectAll() {
- CharsetRecognizer csr;
- int i;
- CharsetMatch charsetMatch;
- int confidence;
- ArrayList<CharsetMatch> matches = new ArrayList<>();
- // Iterate over all possible charsets, remember all that
- // give a match quality > 0.
- for (i = 0; i < ALL_CS_RECOGNIZERS.size(); i++) {
- csr = ALL_CS_RECOGNIZERS.get(i).recognizer;
- charsetMatch = csr.match(this);
- if (charsetMatch != null) {
- confidence = charsetMatch.getConfidence() & 0x000000ff;
- if (confidence > 0) {
- // Just to be safe, constrain
- confidence = Math.min(confidence, MAX_CONFIDENCE);
-
- // Apply charset hint.
- if ((fDeclaredEncoding != null) &&
- (fDeclaredEncoding.equalsIgnoreCase(csr.getName()))) {
- // Reduce lack of confidence (delta between "sure" and current) by 50%.
- confidence += (MAX_CONFIDENCE - confidence) / 2;
- }
- CharsetMatch m = new CharsetMatch(this, csr, confidence, charsetMatch.getName(),
- charsetMatch.getLanguage());
- matches.add(m);
- }
- }
- }
- Collections.sort(matches); // CharsetMatch compares on confidence
- Collections.reverse(matches); // Put best match first.
- return matches.toArray(CharsetMatch[]::new);
- }
-
- /**
- * Autodetect the charset of an inputStream, and return a Java Reader
- * to access the converted input data.
- * <p>
- * This is a convenience method that is equivalent to
- * <code>this.setDeclaredEncoding(declaredEncoding).setText(in).detect().getReader();</code>
- * <p>
- * For the input stream that supplies the character data, markSupported()
- * must be true; the charset detection will read a small amount of data,
- * then return the stream to its original position via
- * the InputStream.reset() operation. The exact amount that will
- * be read depends on the characteristics of the data itself.
- *
- * @param in The source of the byte data in the unknown charset.
- * @param declaredEncoding A declared encoding for the data, if available,
- * or null or an empty string if none is available.
- *
- * @return an appropriate Java Reader or null if no charsets appear to match the input data or
- * if an error occurred.
- *
- * @stable ICU 3.4
- */
- public Reader getReader(InputStream in, String declaredEncoding) {
- fDeclaredEncoding = declaredEncoding;
-
- try {
- setText(in);
-
- CharsetMatch match = detect();
-
- if (match == null) {
- return null;
- }
-
- return match.getReader();
- } catch (IOException e) {
- return null;
- }
- }
- // gave us a byte array.
-
- /**
- * Autodetect the charset of an inputStream, and return a String
- * containing the converted input data.
- * <p>
- * This is a convenience method that is equivalent to
- * <code>this.setDeclaredEncoding(declaredEncoding).setText(in).detect().getString();</code>
- *
- * @param in The source of the byte data in the unknown charset.
- * @param declaredEncoding A declared encoding for the data, if available,
- * or null or an empty string if none is available.
- *
- * @return a String containing the converted input data, or null if no charsets appear to match
- * the input data or if an error occurred.
- *
- * @stable ICU 3.4
- */
- public String getString(byte[] in, String declaredEncoding) {
- fDeclaredEncoding = declaredEncoding;
-
- try {
- setText(in);
-
- CharsetMatch match = detect();
-
- if (match == null) {
- return null;
- }
-
- return match.getString(-1);
- } catch (IOException e) {
- return null;
- }
- }
-
- /**
- * Test whether or not input filtering is enabled.
- *
- * @return <code>true</code> if input text will be filtered.
- * @stable ICU 3.4
- * @see #enableInputFilter
- */
- public boolean inputFilterEnabled() {
- return fStripTags;
- }
- // been changed from the default. The array index is
- // corresponding to ALL_RECOGNIZER. See setDetectableCharset().
-
- /**
- * Enable filtering of input text. If filtering is enabled,
- * text within angle brackets ("<" and ">") will be removed
- * before detection.
- *
- * @param filter <code>true</code> to enable input text filtering.
- * @return The previous setting.
- * @stable ICU 3.4
- */
- public boolean enableInputFilter(boolean filter) {
- boolean previous = fStripTags;
-
- fStripTags = filter;
-
- return previous;
- }
-
- /**
- * Try to set fDeclaredEncoding to the canonical name for <encoding>, if it exists.
- *
- * @param encoding - name of character encoding
- */
- private void setCanonicalDeclaredEncoding(String encoding) {
- if ((encoding == null) || encoding.isEmpty()) {
- return;
- }
-
- Charset cs = Charset.forName(encoding);
- if (cs != null) {
- fDeclaredEncoding = cs.name();
- }
- }
-
- /*
- * MungeInput - after getting a set of raw input data to be analyzed, preprocess
- * it by removing what appears to be html markup.
- */
- private void MungeInput() {
- int srci = 0;
- int dsti = 0;
- byte b;
- boolean inMarkup = false;
- int openTags = 0;
- int badTags = 0;
-
- //
- // html / xml markup stripping.
- // quick and dirty, not 100% accurate, but hopefully good enough, statistically.
- // discard everything within < brackets >
- // Count how many total '<' and illegal (nested) '<' occur, so we can make some
- // guess as to whether the input was actually marked up at all.
- if (fStripTags) {
- for (srci = 0; srci < fRawLength && dsti < fInputBytes.length; srci++) {
- b = fRawInput[srci];
- if (b == (byte) '<') {
- if (inMarkup) {
- badTags++;
- }
- inMarkup = true;
- openTags++;
- }
-
- if (!inMarkup) {
- fInputBytes[dsti++] = b;
- }
-
- if (b == (byte) '>') {
- inMarkup = false;
- }
- }
-
- fInputLen = dsti;
- }
-
- //
- // If it looks like this input wasn't marked up, or if it looks like it's
- // essentially nothing but markup abandon the markup stripping.
- // Detection will have to work on the unstripped input.
- //
- if (openTags < 5 || openTags / 5 < badTags || (fInputLen < 100 && fRawLength > 600)) {
- int limit = fRawLength;
-
- if (limit > kBufSize) {
- limit = kBufSize;
- }
-
- for (srci = 0; srci < limit; srci++) {
- fInputBytes[srci] = fRawInput[srci];
- }
- fInputLen = srci;
- }
-
- //
- // Tally up the byte occurence statistics.
- // These are available for use by the various detectors.
- //
- Arrays.fill(fByteStats, (short) 0);
- for (srci = 0; srci < fInputLen; srci++) {
- int val = fInputBytes[srci] & 0x00ff;
- fByteStats[val]++;
- }
-
- fC1Bytes = false;
- for (int i = 0x80; i <= 0x9F; i += 1) {
- if (fByteStats[i] != 0) {
- fC1Bytes = true;
- break;
- }
- }
- }
-
- /**
- * Get the names of charsets that can be recognized by this CharsetDetector instance.
- *
- * @return an array of the names of charsets that can be recognized by this CharsetDetector
- * instance.
- * @internal
- * @deprecated This API is ICU internal only.
- */
- @Deprecated
- public String[] getDetectableCharsets() {
- List<String> csnames = new ArrayList<>(ALL_CS_RECOGNIZERS.size());
- for (int i = 0; i < ALL_CS_RECOGNIZERS.size(); i++) {
- CSRecognizerInfo rcinfo = ALL_CS_RECOGNIZERS.get(i);
- boolean active = (fEnabledRecognizers == null) ? rcinfo.isDefaultEnabled :
- fEnabledRecognizers[i];
- if (active) {
- csnames.add(rcinfo.recognizer.getName());
- }
- }
- return csnames.toArray(String[]::new);
- }
-
- /**
- * Enable or disable individual charset encoding.
- * A name of charset encoding must be included in the names returned by
- * {@link #getAllDetectableCharsets()}.
- *
- * @param encoding the name of charset encoding.
- * @param enabled <code>true</code> to enable, or <code>false</code> to disable the
- * charset encoding.
- * @return A reference to this <code>CharsetDetector</code>.
- * @throws IllegalArgumentException when the name of charset encoding is
- * not supported.
- * @internal
- * @deprecated This API is ICU internal only.
- */
- @Deprecated
- public CharsetDetector setDetectableCharset(String encoding, boolean enabled) {
- int modIdx = -1;
- boolean isDefaultVal = false;
- for (int i = 0; i < ALL_CS_RECOGNIZERS.size(); i++) {
- CSRecognizerInfo csrinfo = ALL_CS_RECOGNIZERS.get(i);
- if (csrinfo.recognizer.getName().equals(encoding)) {
- modIdx = i;
- isDefaultVal = (csrinfo.isDefaultEnabled == enabled);
- break;
- }
- }
- if (modIdx < 0) {
- // No matching encoding found
- throw new IllegalArgumentException("Invalid encoding: " + "\"" + encoding + "\"");
- }
-
- if (fEnabledRecognizers == null && !isDefaultVal) {
- // Create an array storing the non default setting
- fEnabledRecognizers = new boolean[ALL_CS_RECOGNIZERS.size()];
-
- // Initialize the array with default info
- for (int i = 0; i < ALL_CS_RECOGNIZERS.size(); i++) {
- fEnabledRecognizers[i] = ALL_CS_RECOGNIZERS.get(i).isDefaultEnabled;
- }
- }
-
- if (fEnabledRecognizers != null) {
- fEnabledRecognizers[modIdx] = enabled;
- }
-
- return this;
- }
-
- private static class CSRecognizerInfo {
- CharsetRecognizer recognizer;
- boolean isDefaultEnabled;
-
- CSRecognizerInfo(CharsetRecognizer recognizer, boolean isDefaultEnabled) {
- this.recognizer = recognizer;
- this.isDefaultEnabled = isDefaultEnabled;
- }
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetMatch.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetMatch.java
deleted file mode 100644
index beb56f3943..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetMatch.java
+++ /dev/null
@@ -1,291 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/**
- * ******************************************************************************
- * Copyright (C) 2005-2016, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- * ******************************************************************************
- */
-package org.apache.tika.parser.txt;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-
-import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
-
-/**
- * This class represents a charset that has been identified by a CharsetDetector
- * as a possible encoding for a set of input data. From an instance of this
- * class, you can ask for a confidence level in the charset identification,
- * or for Java Reader or String to access the original byte data in Unicode form.
- * <p>
- * Instances of this class are created only by CharsetDetectors.
- * <p>
- * Note: this class has a natural ordering that is inconsistent with equals.
- * The natural ordering is based on the match confidence value.
- *
- * @stable ICU 3.4
- */
-public class CharsetMatch implements Comparable<CharsetMatch> {
-
-
- //
- // Private Data
- //
- private int fConfidence;
- private byte[] fRawInput = null; // Original, untouched input bytes.
- // If user gave us a byte array, this is it.
- private int fRawLength; // Length of data in fRawInput array.
- private InputStream fInputStream = null; // User's input stream, or null if the user
- private String fCharsetName; // The name of the charset this CharsetMatch
- // represents. Filled in by the recognizer.
- private String fLang; // The language, if one was determined by
-
- /*
- * Constructor. Implementation internal
- */
- CharsetMatch(CharsetDetector det, CharsetRecognizer rec, int conf) {
- fConfidence = conf;
-
- // The references to the original application input data must be copied out
- // of the charset recognizer to here, in case the application resets the
- // recognizer before using this CharsetMatch.
- if (det.fInputStream == null) {
- // We only want the existing input byte data if it came straight from the user,
- // not if is just the head of a stream.
- fRawInput = det.fRawInput;
- fRawLength = det.fRawLength;
- }
- fInputStream = det.fInputStream;
- fCharsetName = rec.getName();
- fLang = rec.getLanguage();
- }
-
- /*
- * Constructor. Implementation internal
- */
- CharsetMatch(CharsetDetector det, CharsetRecognizer rec, int conf, String csName, String lang) {
- fConfidence = conf;
-
- // The references to the original application input data must be copied out
- // of the charset recognizer to here, in case the application resets the
- // recognizer before using this CharsetMatch.
- if (det.fInputStream == null) {
- // We only want the existing input byte data if it came straight from the user,
- // not if is just the head of a stream.
- fRawInput = det.fRawInput;
- fRawLength = det.fRawLength;
- }
- fInputStream = det.fInputStream;
- fCharsetName = csName;
- fLang = lang;
- }
-
- /**
- * Create a java.io.Reader for reading the Unicode character data corresponding
- * to the original byte data supplied to the Charset detect operation.
- * <p>
- * CAUTION: if the source of the byte data was an InputStream, a Reader
- * can be created for only one matching char set using this method. If more
- * than one charset needs to be tried, the caller will need to reset
- * the InputStream and create InputStreamReaders itself, based on the charset name.
- *
- * @return the Reader for the Unicode character data.
- * @stable ICU 3.4
- */
- public Reader getReader() {
- InputStream inputStream = fInputStream;
-
- if (inputStream == null) {
- try {
- inputStream = UnsynchronizedByteArrayInputStream.builder().setByteArray(fRawInput).setLength(fRawLength).get();
- }
- catch (IOException ex) {
- return null;
- }
- }
-
- try {
- inputStream.reset();
- return new InputStreamReader(inputStream, getName());
- } catch (IOException e) {
- return null;
- }
- }
-
- /**
- * Create a Java String from Unicode character data corresponding
- * to the original byte data supplied to the Charset detect operation.
- *
- * @return a String created from the converted input data.
- * @throws IOException
- * @stable ICU 3.4
- */
- public String getString() throws IOException {
- return getString(-1);
-
- }
-
- /**
- * Create a Java String from Unicode character data corresponding
- * to the original byte data supplied to the Charset detect operation.
- * The length of the returned string is limited to the specified size;
- * the string will be trunctated to this length if necessary. A limit value of
- * zero or less is ignored, and treated as no limit.
- *
- * @param maxLength The maximium length of the String to be created when the
- * source of the data is an input stream, or -1 for
- * unlimited length.
- * @return a String created from the converted input data.
- * @throws IOException
- * @stable ICU 3.4
- */
- public String getString(int maxLength) throws IOException {
- String result = null;
- if (fInputStream != null) {
- StringBuilder sb = new StringBuilder();
- char[] buffer = new char[1024];
- try (Reader reader = getReader()) {
- int max = maxLength < 0 ? Integer.MAX_VALUE : maxLength;
- int bytesRead;
-
- while ((bytesRead = reader.read(buffer, 0, Math.min(max, 1024))) > 0) {
- sb.append(buffer, 0, bytesRead);
- max -= bytesRead;
- }
- }
-
- return sb.toString();
- } else {
- String name = getNormalizedName();
- result = new String(fRawInput, name);
- }
- return result;
-
- }
-
- /**
- * strips e.g. _rtl, _ltr off of charset names so that they can be used as a charset.
- *
- * @return
- */
- public String getNormalizedName() {
- String name = getName();
- /*
- * getName() may return a name with a suffix 'rtl' or 'ltr'. This cannot
- * be used to open a charset (e.g. IBM424_rtl). The ending '_rtl' or '_ltr'
- * should be stripped off before creating the string.
- */
- int startSuffix =
- !name.contains("_rtl") ? name.indexOf("_ltr") : name.indexOf("_rtl");
- if (startSuffix > 0) {
- name = name.substring(0, startSuffix);
- }
- return name;
- }
-
- /**
- * Get an indication of the confidence in the charset detected.
- * Confidence values range from 0-100, with larger numbers indicating
- * a better match of the input data to the characteristics of the
- * charset.
- *
- * @return the confidence in the charset match
- * @stable ICU 3.4
- */
- public int getConfidence() {
- return fConfidence;
- }
-
- /**
- * Get the name of the detected charset.
- * The name will be one that can be used with other APIs on the
- * platform that accept charset names. It is the "Canonical name"
- * as defined by the class java.nio.charset.Charset; for
- * charsets that are registered with the IANA charset registry,
- * this is the MIME-preferred registerd name.
- *
- * @return The name of the charset.
- * @stable ICU 3.4
- * @see java.nio.charset.Charset
- * @see java.io.InputStreamReader
- */
- public String getName() {
- return fCharsetName;
- }
- // gave us a byte array.
-
- /**
- * Get the ISO code for the language of the detected charset.
- *
- * @return The ISO code for the language or <code>null</code> if the
- * language cannot be determined.
- * @stable ICU 3.4
- */
- public String getLanguage() {
- return fLang;
- }
-
- /**
- * Compare to other CharsetMatch objects.
- * Comparison is based on the match confidence value, which
- * allows CharsetDetector.detectAll() to order its results.
- *
- * @param other the CharsetMatch object to compare against.
- * @return a negative integer, zero, or a positive integer as the
- * confidence level of this CharsetMatch
- * is less than, equal to, or greater than that of
- * the argument.
- * @throws ClassCastException if the argument is not a CharsetMatch.
- * @stable ICU 4.4
- */
- @Override
- public int compareTo(CharsetMatch other) {
- int compareResult = 0;
- if (this.fConfidence > other.fConfidence) {
- compareResult = 1;
- } else if (this.fConfidence < other.fConfidence) {
- compareResult = -1;
- }
- return compareResult;
- }
-
- /**
- * compare this CharsetMatch to another based on confidence value
- *
- * @param o the CharsetMatch object to compare against
- * @return true if equal
- */
- @Override
- public boolean equals(Object o) {
- if (o instanceof CharsetMatch) {
- CharsetMatch that = (CharsetMatch) o;
- return (this.fConfidence == that.fConfidence);
- }
-
- return false;
- }
-
- /**
- * generates a hashCode based on the confidence value
- *
- * @return the hashCode
- */
- @Override
- public int hashCode() {
- return fConfidence;
- }
- // gave us a byte array.
-
- @Override
- public String toString() {
- String s = "Match of " + fCharsetName;
- if (getLanguage() != null) {
- s += " in " + getLanguage();
- }
- s += " with confidence " + fConfidence;
- return s;
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_2022.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_2022.java
deleted file mode 100644
index 32043abc52..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_2022.java
+++ /dev/null
@@ -1,159 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/*
- *******************************************************************************
- * Copyright (C) 2005 - 2012, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- *******************************************************************************
- */
-package org.apache.tika.parser.txt;
-
-/**
- * class CharsetRecog_2022 part of the ICU charset detection imlementation.
- * This is a superclass for the individual detectors for
- * each of the detectable members of the ISO 2022 family
- * of encodings.
- * <p>
- * The separate classes are nested within this class.
- */
-abstract class CharsetRecog_2022 extends CharsetRecognizer {
-
-
- /**
- * Matching function shared among the 2022 detectors JP, CN and KR
- * Counts up the number of legal an unrecognized escape sequences in
- * the sample of text, and computes a score based on the total number &
- * the proportion that fit the encoding.
- *
- * @param text the byte buffer containing text to analyse
- * @param textLen the size of the text in the byte.
- * @param escapeSequences the byte escape sequences to test for.
- * @return match quality, in the range of 0-100.
- */
- int match(byte[] text, int textLen, byte[][] escapeSequences) {
- int i, j;
- int escN;
- int hits = 0;
- int misses = 0;
- int shifts = 0;
- int quality;
- scanInput:
- for (i = 0; i < textLen; i++) {
- if (text[i] == 0x1b) {
- checkEscapes:
- for (escN = 0; escN < escapeSequences.length; escN++) {
- byte[] seq = escapeSequences[escN];
-
- if ((textLen - i) < seq.length) {
- continue checkEscapes;
- }
-
- for (j = 1; j < seq.length; j++) {
- if (seq[j] != text[i + j]) {
- continue checkEscapes;
- }
- }
-
- hits++;
- i += seq.length - 1;
- continue scanInput;
- }
-
- misses++;
- }
-
- if (text[i] == 0x0e || text[i] == 0x0f) {
- // Shift in/out
- shifts++;
- }
- }
-
- if (hits == 0) {
- return 0;
- }
-
- //
- // Initial quality is based on relative proportion of recongized vs.
- // unrecognized escape sequences.
- // All good: quality = 100;
- // half or less good: quality = 0;
- // linear inbetween.
- quality = (100 * hits - 100 * misses) / (hits + misses);
-
- // Back off quality if there were too few escape sequences seen.
- // Include shifts in this computation, so that KR does not get penalized
- // for having only a single Escape sequence, but many shifts.
- if (hits + shifts < 5) {
- quality -= (5 - (hits + shifts)) * 10;
- }
-
- if (quality < 0) {
- quality = 0;
- }
- return quality;
- }
-
-
- static class CharsetRecog_2022JP extends CharsetRecog_2022 {
- private byte[][] escapeSequences = {{0x1b, 0x24, 0x28, 0x43}, // KS X 1001:1992
- {0x1b, 0x24, 0x28, 0x44}, // JIS X 212-1990
- {0x1b, 0x24, 0x40}, // JIS C 6226-1978
- {0x1b, 0x24, 0x41}, // GB 2312-80
- {0x1b, 0x24, 0x42}, // JIS X 208-1983
- {0x1b, 0x26, 0x40}, // JIS X 208 1990, 1997
- {0x1b, 0x28, 0x42}, // ASCII
- {0x1b, 0x28, 0x48}, // JIS-Roman
- {0x1b, 0x28, 0x49}, // Half-width katakana
- {0x1b, 0x28, 0x4a}, // JIS-Roman
- {0x1b, 0x2e, 0x41}, // ISO 8859-1
- {0x1b, 0x2e, 0x46} // ISO 8859-7
- };
-
- String getName() {
- return "ISO-2022-JP";
- }
-
- CharsetMatch match(CharsetDetector det) {
- int confidence = match(det.fInputBytes, det.fInputLen, escapeSequences);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- static class CharsetRecog_2022KR extends CharsetRecog_2022 {
- private byte[][] escapeSequences = {{0x1b, 0x24, 0x29, 0x43}};
-
- String getName() {
- return "ISO-2022-KR";
- }
-
- CharsetMatch match(CharsetDetector det) {
- int confidence = match(det.fInputBytes, det.fInputLen, escapeSequences);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- static class CharsetRecog_2022CN extends CharsetRecog_2022 {
- private byte[][] escapeSequences = {{0x1b, 0x24, 0x29, 0x41}, // GB 2312-80
- {0x1b, 0x24, 0x29, 0x47}, // CNS 11643-1992 Plane 1
- {0x1b, 0x24, 0x2A, 0x48}, // CNS 11643-1992 Plane 2
- {0x1b, 0x24, 0x29, 0x45}, // ISO-IR-165
- {0x1b, 0x24, 0x2B, 0x49}, // CNS 11643-1992 Plane 3
- {0x1b, 0x24, 0x2B, 0x4A}, // CNS 11643-1992 Plane 4
- {0x1b, 0x24, 0x2B, 0x4B}, // CNS 11643-1992 Plane 5
- {0x1b, 0x24, 0x2B, 0x4C}, // CNS 11643-1992 Plane 6
- {0x1b, 0x24, 0x2B, 0x4D}, // CNS 11643-1992 Plane 7
- {0x1b, 0x4e}, // SS2
- {0x1b, 0x4f}, // SS3
- };
-
- String getName() {
- return "ISO-2022-CN";
- }
-
- CharsetMatch match(CharsetDetector det) {
- int confidence = match(det.fInputBytes, det.fInputLen, escapeSequences);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_UTF8.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_UTF8.java
deleted file mode 100644
index 51361ed1a2..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_UTF8.java
+++ /dev/null
@@ -1,97 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/**
- * ******************************************************************************
- * Copyright (C) 2005 - 2014, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- * ******************************************************************************
- */
-package org.apache.tika.parser.txt;
-
-/**
- * Charset recognizer for UTF-8
- */
-class CharsetRecog_UTF8 extends CharsetRecognizer {
-
- String getName() {
- return "UTF-8";
- }
-
- /* (non-Javadoc)
- * @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector)
- */
- CharsetMatch match(CharsetDetector det) {
- boolean hasBOM = false;
- int numValid = 0;
- int numInvalid = 0;
- byte[] input = det.fRawInput;
- int i;
- int trailBytes = 0;
- int confidence;
-
- if (det.fRawLength >= 3 && (input[0] & 0xFF) == 0xef && (input[1] & 0xFF) == 0xbb &&
- (input[2] & 0xFF) == 0xbf) {
- hasBOM = true;
- }
-
- // Scan for multi-byte sequences
- for (i = 0; i < det.fRawLength; i++) {
- int b = input[i];
- if ((b & 0x80) == 0) {
- continue; // ASCII
- }
-
- // Hi bit on char found. Figure out how long the sequence should be
- if ((b & 0x0e0) == 0x0c0) {
- trailBytes = 1;
- } else if ((b & 0x0f0) == 0x0e0) {
- trailBytes = 2;
- } else if ((b & 0x0f8) == 0xf0) {
- trailBytes = 3;
- } else {
- numInvalid++;
- continue;
- }
-
- // Verify that we've got the right number of trail bytes in the sequence
- for (; ; ) {
- i++;
- if (i >= det.fRawLength) {
- break;
- }
- b = input[i];
- if ((b & 0xc0) != 0x080) {
- numInvalid++;
- break;
- }
- if (--trailBytes == 0) {
- numValid++;
- break;
- }
- }
- }
-
- // Cook up some sort of confidence score, based on presense of a BOM
- // and the existence of valid and/or invalid multi-byte sequences.
- confidence = 0;
- if (hasBOM && numInvalid == 0) {
- confidence = 100;
- } else if (hasBOM && numValid > numInvalid * 10) {
- confidence = 80;
- } else if (numValid > 3 && numInvalid == 0) {
- confidence = 100;
- } else if (numValid > 0 && numInvalid == 0) {
- confidence = 80;
- } else if (numValid == 0 && numInvalid == 0) {
- // Plain ASCII. Confidence must be > 10, it's more likely than UTF-16, which
- // accepts ASCII with confidence = 10.
- // TODO: add plain ASCII as an explicitly detected type.
- confidence = 15;
- } else if (numValid > numInvalid * 10) {
- // Probably corruput utf-8 data. Valid sequences aren't likely by chance.
- confidence = 25;
- }
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_Unicode.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_Unicode.java
deleted file mode 100644
index 1d9c4504a4..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_Unicode.java
+++ /dev/null
@@ -1,186 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/*
- *******************************************************************************
- * Copyright (C) 1996-2013, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- *******************************************************************************
- *
- */
-
-package org.apache.tika.parser.txt;
-
-/**
- * This class matches UTF-16 and UTF-32, both big- and little-endian. The
- * BOM will be used if it is present.
- */
-abstract class CharsetRecog_Unicode extends CharsetRecognizer {
-
- static int codeUnit16FromBytes(byte hi, byte lo) {
- return ((hi & 0xff) << 8) | (lo & 0xff);
- }
-
- // UTF-16 confidence calculation. Very simple minded, but better than nothing.
- // Any 8 bit non-control characters bump the confidence up. These have a zero high byte,
- // and are very likely to be UTF-16, although they could also be part of a UTF-32 code.
- // NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32.
- // NULs should be rare in actual text.
- static int adjustConfidence(int codeUnit, int confidence) {
- if (codeUnit == 0) {
- confidence -= 10;
- } else if ((codeUnit >= 0x20 && codeUnit <= 0xff) || codeUnit == 0x0a) {
- confidence += 10;
- }
- if (confidence < 0) {
- confidence = 0;
- } else if (confidence > 100) {
- confidence = 100;
- }
- return confidence;
- }
-
- /* (non-Javadoc)
- * @see com.ibm.icu.text.CharsetRecognizer#getName()
- */
- abstract String getName();
-
- /* (non-Javadoc)
- * @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector)
- */
- abstract CharsetMatch match(CharsetDetector det);
-
- static class CharsetRecog_UTF_16_BE extends CharsetRecog_Unicode {
- String getName() {
- return "UTF-16BE";
- }
-
- CharsetMatch match(CharsetDetector det) {
- byte[] input = det.fRawInput;
- int confidence = 10;
-
- int bytesToCheck = Math.min(input.length, 30);
- for (int charIndex = 0; charIndex < bytesToCheck - 1; charIndex += 2) {
- int codeUnit = codeUnit16FromBytes(input[charIndex], input[charIndex + 1]);
- if (charIndex == 0 && codeUnit == 0xFEFF) {
- confidence = 100;
- break;
- }
- confidence = adjustConfidence(codeUnit, confidence);
- if (confidence == 0 || confidence == 100) {
- break;
- }
- }
- if (bytesToCheck < 4 && confidence < 100) {
- confidence = 0;
- }
- if (confidence > 0) {
- return new CharsetMatch(det, this, confidence);
- }
- return null;
- }
- }
-
- static class CharsetRecog_UTF_16_LE extends CharsetRecog_Unicode {
- String getName() {
- return "UTF-16LE";
- }
-
- CharsetMatch match(CharsetDetector det) {
- byte[] input = det.fRawInput;
- int confidence = 10;
-
- int bytesToCheck = Math.min(input.length, 30);
- for (int charIndex = 0; charIndex < bytesToCheck - 1; charIndex += 2) {
- int codeUnit = codeUnit16FromBytes(input[charIndex + 1], input[charIndex]);
- if (charIndex == 0 && codeUnit == 0xFEFF) {
- confidence = 100;
- break;
- }
- confidence = adjustConfidence(codeUnit, confidence);
- if (confidence == 0 || confidence == 100) {
- break;
- }
- }
- if (bytesToCheck < 4 && confidence < 100) {
- confidence = 0;
- }
- if (confidence > 0) {
- return new CharsetMatch(det, this, confidence);
- }
- return null;
- }
- }
-
- static abstract class CharsetRecog_UTF_32 extends CharsetRecog_Unicode {
- abstract int getChar(byte[] input, int index);
-
- abstract String getName();
-
- CharsetMatch match(CharsetDetector det) {
- byte[] input = det.fRawInput;
- int limit = (det.fRawLength / 4) * 4;
- int numValid = 0;
- int numInvalid = 0;
- boolean hasBOM = false;
- int confidence = 0;
-
- if (limit == 0) {
- return null;
- }
- if (getChar(input, 0) == 0x0000FEFF) {
- hasBOM = true;
- }
-
- for (int i = 0; i < limit; i += 4) {
- int ch = getChar(input, i);
-
- if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
- numInvalid += 1;
- } else {
- numValid += 1;
- }
- }
-
-
- // Cook up some sort of confidence score, based on presence of a BOM
- // and the existence of valid and/or invalid multi-byte sequences.
- if (hasBOM && numInvalid == 0) {
- confidence = 100;
- } else if (hasBOM && numValid > numInvalid * 10) {
- confidence = 80;
- } else if (numValid > 3 && numInvalid == 0) {
- confidence = 100;
- } else if (numValid > 0 && numInvalid == 0) {
- confidence = 80;
- } else if (numValid > numInvalid * 10) {
- // Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance.
- confidence = 25;
- }
-
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- static class CharsetRecog_UTF_32_BE extends CharsetRecog_UTF_32 {
- int getChar(byte[] input, int index) {
- return (input[index + 0] & 0xFF) << 24 | (input[index + 1] & 0xFF) << 16 |
- (input[index + 2] & 0xFF) << 8 | (input[index + 3] & 0xFF);
- }
-
- String getName() {
- return "UTF-32BE";
- }
- }
-
-
- static class CharsetRecog_UTF_32_LE extends CharsetRecog_UTF_32 {
- int getChar(byte[] input, int index) {
- return (input[index + 3] & 0xFF) << 24 | (input[index + 2] & 0xFF) << 16 |
- (input[index + 1] & 0xFF) << 8 | (input[index + 0] & 0xFF);
- }
-
- String getName() {
- return "UTF-32LE";
- }
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_mbcs.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_mbcs.java
deleted file mode 100644
index 91a690276d..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_mbcs.java
+++ /dev/null
@@ -1,536 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/*
- ****************************************************************************
- * Copyright (C) 2005-2012, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- ****************************************************************************
- *
- */
-package org.apache.tika.parser.txt;
-
-import java.util.Arrays;
-
-/**
- * CharsetRecognizer implemenation for Asian - double or multi-byte - charsets.
- * Match is determined mostly by the input data adhering to the
- * encoding scheme for the charset, and, optionally,
- * frequency-of-occurence of characters.
- * <p/>
- * Instances of this class are singletons, one per encoding
- * being recognized. They are created in the main
- * CharsetDetector class and kept in the global list of available
- * encodings to be checked. The specific encoding being recognized
- * is determined by subclass.
- */
-abstract class CharsetRecog_mbcs extends CharsetRecognizer {
-
- /**
- * Get the IANA name of this charset.
- *
- * @return the charset name.
- */
- abstract String getName();
-
-
- /**
- * Test the match of this charset with the input text data
- * which is obtained via the CharsetDetector object.
- *
- * @param det The CharsetDetector, which contains the input text
- * to be checked for being in this charset.
- * @return Two values packed into one int (Damn java, anyhow)
- * <br/>
- * bits 0-7: the match confidence, ranging from 0-100
- * <br/>
- * bits 8-15: The match reason, an enum-like value.
- */
- int match(CharsetDetector det, int[] commonChars) {
- @SuppressWarnings("unused") int singleByteCharCount = 0; //TODO Do we really need this?
- int doubleByteCharCount = 0;
- int commonCharCount = 0;
- int badCharCount = 0;
- int totalCharCount = 0;
- int confidence = 0;
- iteratedChar iter = new iteratedChar();
-
- detectBlock:
- {
- for (iter.reset(); nextChar(iter, det); ) {
- totalCharCount++;
- if (iter.error) {
- badCharCount++;
- } else {
- long cv = iter.charValue & 0xFFFFFFFFL;
-
- if (cv <= 0xff) {
- singleByteCharCount++;
- } else {
- doubleByteCharCount++;
- if (commonChars != null) {
- // NOTE: This assumes that there are no 4-byte common chars.
- if (Arrays.binarySearch(commonChars, (int) cv) >= 0) {
- commonCharCount++;
- }
- }
- }
- }
- if (badCharCount >= 2 && badCharCount * 5 >= doubleByteCharCount) {
- // Bail out early if the byte data is not matching the encoding scheme.
- break detectBlock;
- }
- }
-
- if (doubleByteCharCount <= 10 && badCharCount == 0) {
- // Not many multi-byte chars.
- if (doubleByteCharCount == 0 && totalCharCount < 10) {
- // There weren't any multibyte sequences, and there was a low density of
- // non-ASCII single bytes. We don't have enough data to have any confidence.
- // Statistical analysis of single byte non-ASCII charcters
- // would probably help here.
- confidence = 0;
- } else {
- // ASCII or ISO file? It's probably not our encoding,
- // but is not incompatible with our encoding, so don't give it a zero.
- confidence = 10;
- }
-
- break detectBlock;
- }
-
- //
- // No match if there are too many characters that don't fit the encoding scheme.
- // (should we have zero tolerance for these?)
- //
- if (doubleByteCharCount < 20 * badCharCount) {
- confidence = 0;
- break detectBlock;
- }
-
- if (commonChars == null) {
- // We have no statistics on frequently occuring characters.
- // Assess confidence purely on having a reasonable number of
- // multi-byte characters (the more the better
- confidence = 30 + doubleByteCharCount - 20 * badCharCount;
- if (confidence > 100) {
- confidence = 100;
- }
- } else {
- //
- // Frequency of occurence statistics exist.
- //
- double maxVal = Math.log((float) doubleByteCharCount / 4);
- double scaleFactor = 90.0 / maxVal;
- confidence = (int) (Math.log(commonCharCount + 1) * scaleFactor + 10);
- confidence = Math.min(confidence, 100);
- }
- } // end of detectBlock:
-
- return confidence;
- }
-
- /**
- * Get the next character (however many bytes it is) from the input data
- * Subclasses for specific charset encodings must implement this function
- * to get characters according to the rules of their encoding scheme.
- * <p>
- * This function is not a method of class iteratedChar only because
- * that would require a lot of extra derived classes, which is awkward.
- *
- * @param it The iteratedChar "struct" into which the returned char is placed.
- * @param det The charset detector, which is needed to get at the input byte data
- * being iterated over.
- * @return True if a character was returned, false at end of input.
- */
- abstract boolean nextChar(iteratedChar it, CharsetDetector det);
-
- // "Character" iterated character class.
- // Recognizers for specific mbcs encodings make their "characters" available
- // by providing a nextChar() function that fills in an instance of iteratedChar
- // with the next char from the input.
- // The returned characters are not converted to Unicode, but remain as the raw
- // bytes (concatenated into an int) from the codepage data.
- //
- // For Asian charsets, use the raw input rather than the input that has been
- // stripped of markup. Detection only considers multi-byte chars, effectively
- // stripping markup anyway, and double byte chars do occur in markup too.
- //
- static class iteratedChar {
- int charValue = 0; // 1-4 bytes from the raw input data
- int nextIndex = 0;
- boolean error = false;
- boolean done = false;
-
- void reset() {
- charValue = 0;
- nextIndex = 0;
- error = false;
- done = false;
- }
-
- int nextByte(CharsetDetector det) {
- if (nextIndex >= det.fRawLength) {
- done = true;
- return -1;
- }
- return (int) det.fRawInput[nextIndex++] & 0x00ff;
- }
- }
-
- /**
- * Shift-JIS charset recognizer.
- */
- static class CharsetRecog_sjis extends CharsetRecog_mbcs {
- static int[] commonChars =
- // TODO: This set of data comes from the character frequency-
- // of-occurence analysis tool. The data needs to be moved
- // into a resource and loaded from there.
- {0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0,
- 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, 0x82b3,
- 0x82b5, 0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, 0x82c8,
- 0x82c9, 0x82cc, 0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, 0x82ea,
- 0x82f0, 0x82f1, 0x8341, 0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362,
- 0x8367, 0x8375, 0x8376, 0x8389, 0x838a, 0x838b, 0x838d, 0x8393, 0x8e96,
- 0x93fa, 0x95aa};
-
- boolean nextChar(iteratedChar it, CharsetDetector det) {
- it.error = false;
- int firstByte;
- firstByte = it.charValue = it.nextByte(det);
- if (firstByte < 0) {
- return false;
- }
-
- if (firstByte <= 0x7f || (firstByte > 0xa0 && firstByte <= 0xdf)) {
- return true;
- }
-
- int secondByte = it.nextByte(det);
- if (secondByte < 0) {
- return false;
- }
- it.charValue = (firstByte << 8) | secondByte;
- if (!((secondByte >= 0x40 && secondByte <= 0x7f) ||
- (secondByte >= 0x80 && secondByte <= 0xff))) {
- // Illegal second byte value.
- it.error = true;
- }
- return true;
- }
-
- CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, commonChars);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
- String getName() {
- return "Shift_JIS";
- }
-
- public String getLanguage() {
- return "ja";
- }
-
-
- }
-
-
- /**
- * Big5 charset recognizer.
- */
- static class CharsetRecog_big5 extends CharsetRecog_mbcs {
- static int[] commonChars =
- // TODO: This set of data comes from the character frequency-
- // of-occurence analysis tool. The data needs to be moved
- // into a resource and loaded from there.
- {0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446,
- 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, 0xa477,
- 0xa4a3, 0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, 0xa4fd,
- 0xa540, 0xa548, 0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, 0xa662,
- 0xa668, 0xa670, 0xa6a8, 0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, 0xa6f2,
- 0xa740, 0xa751, 0xa759, 0xa7da, 0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, 0xa8d3,
- 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3, 0xaa6b, 0xaaba, 0xaabe, 0xaacc,
- 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59, 0xaec9, 0xafe0, 0xb0ea,
- 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c, 0xb5a5, 0xb5bd,
- 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44, 0xbba1,
- 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f};
-
- boolean nextChar(iteratedChar it, CharsetDetector det) {
- it.error = false;
- int firstByte;
- firstByte = it.charValue = it.nextByte(det);
- if (firstByte < 0) {
- return false;
- }
-
- if (firstByte <= 0x7f || firstByte == 0xff) {
- // single byte character.
- return true;
- }
-
- int secondByte = it.nextByte(det);
- if (secondByte < 0) {
- return false;
- }
- it.charValue = (it.charValue << 8) | secondByte;
-
- if (secondByte < 0x40 || secondByte == 0x7f || secondByte == 0xff) {
- it.error = true;
- }
- return true;
- }
-
- CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, commonChars);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
- String getName() {
- return "Big5";
- }
-
-
- public String getLanguage() {
- return "zh";
- }
- }
-
-
- /**
- * EUC charset recognizers. One abstract class that provides the common function
- * for getting the next character according to the EUC encoding scheme,
- * and nested derived classes for EUC_KR, EUC_JP, EUC_CN.
- */
- abstract static class CharsetRecog_euc extends CharsetRecog_mbcs {
-
- /*
- * (non-Javadoc)
- * Get the next character value for EUC based encodings.
- * Character "value" is simply the raw bytes that make up the character
- * packed into an int.
- */
- boolean nextChar(iteratedChar it, CharsetDetector det) {
- it.error = false;
- int firstByte = 0;
- int secondByte = 0;
- int thirdByte = 0;
- //int fourthByte = 0;
-
- buildChar:
- {
- firstByte = it.charValue = it.nextByte(det);
- if (firstByte < 0) {
- // Ran off the end of the input data
- it.done = true;
- break buildChar;
- }
- if (firstByte <= 0x8d) {
- // single byte char
- break buildChar;
- }
-
- secondByte = it.nextByte(det);
- it.charValue = (it.charValue << 8) | secondByte;
-
- if (firstByte >= 0xA1 && firstByte <= 0xfe) {
- // Two byte Char
- if (secondByte < 0xa1) {
- it.error = true;
- }
- break buildChar;
- }
- if (firstByte == 0x8e) {
- // Code Set 2.
- // In EUC-JP, total char size is 2 bytes, only one byte of actual char value.
- // In EUC-TW, total char size is 4 bytes, three bytes
- // contribute to char value.
- // We don't know which we've got.
- // Treat it like EUC-JP. If the data really was EUC-TW, the following two
- // bytes will look like a well formed 2 byte char.
- if (secondByte < 0xa1) {
- it.error = true;
- }
- break buildChar;
- }
-
- if (firstByte == 0x8f) {
- // Code set 3.
- // Three byte total char size, two bytes of actual char value.
- thirdByte = it.nextByte(det);
- it.charValue = (it.charValue << 8) | thirdByte;
- if (thirdByte < 0xa1) {
- it.error = true;
- }
- }
- }
-
- return (it.done == false);
- }
-
- /**
- * The charset recognize for EUC-JP. A singleton instance of this class
- * is created and kept by the public CharsetDetector class
- */
- static class CharsetRecog_euc_jp extends CharsetRecog_euc {
- static int[] commonChars =
- // TODO: This set of data comes from the character frequency-
- // of-occurence analysis tool. The data needs to be moved
- // into a resource and loaded from there.
- {0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2,
- 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, 0xa4b1,
- 0xa4b3, 0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, 0xa4c1,
- 0xa4c3, 0xa4c4, 0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, 0xa4ce,
- 0xa4cf, 0xa4d0, 0xa4de, 0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, 0xa4e9,
- 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef, 0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, 0xa5a4,
- 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af, 0xa5b0, 0xa5b3, 0xa5b5, 0xa5b7,
- 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7, 0xa5c8, 0xa5c9, 0xa5cb,
- 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1, 0xa5e5, 0xa5e9,
- 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee, 0xbbc8,
- 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1};
-
- String getName() {
- return "EUC-JP";
- }
-
- CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, commonChars);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
- public String getLanguage() {
- return "ja";
- }
- }
-
- /**
- * The charset recognize for EUC-KR. A singleton instance of this class
- * is created and kept by the public CharsetDetector class
- */
- static class CharsetRecog_euc_kr extends CharsetRecog_euc {
- static int[] commonChars =
- // TODO: This set of data comes from the character frequency-
- // of-occurence analysis tool. The data needs to be moved
- // into a resource and loaded from there.
- {0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc,
- 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, 0xb4cf,
- 0xb4d9, 0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, 0xb7af,
- 0xb7c2, 0xb7ce, 0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, 0xb9ab,
- 0xb9ae, 0xb9cc, 0xb9ce, 0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, 0xbbe7,
- 0xbbf3, 0xbbfd, 0xbcad, 0xbcba, 0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, 0xbdc3,
- 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee, 0xbef8, 0xbefa, 0xbfa1, 0xbfa9,
- 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7, 0xc0af, 0xc0b8, 0xc0ba,
- 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6, 0xc0da, 0xc0e5,
- 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6, 0xc1f8,
- 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad};
-
- String getName() {
- return "EUC-KR";
- }
-
- CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, commonChars);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
- public String getLanguage() {
- return "ko";
- }
- }
- }
-
- /**
- * GB-18030 recognizer. Uses simplified Chinese statistics.
- */
- static class CharsetRecog_gb_18030 extends CharsetRecog_mbcs {
-
- static int[] commonChars =
- // TODO: This set of data comes from the character frequency-
- // of-occurence analysis tool. The data needs to be moved
- // into a resource and loaded from there.
- {0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac,
- 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, 0xb5bd,
- 0xb5c4, 0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, 0xb7d6,
- 0xb7dd, 0xb8b4, 0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, 0xb9fd,
- 0xbacd, 0xbba7, 0xbbd6, 0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, 0xbdcc,
- 0xbecd, 0xbedd, 0xbfb4, 0xbfc6, 0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, 0xc2db,
- 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7, 0xc7f8, 0xc8ab, 0xc8cb, 0xc8d5,
- 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7, 0xcad0, 0xcad6, 0xcaf5,
- 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5, 0xcfb5, 0xcfc2,
- 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2, 0xd2b5,
- 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, 0xd6d0};
-
- /*
- * (non-Javadoc)
- * Get the next character value for EUC based encodings.
- * Character "value" is simply the raw bytes that make up the character
- * packed into an int.
- */
- boolean nextChar(iteratedChar it, CharsetDetector det) {
- it.error = false;
- int firstByte = 0;
- int secondByte = 0;
- int thirdByte = 0;
- int fourthByte = 0;
-
- buildChar:
- {
- firstByte = it.charValue = it.nextByte(det);
-
- if (firstByte < 0) {
- // Ran off the end of the input data
- it.done = true;
- break buildChar;
- }
-
- if (firstByte <= 0x80) {
- // single byte char
- break buildChar;
- }
-
- secondByte = it.nextByte(det);
- it.charValue = (it.charValue << 8) | secondByte;
-
- if (firstByte >= 0x81 && firstByte <= 0xFE) {
- // Two byte Char
- if ((secondByte >= 0x40 && secondByte <= 0x7E) ||
- (secondByte >= 80 && secondByte <= 0xFE)) {
- break buildChar;
- }
-
- // Four byte char
- if (secondByte >= 0x30 && secondByte <= 0x39) {
- thirdByte = it.nextByte(det);
-
- if (thirdByte >= 0x81 && thirdByte <= 0xFE) {
- fourthByte = it.nextByte(det);
-
- if (fourthByte >= 0x30 && fourthByte <= 0x39) {
- it.charValue = (it.charValue << 16) | (thirdByte << 8) | fourthByte;
- break buildChar;
- }
- }
- }
-
- it.error = true;
- break buildChar;
- }
- }
-
- return (it.done == false);
- }
-
- String getName() {
- return "GB18030";
- }
-
- CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, commonChars);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
- public String getLanguage() {
- return "zh";
- }
- }
-
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_sbcs.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_sbcs.java
deleted file mode 100644
index 033d35e73e..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecog_sbcs.java
+++ /dev/null
@@ -1,1807 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/*
- ****************************************************************************
- * Copyright (C) 2005-2013, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- ************************************************************************** *
- *
- */
-
-package org.apache.tika.parser.txt;
-
-/**
- * This class recognizes single-byte encodings. Because the encoding scheme is so
- * simple, language statistics are used to do the matching.
- * <p/>
- * The Recognizer works by first mapping from bytes in the encoding under test
- * into that Recognizer's ngram space. Normally this means performing a
- * lowercase, and excluding codepoints that don't correspond to numbers of
- * letters. (Accented letters may or may not be ignored or normalised, depending
- * on the needs of the ngrams)
- * Then, ngram analysis is run against the transformed text, and a confidence
- * is calculated.
- * <p/>
- * For many of our Recognizers, we have one ngram set per language in each
- * encoding, and do a simultanious language+charset detection.
- * <p/>
- * When adding new Recognizers, the easiest way is to byte map to an existing
- * encoding for which we have ngrams, excluding non text, and re-use the ngrams.
- *
- * @internal
- */
-abstract class CharsetRecog_sbcs extends CharsetRecognizer {
-
- /* (non-Javadoc)
- * @see com.ibm.icu.text.CharsetRecognizer#getName()
- */
- abstract String getName();
-
- int match(CharsetDetector det, int[] ngrams, byte[] byteMap) {
- return match(det, ngrams, byteMap, (byte) 0x20);
- }
-
- int match(CharsetDetector det, int[] ngrams, byte[] byteMap, byte spaceChar) {
- NGramParser parser = new NGramParser(ngrams, byteMap);
- return parser.parse(det, spaceChar);
- }
-
- int matchIBM420(CharsetDetector det, int[] ngrams, byte[] byteMap, byte spaceChar) {
- NGramParser_IBM420 parser = new NGramParser_IBM420(ngrams, byteMap);
- return parser.parse(det, spaceChar);
- }
-
- static class NGramParser {
- // private static final int N_GRAM_SIZE = 3;
- private static final int N_GRAM_MASK = 0xFFFFFF;
-
- protected int byteIndex = 0;
- protected byte[] byteMap;
- protected byte spaceChar;
- private int ngram = 0;
- private int[] ngramList;
- private int ngramCount;
- private int hitCount;
-
- public NGramParser(int[] theNgramList, byte[] theByteMap) {
- ngramList = theNgramList;
- byteMap = theByteMap;
-
- ngram = 0;
-
- ngramCount = hitCount = 0;
- }
-
- /*
- * Binary search for value in table, which must have exactly 64 entries.
- */
- private static int search(int[] table, int value) {
- int index = 0;
-
- if (table[index + 32] <= value) {
- index += 32;
- }
-
- if (table[index + 16] <= value) {
- index += 16;
- }
-
- if (table[index + 8] <= value) {
- index += 8;
- }
-
- if (table[index + 4] <= value) {
- index += 4;
- }
-
- if (table[index + 2] <= value) {
- index += 2;
- }
-
- if (table[index + 1] <= value) {
- index += 1;
- }
-
- if (table[index] > value) {
- index -= 1;
- }
-
- if (index < 0 || table[index] != value) {
- return -1;
- }
-
- return index;
- }
-
- private void lookup(int thisNgram) {
- ngramCount += 1;
-
- if (search(ngramList, thisNgram) >= 0) {
- hitCount += 1;
- }
-
- }
-
- protected void addByte(int b) {
- ngram = ((ngram << 8) + (b & 0xFF)) & N_GRAM_MASK;
- lookup(ngram);
- }
-
- private int nextByte(CharsetDetector det) {
- if (byteIndex >= det.fInputLen) {
- return -1;
- }
-
- return det.fInputBytes[byteIndex++] & 0xFF;
- }
-
- protected void parseCharacters(CharsetDetector det) {
- int b;
- boolean ignoreSpace = false;
-
- while ((b = nextByte(det)) >= 0) {
- byte mb = byteMap[b];
-
- // TODO: 0x20 might not be a space in all character sets...
- if (mb != 0) {
- if (!(mb == spaceChar && ignoreSpace)) {
- addByte(mb);
- }
-
- ignoreSpace = (mb == spaceChar);
- }
- }
-
- }
-
- public int parse(CharsetDetector det) {
- return parse(det, (byte) 0x20);
- }
-
- public int parse(CharsetDetector det, byte spaceCh) {
-
- this.spaceChar = spaceCh;
-
- parseCharacters(det);
-
- // TODO: Is this OK? The buffer could have ended in the middle of a word...
- addByte(spaceChar);
-
- double rawPercent = (double) hitCount / (double) ngramCount;
-
-// if (rawPercent <= 2.0) {
-// return 0;
-// }
-
- // TODO - This is a bit of a hack to take care of a case
- // were we were getting a confidence of 135...
- if (rawPercent > 0.33) {
- return 98;
- }
-
- return (int) (rawPercent * 300.0);
- }
- }
-
- static class NGramParser_IBM420 extends NGramParser {
- protected static byte[] unshapeMap = {
-/* -0 -1 -2 -3
- -4 -5 -6 -7 -8
- -9 -A -B -C -D -E -F */
-/* 0- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 1- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 2- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 3- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 4- */ (byte) 0x40, (byte) 0x40, (byte) 0x42, (byte) 0x42, (byte) 0x44, (byte) 0x45,
- (byte) 0x46, (byte) 0x47, (byte) 0x47, (byte) 0x49, (byte) 0x4A, (byte) 0x4B,
- (byte) 0x4C, (byte) 0x4D, (byte) 0x4E, (byte) 0x4F,
-/* 5- */ (byte) 0x50, (byte) 0x49, (byte) 0x52, (byte) 0x53, (byte) 0x54, (byte) 0x55,
- (byte) 0x56, (byte) 0x56, (byte) 0x58, (byte) 0x58, (byte) 0x5A, (byte) 0x5B,
- (byte) 0x5C, (byte) 0x5D, (byte) 0x5E, (byte) 0x5F,
-/* 6- */ (byte) 0x60, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x63, (byte) 0x65,
- (byte) 0x65, (byte) 0x67, (byte) 0x67, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F,
-/* 7- */ (byte) 0x69, (byte) 0x71, (byte) 0x71, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x77, (byte) 0x79, (byte) 0x7A, (byte) 0x7B,
- (byte) 0x7C, (byte) 0x7D, (byte) 0x7E, (byte) 0x7F,
-/* 8- */ (byte) 0x80, (byte) 0x81, (byte) 0x82, (byte) 0x83, (byte) 0x84, (byte) 0x85,
- (byte) 0x86, (byte) 0x87, (byte) 0x88, (byte) 0x89, (byte) 0x80, (byte) 0x8B,
- (byte) 0x8B, (byte) 0x8D, (byte) 0x8D, (byte) 0x8F,
-/* 9- */ (byte) 0x90, (byte) 0x91, (byte) 0x92, (byte) 0x93, (byte) 0x94, (byte) 0x95,
- (byte) 0x96, (byte) 0x97, (byte) 0x98, (byte) 0x99, (byte) 0x9A, (byte) 0x9A,
- (byte) 0x9A, (byte) 0x9A, (byte) 0x9E, (byte) 0x9E,
-/* A- */ (byte) 0x9E, (byte) 0xA1, (byte) 0xA2, (byte) 0xA3, (byte) 0xA4, (byte) 0xA5,
- (byte) 0xA6, (byte) 0xA7, (byte) 0xA8, (byte) 0xA9, (byte) 0x9E, (byte) 0xAB,
- (byte) 0xAB, (byte) 0xAD, (byte) 0xAD, (byte) 0xAF,
-/* B- */ (byte) 0xAF, (byte) 0xB1, (byte) 0xB2, (byte) 0xB3, (byte) 0xB4, (byte) 0xB5,
- (byte) 0xB6, (byte) 0xB7, (byte) 0xB8, (byte) 0xB9, (byte) 0xB1, (byte) 0xBB,
- (byte) 0xBB, (byte) 0xBD, (byte) 0xBD, (byte) 0xBF,
-/* C- */ (byte) 0xC0, (byte) 0xC1, (byte) 0xC2, (byte) 0xC3, (byte) 0xC4, (byte) 0xC5,
- (byte) 0xC6, (byte) 0xC7, (byte) 0xC8, (byte) 0xC9, (byte) 0xCA, (byte) 0xBF,
- (byte) 0xCC, (byte) 0xBF, (byte) 0xCE, (byte) 0xCF,
-/* D- */ (byte) 0xD0, (byte) 0xD1, (byte) 0xD2, (byte) 0xD3, (byte) 0xD4, (byte) 0xD5,
- (byte) 0xD6, (byte) 0xD7, (byte) 0xD8, (byte) 0xD9, (byte) 0xDA, (byte) 0xDA,
- (byte) 0xDC, (byte) 0xDC, (byte) 0xDC, (byte) 0xDF,
-/* E- */ (byte) 0xE0, (byte) 0xE1, (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5,
- (byte) 0xE6, (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF,
-/* F- */ (byte) 0xF0, (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0xF7, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB,
- (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0xFF,};
- private byte alef = 0x00;
-
-
- public NGramParser_IBM420(int[] theNgramList, byte[] theByteMap) {
- super(theNgramList, theByteMap);
- }
-
- private byte isLamAlef(byte b) {
- if (b == (byte) 0xb2 || b == (byte) 0xb3) {
- return (byte) 0x47;
- } else if (b == (byte) 0xb4 || b == (byte) 0xb5) {
- return (byte) 0x49;
- } else if (b == (byte) 0xb8 || b == (byte) 0xb9) {
- return (byte) 0x56;
- } else {
- return (byte) 0x00;
- }
- }
-
- /*
- * Arabic shaping needs to be done manually. Cannot call ArabicShaping class
- * because CharsetDetector is dealing with bytes not Unicode code points. We could
- * convert the bytes to Unicode code points but that would leave us dependent
- * on CharsetICU which we try to avoid. IBM420 converter amongst different versions
- * of JDK can produce different results and therefore is also avoided.
- */
- private int nextByte(CharsetDetector det) {
- if (byteIndex >= det.fInputLen || det.fInputBytes[byteIndex] == 0) {
- return -1;
- }
- int next;
-
- alef = isLamAlef(det.fInputBytes[byteIndex]);
- if (alef != (byte) 0x00) {
- next = 0xB1 & 0xFF;
- } else {
- next = unshapeMap[det.fInputBytes[byteIndex] & 0xFF] & 0xFF;
- }
-
- byteIndex++;
-
- return next;
- }
-
- protected void parseCharacters(CharsetDetector det) {
- int b;
- boolean ignoreSpace = false;
-
- while ((b = nextByte(det)) >= 0) {
- byte mb = byteMap[b];
-
- // TODO: 0x20 might not be a space in all character sets...
- if (mb != 0) {
- if (!(mb == spaceChar && ignoreSpace)) {
- addByte(mb);
- }
-
- ignoreSpace = (mb == spaceChar);
- }
- if (alef != (byte) 0x00) {
- mb = byteMap[alef & 0xFF];
-
- // TODO: 0x20 might not be a space in all character sets...
- if (mb != 0) {
- if (!(mb == spaceChar && ignoreSpace)) {
- addByte(mb);
- }
-
- ignoreSpace = (mb == spaceChar);
- }
-
- }
- }
- }
- }
-
- static class NGramsPlusLang {
- int[] fNGrams;
- String fLang;
-
- NGramsPlusLang(String la, int[] ng) {
- fLang = la;
- fNGrams = ng;
- }
- }
-
- static class CharsetRecog_8859_1 extends CharsetRecog_sbcs {
- protected static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xAA,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xB5, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xBA, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0xE0, (byte) 0xE1, (byte) 0xE2, (byte) 0xE3,
- (byte) 0xE4, (byte) 0xE5, (byte) 0xE6, (byte) 0xE7, (byte) 0xE8,
- (byte) 0xE9, (byte) 0xEA, (byte) 0xEB, (byte) 0xEC, (byte) 0xED,
- (byte) 0xEE, (byte) 0xEF, (byte) 0xF0, (byte) 0xF1, (byte) 0xF2,
- (byte) 0xF3, (byte) 0xF4, (byte) 0xF5, (byte) 0xF6, (byte) 0x20,
- (byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC,
- (byte) 0xFD, (byte) 0xFE, (byte) 0xDF, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0xF0,
- (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0x20, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA,
- (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0xFF,};
-
-
- private static NGramsPlusLang[] ngrams_8859_1 = new NGramsPlusLang[]{
- new NGramsPlusLang("da",
- new int[]{0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F,
- 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369,
- 0x207374, 0x207469, 0x207669, 0x616620, 0x616E20, 0x616E64,
- 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574,
- 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220,
- 0x657265, 0x657320, 0x657420, 0x666F72, 0x676520, 0x67656E,
- 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65,
- 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520,
- 0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520,
- 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073,
- 0x746520, 0x746572, 0x74696C, 0x766572,}),
-
- new NGramsPlusLang("de",
- new int[]{0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469,
- 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363,
- 0x207365, 0x20756E, 0x207665, 0x20766F, 0x207765, 0x207A75,
- 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E,
- 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974,
- 0x656E20, 0x657220, 0x657320, 0x67656E, 0x68656E, 0x687420,
- 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963,
- 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465,
- 0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465,
- 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E,
- 0x746572, 0x756E64, 0x756E67, 0x766572,}),
-
- new NGramsPlusLang("en",
- new int[]{0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861,
- 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265,
- 0x207361, 0x207374, 0x207468, 0x20746F, 0x207768, 0x616964,
- 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465,
- 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074,
- 0x656420, 0x656E74, 0x657220, 0x657320, 0x666F72, 0x686174,
- 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E,
- 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420,
- 0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061,
- 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861,
- 0x746865, 0x74696F, 0x746F20, 0x747320,}),
-
- new NGramsPlusLang("es",
- new int[]{0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E,
- 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F,
- 0x207072, 0x207175, 0x207265, 0x207365, 0x20756E, 0x207920,
- 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369,
- 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E,
- 0x646520, 0x64656C, 0x646F20, 0x652064, 0x652065, 0x65206C,
- 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E,
- 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064,
- 0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565,
- 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520,
- 0x746520, 0x746F20, 0x756520, 0xF36E20,}),
-
- new NGramsPlusLang("fr",
- new int[]{0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E,
- 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072,
- 0x207175, 0x207365, 0x20736F, 0x20756E, 0x20E020, 0x616E74,
- 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520,
- 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070,
- 0x652073, 0x656E20, 0x656E74, 0x657220, 0x657320, 0x657420,
- 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520,
- 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420,
- 0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520,
- 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064,
- 0x746520, 0x74696F, 0x756520, 0x757220,}),
-
- new NGramsPlusLang("it",
- new int[]{0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520,
- 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E,
- 0x612063, 0x612064, 0x612070, 0x612073, 0x61746F, 0x636865,
- 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064,
- 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C,
- 0x656E74, 0x657220, 0x686520, 0x692061, 0x692063, 0x692064,
- 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120,
- 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20,
- 0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20,
- 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120,
- 0x746520, 0x746920, 0x746F20, 0x7A696F,}),
-
- new NGramsPlusLang("nl",
- new int[]{0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565,
- 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65,
- 0x206F70, 0x207465, 0x207661, 0x207665, 0x20766F, 0x207765,
- 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220,
- 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062,
- 0x652076, 0x65656E, 0x656572, 0x656E20, 0x657220, 0x657273,
- 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67,
- 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F,
- 0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020,
- 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E,
- 0x746572, 0x76616E, 0x766572, 0x766F6F,}),
-
- new NGramsPlusLang("no",
- new int[]{0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F,
- 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365,
- 0x20736B, 0x20736F, 0x207374, 0x207469, 0x207669, 0x20E520,
- 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574,
- 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265,
- 0x657420, 0x657474, 0x666F72, 0x67656E, 0x696B6B, 0x696C20,
- 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564,
- 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65,
- 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520,
- 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572,
- 0x74696C, 0x747420, 0x747465, 0x766572,}),
-
- new NGramsPlusLang("pt",
- new int[]{0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520,
- 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F,
- 0x207072, 0x207175, 0x207265, 0x207365, 0x20756D, 0x612061,
- 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20,
- 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120,
- 0x646520, 0x646F20, 0x646F73, 0x652061, 0x652064, 0x656D20,
- 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E,
- 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065,
- 0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573,
- 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520,
- 0x746F20, 0x756520, 0xE36F20, 0xE7E36F,
-
- }),
-
- new NGramsPlusLang("sv",
- new int[]{0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861,
- 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5,
- 0x20736B, 0x20736F, 0x207374, 0x207469, 0x207661, 0x207669,
- 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474,
- 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420,
- 0x656E20, 0x657220, 0x657420, 0x66F672, 0x67656E, 0x696C6C,
- 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120,
- 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20,
- 0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61,
- 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C,
- 0x747420, 0x766172, 0xE47220, 0xF67220,}),
-
- };
-
-
- public CharsetMatch match(CharsetDetector det) {
- String name = det.fC1Bytes ? "windows-1252" : "ISO-8859-1";
- int bestConfidenceSoFar = -1;
- String lang = null;
- for (NGramsPlusLang ngl : ngrams_8859_1) {
- int confidence = match(det, ngl.fNGrams, byteMap);
- if (confidence > bestConfidenceSoFar) {
- bestConfidenceSoFar = confidence;
- lang = ngl.fLang;
- }
- }
- return bestConfidenceSoFar <= 0 ? null :
- new CharsetMatch(det, this, bestConfidenceSoFar, name, lang);
- }
-
-
- public String getName() {
- return "ISO-8859-1";
- }
- }
-
-
- static class CharsetRecog_8859_2 extends CharsetRecog_sbcs {
- protected static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xB1, (byte) 0x20, (byte) 0xB3, (byte) 0x20, (byte) 0xB5,
- (byte) 0xB6, (byte) 0x20, (byte) 0x20, (byte) 0xB9, (byte) 0xBA,
- (byte) 0xBB, (byte) 0xBC, (byte) 0x20, (byte) 0xBE, (byte) 0xBF,
- (byte) 0x20, (byte) 0xB1, (byte) 0x20, (byte) 0xB3, (byte) 0x20,
- (byte) 0xB5, (byte) 0xB6, (byte) 0xB7, (byte) 0x20, (byte) 0xB9,
- (byte) 0xBA, (byte) 0xBB, (byte) 0xBC, (byte) 0x20, (byte) 0xBE,
- (byte) 0xBF, (byte) 0xE0, (byte) 0xE1, (byte) 0xE2, (byte) 0xE3,
- (byte) 0xE4, (byte) 0xE5, (byte) 0xE6, (byte) 0xE7, (byte) 0xE8,
- (byte) 0xE9, (byte) 0xEA, (byte) 0xEB, (byte) 0xEC, (byte) 0xED,
- (byte) 0xEE, (byte) 0xEF, (byte) 0xF0, (byte) 0xF1, (byte) 0xF2,
- (byte) 0xF3, (byte) 0xF4, (byte) 0xF5, (byte) 0xF6, (byte) 0x20,
- (byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC,
- (byte) 0xFD, (byte) 0xFE, (byte) 0xDF, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0xF0,
- (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0x20, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA,
- (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0x20,};
-
- private static NGramsPlusLang[] ngrams_8859_2 = new NGramsPlusLang[]{
- new NGramsPlusLang("cs",
- new int[]{0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65,
- 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F,
- 0x207365, 0x20736F, 0x207374, 0x20746F, 0x207620, 0x207679,
- 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073,
- 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374,
- 0x6A6520, 0x6B7465, 0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920,
- 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374,
- 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865,
- 0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572,
- 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F,
- 0xED2070, 0xED2073, 0xED6D20, 0xF86564,}),
- new NGramsPlusLang("hu",
- new int[]{0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665,
- 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6,
- 0x206C65, 0x206D61, 0x206D65, 0x206D69, 0x206E65, 0x20737A,
- 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073,
- 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779,
- 0x656B20, 0x656C20, 0x656C65, 0x656D20, 0x656E20, 0x657265,
- 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74,
- 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061,
- 0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061,
- 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420,
- 0x74E173, 0x7A6572, 0xE16E20, 0xE97320,}),
- new NGramsPlusLang("pl",
- new int[]{0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61,
- 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072,
- 0x207369, 0x207720, 0x207769, 0x207779, 0x207A20, 0x207A61,
- 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79,
- 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A,
- 0x65676F, 0x656A20, 0x656D20, 0x656E69, 0x676F20, 0x696120,
- 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965,
- 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769,
- 0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79,
- 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20,
- 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720,}),
- new NGramsPlusLang("ro",
- new int[]{0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375,
- 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072,
- 0x207365, 0x2073E3, 0x20756E, 0x20BA69, 0x20EE6E, 0x612063,
- 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172,
- 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063,
- 0x652064, 0x652070, 0x652073, 0x656120, 0x656920, 0x656C65,
- 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070,
- 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72,
- 0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520,
- 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320,
- 0x756920, 0x756C20, 0xBA6920, 0xEE6E20,})};
-
- public CharsetMatch match(CharsetDetector det) {
- String name = det.fC1Bytes ? "windows-1250" : "ISO-8859-2";
- int bestConfidenceSoFar = -1;
- String lang = null;
- for (NGramsPlusLang ngl : ngrams_8859_2) {
- int confidence = match(det, ngl.fNGrams, byteMap);
- if (confidence > bestConfidenceSoFar) {
- bestConfidenceSoFar = confidence;
- lang = ngl.fLang;
- }
- }
- return bestConfidenceSoFar <= 0 ? null :
- new CharsetMatch(det, this, bestConfidenceSoFar, name, lang);
- }
-
- public String getName() {
- return "ISO-8859-2";
- }
-
- }
-
-
- abstract static class CharsetRecog_8859_5 extends CharsetRecog_sbcs {
- protected static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0xF7, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA,
- (byte) 0xFB, (byte) 0xFC, (byte) 0x20, (byte) 0xFE, (byte) 0xFF,
- (byte) 0xD0, (byte) 0xD1, (byte) 0xD2, (byte) 0xD3, (byte) 0xD4,
- (byte) 0xD5, (byte) 0xD6, (byte) 0xD7, (byte) 0xD8, (byte) 0xD9,
- (byte) 0xDA, (byte) 0xDB, (byte) 0xDC, (byte) 0xDD, (byte) 0xDE,
- (byte) 0xDF, (byte) 0xE0, (byte) 0xE1, (byte) 0xE2, (byte) 0xE3,
- (byte) 0xE4, (byte) 0xE5, (byte) 0xE6, (byte) 0xE7, (byte) 0xE8,
- (byte) 0xE9, (byte) 0xEA, (byte) 0xEB, (byte) 0xEC, (byte) 0xED,
- (byte) 0xEE, (byte) 0xEF, (byte) 0xD0, (byte) 0xD1, (byte) 0xD2,
- (byte) 0xD3, (byte) 0xD4, (byte) 0xD5, (byte) 0xD6, (byte) 0xD7,
- (byte) 0xD8, (byte) 0xD9, (byte) 0xDA, (byte) 0xDB, (byte) 0xDC,
- (byte) 0xDD, (byte) 0xDE, (byte) 0xDF, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0x20,
- (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0xF7, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA,
- (byte) 0xFB, (byte) 0xFC, (byte) 0x20, (byte) 0xFE, (byte) 0xFF,};
-
- public String getName() {
- return "ISO-8859-5";
- }
- }
-
- static class CharsetRecog_8859_5_ru extends CharsetRecog_8859_5 {
- private static int[] ngrams =
- {0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0,
- 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2,
- 0x20E2DE, 0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC,
- 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20,
- 0xDBD5DD, 0xDBD820, 0xDBECDD, 0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF,
- 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0,
- 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2, 0xDFDEDB, 0xDFE0D5,
- 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8,
- 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520,};
-
- public String getLanguage() {
- return "ru";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- abstract static class CharsetRecog_8859_6 extends CharsetRecog_sbcs {
- protected static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0xC1, (byte) 0xC2, (byte) 0xC3,
- (byte) 0xC4, (byte) 0xC5, (byte) 0xC6, (byte) 0xC7, (byte) 0xC8,
- (byte) 0xC9, (byte) 0xCA, (byte) 0xCB, (byte) 0xCC, (byte) 0xCD,
- (byte) 0xCE, (byte) 0xCF, (byte) 0xD0, (byte) 0xD1, (byte) 0xD2,
- (byte) 0xD3, (byte) 0xD4, (byte) 0xD5, (byte) 0xD6, (byte) 0xD7,
- (byte) 0xD8, (byte) 0xD9, (byte) 0xDA, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,};
-
- public String getName() {
- return "ISO-8859-6";
- }
- }
-
- static class CharsetRecog_8859_6_ar extends CharsetRecog_8859_6 {
- private static int[] ngrams =
- {0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7,
- 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7,
- 0xC7E4C8, 0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9,
- 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA,
- 0xC820C7, 0xC920C7, 0xC920E1, 0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7,
- 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9,
- 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20, 0xE520C7, 0xE5C720,
- 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620,
- 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620,};
-
- public String getLanguage() {
- return "ar";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- abstract static class CharsetRecog_8859_7 extends CharsetRecog_sbcs {
- protected static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xA1, (byte) 0xA2, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0xDC, (byte) 0x20, (byte) 0xDD, (byte) 0xDE,
- (byte) 0xDF, (byte) 0x20, (byte) 0xFC, (byte) 0x20, (byte) 0xFD,
- (byte) 0xFE, (byte) 0xC0, (byte) 0xE1, (byte) 0xE2, (byte) 0xE3,
- (byte) 0xE4, (byte) 0xE5, (byte) 0xE6, (byte) 0xE7, (byte) 0xE8,
- (byte) 0xE9, (byte) 0xEA, (byte) 0xEB, (byte) 0xEC, (byte) 0xED,
- (byte) 0xEE, (byte) 0xEF, (byte) 0xF0, (byte) 0xF1, (byte) 0x20,
- (byte) 0xF3, (byte) 0xF4, (byte) 0xF5, (byte) 0xF6, (byte) 0xF7,
- (byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xDC,
- (byte) 0xDD, (byte) 0xDE, (byte) 0xDF, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0xF0,
- (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0xF7, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA,
- (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0x20,};
-
- public String getName() {
- return "ISO-8859-7";
- }
- }
-
- static class CharsetRecog_8859_7_el extends CharsetRecog_8859_7 {
- private static int[] ngrams =
- {0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5,
- 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5,
- 0x20F4E7, 0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20,
- 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20,
- 0xE7F220, 0xE920F4, 0xE9E120, 0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4,
- 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520,
- 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5, 0xF220EA, 0xF220F0,
- 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED,
- 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20,};
-
- public String getLanguage() {
- return "el";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- String name = det.fC1Bytes ? "windows-1253" : "ISO-8859-7";
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence, name, "el");
- }
- }
-
- abstract static class CharsetRecog_8859_8 extends CharsetRecog_sbcs {
- protected static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xB5, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0xF0,
- (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0xF7, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,};
-
- public String getName() {
- return "ISO-8859-8";
- }
- }
-
- static class CharsetRecog_8859_8_I_he extends CharsetRecog_8859_8 {
- private static int[] ngrams =
- {0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5,
- 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4,
- 0x20EEE0, 0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4,
- 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20,
- 0xE5F820, 0xE5FA20, 0xE920E4, 0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20,
- 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420,
- 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE, 0xED20F9, 0xEEE420,
- 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED,
- 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9,};
-
- public String getName() {
- return "ISO-8859-8-I";
- }
-
- public String getLanguage() {
- return "he";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- String name = det.fC1Bytes ? "windows-1255" : "ISO-8859-8-I";
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence, name, "he");
- }
- }
-
- static class CharsetRecog_8859_8_he extends CharsetRecog_8859_8 {
- private static int[] ngrams =
- {0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2,
- 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9,
- 0x20FAE0, 0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA,
- 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA,
- 0xE420E4, 0xE420E9, 0xE420EC, 0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA,
- 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420,
- 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920, 0xEDE9E9, 0xEDE9F0,
- 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420,
- 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9,};
-
- public String getLanguage() {
- return "he";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- String name = det.fC1Bytes ? "windows-1255" : "ISO-8859-8";
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence, name, "he");
-
- }
- }
-
- abstract static class CharsetRecog_8859_9 extends CharsetRecog_sbcs {
- protected static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xAA,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xB5, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xBA, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0xE0, (byte) 0xE1, (byte) 0xE2, (byte) 0xE3,
- (byte) 0xE4, (byte) 0xE5, (byte) 0xE6, (byte) 0xE7, (byte) 0xE8,
- (byte) 0xE9, (byte) 0xEA, (byte) 0xEB, (byte) 0xEC, (byte) 0xED,
- (byte) 0xEE, (byte) 0xEF, (byte) 0xF0, (byte) 0xF1, (byte) 0xF2,
- (byte) 0xF3, (byte) 0xF4, (byte) 0xF5, (byte) 0xF6, (byte) 0x20,
- (byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC,
- (byte) 0x69, (byte) 0xFE, (byte) 0xDF, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0xF0,
- (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0x20, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA,
- (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0xFF,};
-
- public String getName() {
- return "ISO-8859-9";
- }
- }
-
- static class CharsetRecog_8859_9_tr extends CharsetRecog_8859_9 {
- private static int[] ngrams =
- {0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C,
- 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665,
- 0x207961, 0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD,
- 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120,
- 0x646520, 0x646920, 0x652062, 0x65206B, 0x656469, 0x656E20, 0x657220,
- 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E,
- 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062, 0x6E206B, 0x6E6461,
- 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E,
- 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD,};
-
- public String getLanguage() {
- return "tr";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- String name = det.fC1Bytes ? "windows-1254" : "ISO-8859-9";
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence, name, "tr");
- }
- }
-
- static class CharsetRecog_windows_1251 extends CharsetRecog_sbcs {
- private static int[] ngrams =
- {0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0,
- 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2,
- 0x20F2EE, 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC,
- 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20,
- 0xEBE5ED, 0xEBE820, 0xEBFCED, 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF,
- 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0,
- 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2, 0xEFEEEB, 0xEFF0E5,
- 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8,
- 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520,};
-
- private static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x90, (byte) 0x83, (byte) 0x20,
- (byte) 0x83, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x9A, (byte) 0x20, (byte) 0x9C,
- (byte) 0x9D, (byte) 0x9E, (byte) 0x9F, (byte) 0x90, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x9A, (byte) 0x20,
- (byte) 0x9C, (byte) 0x9D, (byte) 0x9E, (byte) 0x9F, (byte) 0x20,
- (byte) 0xA2, (byte) 0xA2, (byte) 0xBC, (byte) 0x20, (byte) 0xB4,
- (byte) 0x20, (byte) 0x20, (byte) 0xB8, (byte) 0x20, (byte) 0xBA,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xBF,
- (byte) 0x20, (byte) 0x20, (byte) 0xB3, (byte) 0xB3, (byte) 0xB4,
- (byte) 0xB5, (byte) 0x20, (byte) 0x20, (byte) 0xB8, (byte) 0x20,
- (byte) 0xBA, (byte) 0x20, (byte) 0xBC, (byte) 0xBE, (byte) 0xBE,
- (byte) 0xBF, (byte) 0xE0, (byte) 0xE1, (byte) 0xE2, (byte) 0xE3,
- (byte) 0xE4, (byte) 0xE5, (byte) 0xE6, (byte) 0xE7, (byte) 0xE8,
- (byte) 0xE9, (byte) 0xEA, (byte) 0xEB, (byte) 0xEC, (byte) 0xED,
- (byte) 0xEE, (byte) 0xEF, (byte) 0xF0, (byte) 0xF1, (byte) 0xF2,
- (byte) 0xF3, (byte) 0xF4, (byte) 0xF5, (byte) 0xF6, (byte) 0xF7,
- (byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC,
- (byte) 0xFD, (byte) 0xFE, (byte) 0xFF, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0xF0,
- (byte) 0xF1, (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5,
- (byte) 0xF6, (byte) 0xF7, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA,
- (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0xFF,};
-
- public String getName() {
- return "windows-1251";
- }
-
- public String getLanguage() {
- return "ru";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null :
- new CharsetMatch(det, this, confidence, getName(), "ru");
- }
- }
-
- static class CharsetRecog_IBM866_ru extends CharsetRecog_sbcs {
- private static int[] ngrams =
- {0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0,
- 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2,
- 0x20F2EE, 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC,
- 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20,
- 0xEBE5ED, 0xEBE820, 0xEBFCED, 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF,
- 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0,
- 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2, 0xEFEEEB, 0xEFF0E5,
- 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8,
- 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520,};
-
- // bytemap converts cp866 chars to cp1251 chars, so ngrams are still unchanged
- private static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0xE0, (byte) 0xE1, (byte) 0xE2,
- (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6, (byte) 0xE7,
- (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB, (byte) 0xEC,
- (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0xF0, (byte) 0xF1,
- (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5, (byte) 0xF6,
- (byte) 0xF7, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB,
- (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0xFF, (byte) 0xE0,
- (byte) 0xE1, (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5,
- (byte) 0xE6, (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA,
- (byte) 0xEB, (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xF0, (byte) 0xF1,
- (byte) 0xF2, (byte) 0xF3, (byte) 0xF4, (byte) 0xF5, (byte) 0xF6,
- (byte) 0xF7, (byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB,
- (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0xFF, (byte) 0xB8,
- (byte) 0xB8, (byte) 0xBA, (byte) 0xBA, (byte) 0xBF, (byte) 0xBF,
- (byte) 0xA2, (byte) 0xA2, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,};
-
- public String getName() {
- return "IBM866";
- }
-
- public String getLanguage() {
- return "ru";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- static class CharsetRecog_windows_1256 extends CharsetRecog_sbcs {
- private static int[] ngrams =
- {0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7,
- 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7,
- 0xC7E1C8, 0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA,
- 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA,
- 0xC820C7, 0xC920C7, 0xC920DD, 0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7,
- 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC,
- 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20, 0xE320C7, 0xE3C720,
- 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420,
- 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420,};
-
- private static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x81, (byte) 0x20,
- (byte) 0x83, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x88, (byte) 0x20, (byte) 0x8A, (byte) 0x20, (byte) 0x9C,
- (byte) 0x8D, (byte) 0x8E, (byte) 0x8F, (byte) 0x90, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x98, (byte) 0x20, (byte) 0x9A, (byte) 0x20,
- (byte) 0x9C, (byte) 0x20, (byte) 0x20, (byte) 0x9F, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xAA,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0xB5, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0xC0, (byte) 0xC1, (byte) 0xC2, (byte) 0xC3,
- (byte) 0xC4, (byte) 0xC5, (byte) 0xC6, (byte) 0xC7, (byte) 0xC8,
- (byte) 0xC9, (byte) 0xCA, (byte) 0xCB, (byte) 0xCC, (byte) 0xCD,
- (byte) 0xCE, (byte) 0xCF, (byte) 0xD0, (byte) 0xD1, (byte) 0xD2,
- (byte) 0xD3, (byte) 0xD4, (byte) 0xD5, (byte) 0xD6, (byte) 0x20,
- (byte) 0xD8, (byte) 0xD9, (byte) 0xDA, (byte) 0xDB, (byte) 0xDC,
- (byte) 0xDD, (byte) 0xDE, (byte) 0xDF, (byte) 0xE0, (byte) 0xE1,
- (byte) 0xE2, (byte) 0xE3, (byte) 0xE4, (byte) 0xE5, (byte) 0xE6,
- (byte) 0xE7, (byte) 0xE8, (byte) 0xE9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0xEC, (byte) 0xED, (byte) 0xEE, (byte) 0xEF, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xF4, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xF9, (byte) 0x20,
- (byte) 0xFB, (byte) 0xFC, (byte) 0x20, (byte) 0x20, (byte) 0xFF,};
-
- public String getName() {
- return "windows-1256";
- }
-
- public String getLanguage() {
- return "ar";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- static class CharsetRecog_KOI8_R extends CharsetRecog_sbcs {
- private static int[] ngrams =
- {0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF,
- 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF,
- 0x20DAC1, 0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9,
- 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120,
- 0xCCC5CE, 0xCCC920, 0xCCD8CE, 0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1,
- 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20,
- 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1, 0xD0CFCC, 0xD0D2C5,
- 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9,
- 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF,};
-
- private static byte[] byteMap =
- {(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x61,
- (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65, (byte) 0x66,
- (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x6A, (byte) 0x6B,
- (byte) 0x6C, (byte) 0x6D, (byte) 0x6E, (byte) 0x6F, (byte) 0x70,
- (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x7A,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64,
- (byte) 0x65, (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69,
- (byte) 0x6A, (byte) 0x6B, (byte) 0x6C, (byte) 0x6D, (byte) 0x6E,
- (byte) 0x6F, (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73,
- (byte) 0x74, (byte) 0x75, (byte) 0x76, (byte) 0x77, (byte) 0x78,
- (byte) 0x79, (byte) 0x7A, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0xA3, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0xA3, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0xC0, (byte) 0xC1, (byte) 0xC2, (byte) 0xC3,
- (byte) 0xC4, (byte) 0xC5, (byte) 0xC6, (byte) 0xC7, (byte) 0xC8,
- (byte) 0xC9, (byte) 0xCA, (byte) 0xCB, (byte) 0xCC, (byte) 0xCD,
- (byte) 0xCE, (byte) 0xCF, (byte) 0xD0, (byte) 0xD1, (byte) 0xD2,
- (byte) 0xD3, (byte) 0xD4, (byte) 0xD5, (byte) 0xD6, (byte) 0xD7,
- (byte) 0xD8, (byte) 0xD9, (byte) 0xDA, (byte) 0xDB, (byte) 0xDC,
- (byte) 0xDD, (byte) 0xDE, (byte) 0xDF, (byte) 0xC0, (byte) 0xC1,
- (byte) 0xC2, (byte) 0xC3, (byte) 0xC4, (byte) 0xC5, (byte) 0xC6,
- (byte) 0xC7, (byte) 0xC8, (byte) 0xC9, (byte) 0xCA, (byte) 0xCB,
- (byte) 0xCC, (byte) 0xCD, (byte) 0xCE, (byte) 0xCF, (byte) 0xD0,
- (byte) 0xD1, (byte) 0xD2, (byte) 0xD3, (byte) 0xD4, (byte) 0xD5,
- (byte) 0xD6, (byte) 0xD7, (byte) 0xD8, (byte) 0xD9, (byte) 0xDA,
- (byte) 0xDB, (byte) 0xDC, (byte) 0xDD, (byte) 0xDE, (byte) 0xDF,};
-
- public String getName() {
- return "KOI8-R";
- }
-
- public String getLanguage() {
- return "ru";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, ngrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- abstract static class CharsetRecog_IBM424_he extends CharsetRecog_sbcs {
- protected static byte[] byteMap = {
-/*
- -0 -1 -2 -3 -4
- -5 -6 -7 -8 -9 -A
- -B -C -D -E -F
- */
-/* 0- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 1- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 2- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 3- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 4- */ (byte) 0x40, (byte) 0x41, (byte) 0x42, (byte) 0x43, (byte) 0x44, (byte) 0x45,
- (byte) 0x46, (byte) 0x47, (byte) 0x48, (byte) 0x49, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 5- */ (byte) 0x40, (byte) 0x51, (byte) 0x52, (byte) 0x53, (byte) 0x54, (byte) 0x55,
- (byte) 0x56, (byte) 0x57, (byte) 0x58, (byte) 0x59, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 6- */ (byte) 0x40, (byte) 0x40, (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65,
- (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 7- */ (byte) 0x40, (byte) 0x71, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x00, (byte) 0x40, (byte) 0x40,
-/* 8- */ (byte) 0x40, (byte) 0x81, (byte) 0x82, (byte) 0x83, (byte) 0x84, (byte) 0x85,
- (byte) 0x86, (byte) 0x87, (byte) 0x88, (byte) 0x89, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 9- */ (byte) 0x40, (byte) 0x91, (byte) 0x92, (byte) 0x93, (byte) 0x94, (byte) 0x95,
- (byte) 0x96, (byte) 0x97, (byte) 0x98, (byte) 0x99, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* A- */ (byte) 0xA0, (byte) 0x40, (byte) 0xA2, (byte) 0xA3, (byte) 0xA4, (byte) 0xA5,
- (byte) 0xA6, (byte) 0xA7, (byte) 0xA8, (byte) 0xA9, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* B- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* C- */ (byte) 0x40, (byte) 0x81, (byte) 0x82, (byte) 0x83, (byte) 0x84, (byte) 0x85,
- (byte) 0x86, (byte) 0x87, (byte) 0x88, (byte) 0x89, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* D- */ (byte) 0x40, (byte) 0x91, (byte) 0x92, (byte) 0x93, (byte) 0x94, (byte) 0x95,
- (byte) 0x96, (byte) 0x97, (byte) 0x98, (byte) 0x99, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* E- */ (byte) 0x40, (byte) 0x40, (byte) 0xA2, (byte) 0xA3, (byte) 0xA4, (byte) 0xA5,
- (byte) 0xA6, (byte) 0xA7, (byte) 0xA8, (byte) 0xA9, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* F- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,};
-
- public String getLanguage() {
- return "he";
- }
- }
-
- static class CharsetRecog_IBM424_he_rtl extends CharsetRecog_IBM424_he {
- private static int[] ngrams =
- {0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546,
- 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445,
- 0x405641, 0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045,
- 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740,
- 0x466840, 0x467140, 0x514045, 0x514540, 0x514671, 0x515155, 0x515540,
- 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540,
- 0x554041, 0x554042, 0x554045, 0x554054, 0x554056, 0x554069, 0x564540,
- 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155,
- 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069,};
-
- public String getName() {
- return "IBM424_rtl";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, ngrams, byteMap, (byte) 0x40);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- static class CharsetRecog_IBM424_he_ltr extends CharsetRecog_IBM424_he {
- private static int[] ngrams =
- {0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462,
- 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851,
- 0x407141, 0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071,
- 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071,
- 0x454045, 0x454051, 0x454054, 0x454055, 0x454057, 0x454068, 0x454071,
- 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540,
- 0x544045, 0x544055, 0x544071, 0x546240, 0x546940, 0x555151, 0x555158,
- 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540,
- 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651
-
- };
-
- public String getName() {
- return "IBM424_ltr";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = match(det, ngrams, byteMap, (byte) 0x40);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
- }
-
- abstract static class CharsetRecog_IBM420_ar extends CharsetRecog_sbcs {
-
- protected static byte[] byteMap = {
-/* -0 -1 -2
--3 -4 -5 -6 -7 -8
--9 -A -B -C -D -E -F */
-/* 0- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 1- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 2- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 3- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 4- */ (byte) 0x40, (byte) 0x40, (byte) 0x42, (byte) 0x43, (byte) 0x44, (byte) 0x45,
- (byte) 0x46, (byte) 0x47, (byte) 0x48, (byte) 0x49, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 5- */ (byte) 0x40, (byte) 0x51, (byte) 0x52, (byte) 0x40, (byte) 0x40, (byte) 0x55,
- (byte) 0x56, (byte) 0x57, (byte) 0x58, (byte) 0x59, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 6- */ (byte) 0x40, (byte) 0x40, (byte) 0x62, (byte) 0x63, (byte) 0x64, (byte) 0x65,
- (byte) 0x66, (byte) 0x67, (byte) 0x68, (byte) 0x69, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 7- */ (byte) 0x70, (byte) 0x71, (byte) 0x72, (byte) 0x73, (byte) 0x74, (byte) 0x75,
- (byte) 0x76, (byte) 0x77, (byte) 0x78, (byte) 0x79, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
-/* 8- */ (byte) 0x80, (byte) 0x81, (byte) 0x82, (byte) 0x83, (byte) 0x84, (byte) 0x85,
- (byte) 0x86, (byte) 0x87, (byte) 0x88, (byte) 0x89, (byte) 0x8A, (byte) 0x8B,
- (byte) 0x8C, (byte) 0x8D, (byte) 0x8E, (byte) 0x8F,
-/* 9- */ (byte) 0x90, (byte) 0x91, (byte) 0x92, (byte) 0x93, (byte) 0x94, (byte) 0x95,
- (byte) 0x96, (byte) 0x97, (byte) 0x98, (byte) 0x99, (byte) 0x9A, (byte) 0x9B,
- (byte) 0x9C, (byte) 0x9D, (byte) 0x9E, (byte) 0x9F,
-/* A- */ (byte) 0xA0, (byte) 0x40, (byte) 0xA2, (byte) 0xA3, (byte) 0xA4, (byte) 0xA5,
- (byte) 0xA6, (byte) 0xA7, (byte) 0xA8, (byte) 0xA9, (byte) 0xAA, (byte) 0xAB,
- (byte) 0xAC, (byte) 0xAD, (byte) 0xAE, (byte) 0xAF,
-/* B- */ (byte) 0xB0, (byte) 0xB1, (byte) 0xB2, (byte) 0xB3, (byte) 0xB4, (byte) 0xB5,
- (byte) 0x40, (byte) 0x40, (byte) 0xB8, (byte) 0xB9, (byte) 0xBA, (byte) 0xBB,
- (byte) 0xBC, (byte) 0xBD, (byte) 0xBE, (byte) 0xBF,
-/* C- */ (byte) 0x40, (byte) 0x81, (byte) 0x82, (byte) 0x83, (byte) 0x84, (byte) 0x85,
- (byte) 0x86, (byte) 0x87, (byte) 0x88, (byte) 0x89, (byte) 0x40, (byte) 0xCB,
- (byte) 0x40, (byte) 0xCD, (byte) 0x40, (byte) 0xCF,
-/* D- */ (byte) 0x40, (byte) 0x91, (byte) 0x92, (byte) 0x93, (byte) 0x94, (byte) 0x95,
- (byte) 0x96, (byte) 0x97, (byte) 0x98, (byte) 0x99, (byte) 0xDA, (byte) 0xDB,
- (byte) 0xDC, (byte) 0xDD, (byte) 0xDE, (byte) 0xDF,
-/* E- */ (byte) 0x40, (byte) 0x40, (byte) 0xA2, (byte) 0xA3, (byte) 0xA4, (byte) 0xA5,
- (byte) 0xA6, (byte) 0xA7, (byte) 0xA8, (byte) 0xA9, (byte) 0xEA, (byte) 0xEB,
- (byte) 0x40, (byte) 0xED, (byte) 0xEE, (byte) 0xEF,
-/* F- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
- (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0xFB,
- (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0x40,};
-
-
- public String getLanguage() {
- return "ar";
- }
-
- }
-
- static class CharsetRecog_IBM420_ar_rtl extends CharsetRecog_IBM420_ar {
- private static int[] ngrams =
- {0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56,
- 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156,
- 0x56B158, 0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A,
- 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63,
- 0x584056, 0x624056, 0x6240AB, 0x6240B1, 0x6240BB, 0x6240CF, 0x634056,
- 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA,
- 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40, 0xBB4056, 0xBB5640,
- 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40,
- 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40,};
-
- public String getName() {
- return "IBM420_rtl";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = matchIBM420(det, ngrams, byteMap, (byte) 0x40);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
- }
-
- static class CharsetRecog_IBM420_ar_ltr extends CharsetRecog_IBM420_ar {
- private static int[] ngrams =
- {0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC,
- 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB,
- 0x40BDCF, 0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056,
- 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A,
- 0x5640B1, 0x5640BB, 0x5640BD, 0x5640BF, 0x5640DA, 0x5640DC, 0x565840,
- 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156,
- 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156, 0xB14062, 0xB15640,
- 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640,
- 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156};
-
- public String getName() {
- return "IBM420_ltr";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence = matchIBM420(det, ngrams, byteMap, (byte) 0x40);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
- }
-
- static abstract class CharsetRecog_EBCDIC_500 extends CharsetRecog_sbcs {
-
- // This maps EBCDIC 500 codepoints onto either space (not of interest), or a lower
- // case ISO_8859_1 number/letter/accented-letter codepoint for ngram matching
- // Because we map to ISO_8859_1, we can re-use the ngrams from those detectors
- // To avoid mis-detection, we skip many of the control characters in the 0x00-0x3f range
- protected static byte[] byteMap = {
-/* 0x00-0x07 */ (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00,
- (byte) 0x00, (byte) 0x00,
-/* 0x08-0x0f */ (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x10-0x17 */ (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x18-0x1f */ (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x20-0x27 */ (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x28-0x2f */ (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x00, (byte) 0x00,
-/* 0x30-0x37 */ (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00,
- (byte) 0x00, (byte) 0x00,
-/* 0x38-0x3f */ (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x00, (byte) 0x00,
- (byte) 0x00, (byte) 0x00,
-/* 0x40-0x47 */ (byte) 0x20, (byte) 0x20, (byte) 0xe2, (byte) 0xe4, (byte) 0xe0, (byte) 0xe1,
- (byte) 0xe3, (byte) 0xe5,
-/* 0x48-0x4f */ (byte) 0xe7, (byte) 0xf1, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x50-0x57 */ (byte) 0x20, (byte) 0xe9, (byte) 0xea, (byte) 0xeb, (byte) 0xe8, (byte) 0xed,
- (byte) 0xee, (byte) 0xef,
-/* 0x58-0x5f */ (byte) 0xec, (byte) 0xdf, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x60-0x67 */ (byte) 0x20, (byte) 0x20, (byte) 0xe2, (byte) 0xe4, (byte) 0xe0, (byte) 0xe1,
- (byte) 0xe3, (byte) 0xe5,
-/* 0x68-0x6f */ (byte) 0xe7, (byte) 0xf1, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x70-0x77 */ (byte) 0xf8, (byte) 0xe9, (byte) 0xea, (byte) 0xeb, (byte) 0xe8, (byte) 0xed,
- (byte) 0xee, (byte) 0xef,
-/* 0x78-0x7f */ (byte) 0xec, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x80-0x87 */ (byte) 0xd8, (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f',
- (byte) 'g',
-/* 0x88-0x8f */ (byte) 'h', (byte) 'i', (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0x90-0x97 */ (byte) 0x20, (byte) 'j', (byte) 'k', (byte) 'l', (byte) 'm', (byte) 'n', (byte) 'o',
- (byte) 'p',
-/* 0x98-0x9f */ (byte) 'q', (byte) 'r', (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0xa0-0xa7 */ (byte) 0x20, (byte) 0x20, (byte) 's', (byte) 't', (byte) 'u', (byte) 'v',
- (byte) 'w', (byte) 'x',
-/* 0xa8-0xaf */ (byte) 'y', (byte) 'z', (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0xb0-0xb7 */ (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0xb8-0xbf */ (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
- (byte) 0x20, (byte) 0x20,
-/* 0xc0-0xc7 */ (byte) 0x20, (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f',
- (byte) 'g',
-/* 0xc8-0xcf */ (byte) 'h', (byte) 'i', (byte) 0x20, (byte) 0xf4, (byte) 0xf6, (byte) 0xf2,
- (byte) 0xf3, (byte) 0xf5,
-/* 0xd0-0xd7 */ (byte) 0x20, (byte) 'j', (byte) 'k', (byte) 'l', (byte) 'm', (byte) 'n', (byte) 'o',
- (byte) 'p',
-/* 0xd8-0xdf */ (byte) 'q', (byte) 'r', (byte) 0x20, (byte) 0xfb, (byte) 0xfc, (byte) 0xf9,
- (byte) 0xfa, (byte) 0xff,
-/* 0xe0-0xe7 */ (byte) 0x20, (byte) 0x20, (byte) 's', (byte) 't', (byte) 'u', (byte) 'v',
- (byte) 'w', (byte) 'x',
-/* 0xe8-0xef */ (byte) 'y', (byte) 'z', (byte) 0x20, (byte) 0xf4, (byte) 0xf6, (byte) 0xf2,
- (byte) 0xf3, (byte) 0xf5,
-/* 0xf0-0xf7 */ (byte) '0', (byte) '1', (byte) '2', (byte) '3', (byte) '4', (byte) '5', (byte) '6',
- (byte) '7',
-/* 0xf8-0xff */ (byte) '8', (byte) '9', (byte) 0x20, (byte) 0xfb, (byte) 0xfc, (byte) 0xf9,
- (byte) 0xfa, (byte) 0x20,};
- private final int langIndex;
-
- protected CharsetRecog_EBCDIC_500(int langIndex) {
- this.langIndex = langIndex;
- }
-
- /**
- * @param lang language to find
- * @return the index into CharsetRecog_8859_1.ngrams_8859_1 that matches his language;
- * throws IllegalArgumentException if language can't be found
- */
- static int findLangIndex(String lang) {
- for (int i = 0; i < CharsetRecog_8859_1.ngrams_8859_1.length; i++) {
- NGramsPlusLang ngpl = CharsetRecog_8859_1.ngrams_8859_1[i];
- if (ngpl.fLang.equals(lang)) {
- return i;
- }
- }
- throw new IllegalArgumentException("can't find language: " + lang);
- }
-
- public String getName() {
- return "IBM500";
- }
-
- public CharsetMatch match(CharsetDetector det) {
- int confidence =
- match(det, CharsetRecog_8859_1.ngrams_8859_1[getLangIndex()].fNGrams, byteMap);
- return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
- }
-
- int getLangIndex() {
- return langIndex;
- }
-
-
- }
-
- //The EBCDIC codes were removed from ICU4js trunk as of at least July 26, 2016.
- static class CharsetRecog_EBCDIC_500_en extends CharsetRecog_EBCDIC_500 {
-
- CharsetRecog_EBCDIC_500_en() {
- super(findLangIndex("en"));
- }
-
- public String getLanguage() {
- return "en";
- }
-
-
- }
-
- static class CharsetRecog_EBCDIC_500_de extends CharsetRecog_EBCDIC_500 {
- CharsetRecog_EBCDIC_500_de() {
- super(findLangIndex("de"));
- }
-
- public String getLanguage() {
- return "de";
- }
-
- }
-
- static class CharsetRecog_EBCDIC_500_fr extends CharsetRecog_EBCDIC_500 {
- CharsetRecog_EBCDIC_500_fr() {
- super(findLangIndex("fr"));
- }
-
- public String getLanguage() {
- return "fr";
- }
- }
-
- static class CharsetRecog_EBCDIC_500_es extends CharsetRecog_EBCDIC_500 {
- CharsetRecog_EBCDIC_500_es() {
- super(findLangIndex("es"));
- }
-
- public String getLanguage() {
- return "es";
- }
- }
-
- static class CharsetRecog_EBCDIC_500_it extends CharsetRecog_EBCDIC_500 {
- CharsetRecog_EBCDIC_500_it() {
- super(findLangIndex("it"));
- }
-
- public String getLanguage() {
- return "it";
- }
- }
-
- static class CharsetRecog_EBCDIC_500_nl extends CharsetRecog_EBCDIC_500 {
- CharsetRecog_EBCDIC_500_nl() {
- super(findLangIndex("nl"));
- }
-
- public String getLanguage() {
- return "nl";
- }
-
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecognizer.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecognizer.java
deleted file mode 100644
index 77d137f66a..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/CharsetRecognizer.java
+++ /dev/null
@@ -1,53 +0,0 @@
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/**
- * ******************************************************************************
- * Copyright (C) 2005-2012, International Business Machines Corporation and *
- * others. All Rights Reserved. *
- * ******************************************************************************
- */
-package org.apache.tika.parser.txt;
-
-/**
- * Abstract class for recognizing a single charset.
- * Part of the implementation of ICU's CharsetDetector.
- * <p>
- * Each specific charset that can be recognized will have an instance
- * of some subclass of this class. All interaction between the overall
- * CharsetDetector and the stuff specific to an individual charset happens
- * via the interface provided here.
- * <p>
- * Instances of CharsetDetector DO NOT have or maintain
- * state pertaining to a specific match or detect operation.
- * The WILL be shared by multiple instances of CharsetDetector.
- * They encapsulate const charset-specific information.
- */
-abstract class CharsetRecognizer {
- /**
- * Get the IANA name of this charset.
- *
- * @return the charset name.
- */
- abstract String getName();
-
- /**
- * Get the ISO language code for this charset.
- *
- * @return the language code, or <code>null</code> if the language cannot be determined.
- */
- public String getLanguage() {
- return null;
- }
-
- /**
- * Test the match of this charset with the input text data
- * which is obtained via the CharsetDetector object.
- *
- * @param det The CharsetDetector, which contains the input text
- * to be checked for being in this charset.
- * @return A CharsetMatch object containing details of match
- * with this charset, or null if there was no match.
- */
- abstract CharsetMatch match(CharsetDetector det);
-
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
deleted file mode 100644
index 6f7f297cbe..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.txt;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.tika.config.ConfigDeserializer;
-import org.apache.tika.config.JsonConfig;
-import org.apache.tika.config.TikaComponent;
-import org.apache.tika.detect.EncodingDetector;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.utils.CharsetUtils;
-
-@TikaComponent(spi = false, name = "icu4j-encoding-detector")
-public class Icu4jEncodingDetector implements EncodingDetector {
-
- /**
- * Configuration class for JSON deserialization.
- */
- public static class Config implements Serializable {
- public boolean stripMarkup = false;
- public int markLimit = CharsetDetector.DEFAULT_MARK_LIMIT;
- public List<String> ignoreCharsets = new ArrayList<>();
-
- public boolean isStripMarkup() {
- return stripMarkup;
- }
-
- public void setStripMarkup(boolean stripMarkup) {
- this.stripMarkup = stripMarkup;
- }
-
- public int getMarkLimit() {
- return markLimit;
- }
-
- public void setMarkLimit(int markLimit) {
- this.markLimit = markLimit;
- }
-
- public List<String> getIgnoreCharsets() {
- return ignoreCharsets;
- }
-
- public void setIgnoreCharsets(List<String> ignoreCharsets) {
- this.ignoreCharsets = ignoreCharsets;
- }
- }
-
- private final Config defaultConfig;
- /**
- * Default constructor for SPI loading.
- */
- public Icu4jEncodingDetector() {
- defaultConfig = new Config();
- }
-
- /**
- * Constructor with explicit Config object.
- *
- * @param config the configuration
- */
- public Icu4jEncodingDetector(Config config) {
- defaultConfig = config;
- }
-
- /**
- * Constructor for JSON configuration.
- * Requires Jackson on the classpath.
- *
- * @param jsonConfig JSON configuration
- */
- public Icu4jEncodingDetector(JsonConfig jsonConfig) {
- this(ConfigDeserializer.buildConfig(jsonConfig, Config.class));
- }
-
-
- @Override
- public Charset detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException {
- if (tis == null) {
- return null;
- }
- //TODO: add runtime updates?
- Config config = defaultConfig;
-
- CharsetDetector detector = new CharsetDetector(config.markLimit);
-
- String incomingCharset = metadata.get(Metadata.CONTENT_ENCODING);
- String incomingType = metadata.get(Metadata.CONTENT_TYPE);
- if (incomingCharset == null && incomingType != null) {
- // TIKA-341: Use charset in content-type
- MediaType mt = MediaType.parse(incomingType);
- if (mt != null) {
- incomingCharset = mt.getParameters().get("charset");
- }
- }
-
- if (incomingCharset != null) {
- String cleaned = CharsetUtils.clean(incomingCharset);
- if (cleaned != null) {
- detector.setDeclaredEncoding(cleaned);
- } else {
- // TODO: log a warning?
- }
- }
-
- // TIKA-341 without enabling input filtering (stripping of tags)
- // short HTML tests don't work well
- detector.enableInputFilter(true);
-
- detector.setText(tis);
-
- for (CharsetMatch match : detector.detectAll()) {
- try {
- String n = match.getNormalizedName();
- if (config.ignoreCharsets.contains(n)) {
- continue;
- }
- return CharsetUtils.forName(match.getNormalizedName());
- } catch (IllegalArgumentException e) {
- // ignore
- }
- }
-
- return null;
- }
-
- public Config getDefaultConfig() {
- return defaultConfig;
- }
-
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
deleted file mode 100644
index 2e881f1a54..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.txt;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.charset.Charset;
-
-import org.apache.tika.config.ConfigDeserializer;
-import org.apache.tika.config.JsonConfig;
-import org.apache.tika.config.TikaComponent;
-import org.apache.tika.detect.EncodingDetector;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-
-@TikaComponent(spi = false)
-public class UniversalEncodingDetector implements EncodingDetector {
-
- private static final int BUFSIZE = 1024;
-
- private static final int DEFAULT_MARK_LIMIT = 16 * BUFSIZE;
-
- /**
- * Configuration class for JSON deserialization.
- */
- public static class Config implements Serializable {
- private int markLimit = DEFAULT_MARK_LIMIT;
-
- public int getMarkLimit() {
- return markLimit;
- }
-
- public void setMarkLimit(int markLimit) {
- this.markLimit = markLimit;
- }
- }
-
- private Config defaultConfig = new Config();
-
- /**
- * Default constructor for SPI loading.
- */
- public UniversalEncodingDetector() {
- }
-
- /**
- * Constructor with explicit Config object.
- *
- * @param config the configuration
- */
- public UniversalEncodingDetector(Config config) {
- this.defaultConfig = config;
- }
-
- /**
- * Constructor for JSON configuration.
- * Requires Jackson on the classpath.
- *
- * @param jsonConfig JSON configuration
- */
- public UniversalEncodingDetector(JsonConfig jsonConfig) {
- this(ConfigDeserializer.buildConfig(jsonConfig, Config.class));
- }
-
- public Charset detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException {
- if (tis == null) {
- return null;
- }
-
- int markLimit = defaultConfig.getMarkLimit();
- tis.mark(markLimit);
- try {
- UniversalEncodingListener listener = new UniversalEncodingListener(metadata);
-
- byte[] b = new byte[BUFSIZE];
- int n = 0;
- int m = tis.read(b);
- while (m != -1 && n < markLimit && !listener.isDone()) {
- n += m;
- listener.handleData(b, 0, m);
- m = tis.read(b, 0, Math.min(b.length, markLimit - n));
- }
-
- return listener.dataEnd();
- } catch (LinkageError e) {
- return null; // juniversalchardet is not available
- } finally {
- tis.reset();
- }
- }
-
- public Config getDefaultConfig() {
- return defaultConfig;
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingListener.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingListener.java
deleted file mode 100644
index 8458aabe8d..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingListener.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.txt;
-
-import java.nio.charset.Charset;
-
-import org.mozilla.universalchardet.CharsetListener;
-import org.mozilla.universalchardet.Constants;
-import org.mozilla.universalchardet.UniversalDetector;
-
-import org.apache.tika.detect.TextStatistics;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.utils.CharsetUtils;
-
-/**
- * Helper class used by {@link UniversalEncodingDetector} to access the
- * <code>juniversalchardet</code> detection logic.
- */
-class UniversalEncodingListener implements CharsetListener {
-
- private static final String CHARSET_ISO_8859_1 = "ISO-8859-1";
-
- private static final String CHARSET_ISO_8859_15 = "ISO-8859-15";
-
- private final TextStatistics statistics = new TextStatistics();
-
- private final UniversalDetector detector = new UniversalDetector(this);
-
- private String hint = null;
-
- private Charset charset = null;
-
- public UniversalEncodingListener(Metadata metadata) {
- MediaType type = MediaType.parse(metadata.get(Metadata.CONTENT_TYPE));
- if (type != null) {
- hint = type.getParameters().get("charset");
- }
- if (hint == null) {
- hint = metadata.get(Metadata.CONTENT_ENCODING);
- }
- }
-
- public void report(String name) {
- if (Constants.CHARSET_WINDOWS_1252.equals(name)) {
- if (hint != null) {
- // Use the encoding hint when available
- name = hint;
- } else if (hasNonexistentHexInCharsetWindows1252() || statistics.count('\r') == 0) {
- // If it has nonexistent hex value in charset windows-1252 or
- // If there are no CR(LF)s, then the encoding is more
- // likely to be ISO-8859-1(5) than windows-1252
- if (statistics.count(0xa4) > 0) { // currency/euro sign
- // The general currency sign is hardly ever used in
- // ISO-8859-1, so it's more likely that we're dealing
- // with ISO-8859-15, where the character is used for
- // the euro symbol, which is more commonly used.
- name = CHARSET_ISO_8859_15;
- } else {
- name = CHARSET_ISO_8859_1;
- }
- }
- }
- try {
- this.charset = CharsetUtils.forName(name);
- } catch (IllegalArgumentException e) {
- // ignore
- }
- }
-
- public boolean isDone() {
- return detector.isDone();
- }
-
- public void handleData(byte[] buf, int offset, int length) {
- statistics.addData(buf, offset, length);
- detector.handleData(buf, offset, length);
- }
-
- public Charset dataEnd() {
- detector.dataEnd();
- if (charset == null && statistics.isMostlyAscii()) {
- report(Constants.CHARSET_WINDOWS_1252);
- }
- return charset;
- }
-
- /*
- * hex value 0x81, 0x8d, 0x8f, 0x90, 0x9d don't exist in charset windows-1252.
- * If these value's count > 0, return true
- * */
- private Boolean hasNonexistentHexInCharsetWindows1252() {
- return (statistics.count(0x81) > 0 || statistics.count(0x8d) > 0 ||
- statistics.count(0x8f) > 0 || statistics.count(0x90) > 0 ||
- statistics.count(0x9d) > 0);
- }
-
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector
deleted file mode 100644
index 506ca15f55..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/resources/META-INF/services/org.apache.tika.detect.EncodingDetector
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# TODO: This is a manual service file because EncodingDetectors are not re-sorted at runtime
-# (unlike Parsers/Detectors which call ServiceLoaderUtils.sortLoadedClasses()).
-# Order matters here - first detector that returns non-null wins.
-# Need to implement a runtime priority mechanism for proper ordering across modules.
-org.apache.tika.parser.txt.UniversalEncodingDetector
-org.apache.tika.parser.txt.Icu4jEncodingDetector
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/BOMDetectorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/BOMDetectorTest.java
deleted file mode 100644
index a18dcb4602..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/BOMDetectorTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.txt;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-
-import org.apache.commons.io.ByteOrderMark;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.BOMInputStream;
-import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.detect.EncodingDetector;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-
-public class BOMDetectorTest extends TikaTest {
- @Test
- public void testBasic() throws Exception {
- EncodingDetector detector = new BOMDetector();
- for (ByteOrderMark bom : new ByteOrderMark[]{
- ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE,
- ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_32BE, ByteOrderMark.UTF_32LE
- }) {
- UnsynchronizedByteArrayOutputStream bos = createStream(bom);
- try (BOMInputStream bomInputStream =
- BOMInputStream.builder().
- setInputStream(UnsynchronizedByteArrayInputStream.builder().setByteArray(bos.toByteArray()).get()).
- setByteOrderMarks(ByteOrderMark.UTF_8, ByteOrderMark.UTF_32BE, ByteOrderMark.UTF_32LE,
- ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE).get()) {
- assertEquals(bom, bomInputStream.getBOM());
- }
- try (TikaInputStream tis = TikaInputStream.get(bos.toByteArray())) {
- assertEquals(Charset.forName(bom.getCharsetName()), detector.detect(tis, new Metadata(), new ParseContext()));
- int cnt = 0;
- int c = tis.read();
- while (c > -1) {
- cnt++;
- c = tis.read();
- }
- assertEquals(100 + bom.getBytes().length, cnt);
- }
- }
- }
-
- @Test
- public void testShort() throws Exception {
- EncodingDetector detector = new BOMDetector();
- for (ByteOrderMark bom : new ByteOrderMark[] {
- ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE,
- ByteOrderMark.UTF_32LE
- }) {
- byte[] bytes = new byte[3];
- System.arraycopy(bom.getBytes(), 0, bytes, 0, 1);
- bytes[1] = (byte)32;
- bytes[2] = (byte)32;
- try (TikaInputStream tis = TikaInputStream.get(bytes)) {
- assertNull(detector.detect(tis, new Metadata(), new ParseContext()));
- }
- }
- }
-
- private UnsynchronizedByteArrayOutputStream createStream(ByteOrderMark bom) throws IOException {
- UnsynchronizedByteArrayOutputStream bos = UnsynchronizedByteArrayOutputStream.builder().get();
- IOUtils.write(bom.getBytes(), bos);
- for (int i = 0; i < 100; i++) {
- bos.write(' ');
- }
- return bos;
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java
deleted file mode 100644
index 3cf4b435c7..0000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/CharsetDetectorTest.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.txt;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.Reader;
-import java.nio.file.Files;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.config.loader.TikaLoader;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.AutoDetectParser;
-
-public class CharsetDetectorTest extends TikaTest {
-
- @Test
- public void testTagDropper() throws IOException {
- try (TikaInputStream tis = getResourceAsStream("/test-documents/resume.html")) {
- CharsetDetector detector = new CharsetDetector();
- detector.enableInputFilter(true);
- detector.setText(tis);
- CharsetMatch[] matches = detector.detectAll();
- CharsetMatch mm = null;
- for (CharsetMatch m : matches) {
- if (mm == null || mm.getConfidence() < m.getConfidence()) {
- mm = m;
- }
- }
- assertTrue(mm != null);
- assertEquals("UTF-8", mm.getName());
- }
- }
-
- /* https://issues.apache.org/jira/browse/TIKA-1248
- * Verify empty or null declaredEncoding doesn't cause an exception
- *
- */
-
- @Test
- public void testEmptyOrNullDeclaredCharset() throws IOException {
- try (TikaInputStream tis = getResourceAsStream("/test-documents/resume.html")) {
- CharsetDetector detector = new CharsetDetector();
- Reader reader = detector.getReader(tis, null);
- assertTrue(reader.ready());
-
- reader = detector.getReader(tis, "");
- assertTrue(reader.ready());
- }
- }
-
- @Test
- public void testWin125XHeuristics() throws Exception {
- //TIKA-2219
- CharsetDetector detector = new CharsetDetector();
- try (TikaInputStream tis = getResourceAsStream("/test-documents/testTXT_win-1252.txt")) {
- detector.setText(tis);
- }
- CharsetMatch charset = detector.detect();
- assertEquals("windows-1252", charset.getName());
- }
-
- @Test
- public void testSetTextConsistency() throws Exception {
- //TIKA-2475
- File file = getResourceAsFile("/test-documents/multi-language.txt");
- byte[] fileBytes = Files.readAllBytes(file.toPath());
- TikaInputStream fileStream = TikaInputStream.get(fileBytes);
-
- CharsetDetector fromBytesDetector = new CharsetDetector();
- fromBytesDetector.setText(fileBytes);
-
- CharsetDetector fromStreamDetector = new CharsetDetector();
- fromStreamDetector.setText(fileStream);
-
- assertEquals("ISO-8859-1", fromBytesDetector.detect().getName());
- assertEquals("ISO-8859-1", fromStreamDetector.detect().getName());
- }
-
- @Test
- public void testZeroLength() throws Exception {
- CharsetDetector detector = new CharsetDetector();
- detector.setText(new byte[0]);
- //charset detector returns "UTF-8" when there's no data
- assertEquals("UTF-8", detector.detect().getName());
- }
-
- @Test
- public void testLengthResetCorrectly() throws IOException {
- //test that the underlying array.length is reset correctly
- //first fill the buffer with windows-1256
-
- String computer = "\u0627\u0644\u062D\u0627\u0633\u0648\u0628";
- StringBuilder sb = new StringBuilder();
- CharsetDetector detector = new CharsetDetector();
- for (int i = 0; i < 5000; i++) {
- sb.append(computer);
- }
- detector.setText(sb.toString().getBytes("windows-1256"));
- assertEquals("windows-1256", detector.detect().getName());
-
- sb.setLength(0);
- for (int i = 0; i < 5; i++) {
- sb.append(computer);
- }
- //then fill a small part of the buffer with UTF-8
- detector.setText(sb.toString().getBytes("UTF-8"));
- assertEquals("UTF-8", detector.detect().getName());
- }
-
- @Test
- public void testIgnoreCharset() throws Exception {
- //TIKA-3516, TIKA-3525, TIKA-1236
- AutoDetectParser parser = (AutoDetectParser) TikaLoader.load(
- getConfigPath(CharsetDetectorTest.class, "tika-config-ignore-charset.json"))
- .loadAutoDetectParser();
-
- Metadata m = new Metadata();
- m.set(TikaCoreProperties.RESOURCE_NAME_KEY, "texty-text.txt");
- assertContains("ACTIVE AGE", getXML("testIgnoreCharset.txt",
- parser, m).xml);
-
- m = new Metadata();
- m.set(TikaCoreProperties.RESOURCE_NAME_KEY, "texty-text.txt");
- assertContains("Please check your email", getXML("test_ignore_IBM420.html",
- parser, m).xml);
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml
index 6363afc8a2..6bad4e0cbc 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml
@@ -179,6 +179,36 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-encoding-detector-mojibuster</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-encoding-detector-html</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-encoding-detector-charsoup</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-encoding-detector-icu4j</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-encoding-detector-universal</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-serialization</artifactId>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
index 2524ef404d..bbf2a2cafc 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
@@ -17,20 +17,18 @@
package org.apache.tika.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
-import org.apache.tika.Tika;
import org.apache.tika.TikaLoaderHelper;
import org.apache.tika.TikaTest;
import org.apache.tika.config.loader.TikaLoader;
@@ -38,16 +36,18 @@ import org.apache.tika.detect.CompositeEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.detect.MetaEncodingDetector;
import org.apache.tika.detect.OverrideEncodingDetector;
+import org.apache.tika.detect.WideUnicodeDetector;
import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.ml.chardetect.MojibusterEncodingDetector;
import org.apache.tika.parser.AbstractEncodingDetectorParser;
import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParserDecorator;
import org.apache.tika.parser.html.HtmlEncodingDetector;
+import org.apache.tika.parser.html.charsetdetector.StandardHtmlEncodingDetector;
import org.apache.tika.parser.txt.Icu4jEncodingDetector;
import org.apache.tika.parser.txt.TXTParser;
import org.apache.tika.parser.txt.UniversalEncodingDetector;
@@ -59,12 +59,16 @@ public class TikaEncodingDetectorTest extends TikaTest {
EncodingDetector detector = TikaLoader.loadDefault().loadEncodingDetectors();
assertTrue(detector instanceof CompositeEncodingDetector);
List<EncodingDetector> detectors = ((CompositeEncodingDetector) detector).getDetectors();
- // 3 base detectors + CharSoupEncodingDetector (MetaEncodingDetector)
+ // 3 base detectors (WideUnicode, ML, StandardHtml) + CharSoupEncodingDetector (MetaEncodingDetector)
assertEquals(4, detectors.size());
- assertTrue(detectors.get(0) instanceof HtmlEncodingDetector);
- assertTrue(detectors.get(1) instanceof UniversalEncodingDetector);
- assertTrue(detectors.get(2) instanceof Icu4jEncodingDetector);
+ // meta detector is always last (partitioned by CompositeEncodingDetector)
assertTrue(detectors.get(3) instanceof MetaEncodingDetector);
+ // base detectors — sorted by full class name; check by type
+ Set<Class<?>> baseClasses = detectors.subList(0, 3).stream()
+ .map(Object::getClass).collect(Collectors.toSet());
+ assertTrue(baseClasses.contains(WideUnicodeDetector.class));
+ assertTrue(baseClasses.contains(MojibusterEncodingDetector.class));
+ assertTrue(baseClasses.contains(StandardHtmlEncodingDetector.class));
}
@Test
@@ -81,9 +85,12 @@ public class TikaEncodingDetectorTest extends TikaTest {
assertTrue(detector1 instanceof CompositeEncodingDetector);
List<EncodingDetector> detectors1Children =
((CompositeEncodingDetector) detector1).getDetectors();
+ // WideUnicode + ML base detectors + CharSoup meta (html excluded)
assertEquals(3, detectors1Children.size());
- assertTrue(detectors1Children.get(0) instanceof UniversalEncodingDetector);
- assertTrue(detectors1Children.get(1) instanceof Icu4jEncodingDetector);
+ Set<Class<?>> innerClasses = detectors1Children.subList(0, 2).stream()
+ .map(Object::getClass).collect(Collectors.toSet());
+ assertTrue(innerClasses.contains(WideUnicodeDetector.class));
+ assertTrue(innerClasses.contains(MojibusterEncodingDetector.class));
assertTrue(detectors1Children.get(2) instanceof MetaEncodingDetector);
assertTrue(detectors.get(1) instanceof OverrideEncodingDetector);
@@ -111,29 +118,17 @@ public class TikaEncodingDetectorTest extends TikaTest {
@Test
public void testEncodingDetectorConfigurability() throws Exception {
+ // CP500 (EBCDIC) is now detected by MojibusterEncodingDetector's structural IBM500 rule,
+ // so the default config should handle it successfully.
+ Metadata metadata = getXML("english.cp500.txt").metadata;
+ assertNotNull(metadata.get(TikaCoreProperties.DETECTED_ENCODING));
+
+ // Excluding ICU4J from the config (which is already not in the default chain)
+ // should still work — ML handles EBCDIC detection.
TikaLoader tikaLoader = TikaLoaderHelper.getLoader("TIKA-2273-no-icu4j-encoding-detector.json");
Parser p = tikaLoader.loadAutoDetectParser();
-
- try {
- Metadata metadata = getXML("english.cp500.txt", p).metadata;
- fail("can't detect w/out ICU");
- } catch (TikaException e) {
- assertContains("Failed to detect", e.getMessage());
- }
-
- Tika tika = new Tika(tikaLoader.loadDetectors(), tikaLoader.loadAutoDetectParser());
- Path tmp = null;
- try {
- tmp = Files.createTempFile("tika-encoding-test", ".txt");
- Files.copy(getResourceAsStream("/test-documents/english.cp500.txt"), tmp,
- StandardCopyOption.REPLACE_EXISTING);
- String txt = tika.parseToString(tmp);
- fail("can't detect w/out ICU");
- } catch (TikaException e) {
- assertContains("Failed to detect", e.getMessage());
- } finally {
- Files.delete(tmp);
- }
+ metadata = getXML("english.cp500.txt", p).metadata;
+ assertNotNull(metadata.get(TikaCoreProperties.DETECTED_ENCODING));
}
@@ -182,21 +177,18 @@ public class TikaEncodingDetectorTest extends TikaTest {
((AbstractEncodingDetectorParser) encodingDetectingParser)
.getEncodingDetector();
assertTrue(encodingDetector instanceof CompositeEncodingDetector);
- // HtmlEncodingDetector, UniversalEncodingDetector, CharSoupEncodingDetector
- assertEquals(3, ((CompositeEncodingDetector) encodingDetector).getDetectors().size());
+ // WideUnicode, ML, Html base detectors + CharSoup MetaEncodingDetector
+ // (ICU4J is excluded but was already not in the default chain)
+ assertEquals(4, ((CompositeEncodingDetector) encodingDetector).getDetectors().size());
for (EncodingDetector child : ((CompositeEncodingDetector) encodingDetector)
.getDetectors()) {
assertNotContained("cu4j", child.getClass().getCanonicalName());
}
}
- //also just make sure this is still true
- try {
- Metadata metadata = getXML("english.cp500.txt", p).metadata;
- fail("can't detect w/out ICU");
- } catch (TikaException e) {
- assertContains("Failed to detect", e.getMessage());
- }
+ // ML handles EBCDIC (IBM500) via structural rules, so CP500 is detectable
+ Metadata metadata = getXML("english.cp500.txt", p).metadata;
+ assertNotNull(metadata.get(TikaCoreProperties.DETECTED_ENCODING));
}
@@ -279,11 +271,13 @@ public class TikaEncodingDetectorTest extends TikaTest {
assertTrue(detector instanceof CompositeEncodingDetector);
List<EncodingDetector> detectors =
((CompositeEncodingDetector) detector).getDetectors();
- // 3 base detectors, no MetaEncodingDetector
+ // 3 base detectors (WideUnicode + ML + StandardHtml), no MetaEncodingDetector
assertEquals(3, detectors.size());
- assertTrue(detectors.get(0) instanceof HtmlEncodingDetector);
- assertTrue(detectors.get(1) instanceof UniversalEncodingDetector);
- assertTrue(detectors.get(2) instanceof Icu4jEncodingDetector);
+ Set<Class<?>> excludedCharSoupClasses = detectors.stream()
+ .map(Object::getClass).collect(Collectors.toSet());
+ assertTrue(excludedCharSoupClasses.contains(WideUnicodeDetector.class));
+ assertTrue(excludedCharSoupClasses.contains(MojibusterEncodingDetector.class));
+ assertTrue(excludedCharSoupClasses.contains(StandardHtmlEncodingDetector.class));
for (EncodingDetector d : detectors) {
assertNotContained("CharSoup", d.getClass().getSimpleName());
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/TIKA-2273-exclude-encoding-detector-default.json b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/TIKA-2273-exclude-encoding-detector-default.json
index 240924a28c..78a813b580 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/TIKA-2273-exclude-encoding-detector-default.json
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/TIKA-2273-exclude-encoding-detector-default.json
@@ -4,7 +4,7 @@
{
"default-encoding-detector": {
"exclude": [
- "html-encoding-detector"
+ "standard-html-encoding-detector"
]
}
},